[llvm-branch-commits] [llvm] [MachineLICM] Track peak register pressure when hoisting (PR #242431)

Shilei Tian via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Sat Nov 14 10:16:57 PST 2026


https://github.com/shiltian updated https://github.com/llvm/llvm-project/pull/242431

>From a472f209f39eb101dfe06f3607d697093edf09a0 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i at tianshilei.me>
Date: Fri, 13 Nov 2026 13:25:38 -0500
Subject: [PATCH] [MachineLICM] Track peak register pressure when hoisting

The pressure check in MachineLICM only looks at the estimates saved in
BackTrace. These record pressure at block boundaries, so they can miss an
earlier peak in the current BB or an ancestor BB. If pressure falls before the
candidate, the check can allow it to move even though its def then needs to stay
live across that earlier peak. This can push pressure beyond the limit and cause
spills.

This PR adds PeakPressure alongside RegPressure and saves the previous peak with
the existing boundary estimate in BackTrace. The pressure check considers the
current BB's peak and both saved estimates. Normal hoists adjust the current and
saved peaks, and scope exit restores the previous peak. Preheader scanning skips
peak updates and seeds the starting pressure from its final estimate.

Fixes LCOMPILER-3233.

This PR was assisted by AI but I reviewed all the changes.
---
 llvm/lib/CodeGen/MachineLICM.cpp              |  39 +-
 .../atomic_optimizations_global_pointer.ll    |  22 +-
 .../CodeGen/AMDGPU/licm-regpressure-peak.mir  | 604 +++++++++++++
 ...ite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll | 126 ++-
 llvm/test/CodeGen/AVR/bug-81911.ll            |  42 +-
 .../CodeGen/Hexagon/order-stack-object.ll     |   2 +-
 .../NVPTX/atomicrmw-elementwise-sm60.ll       | 826 +++++++++---------
 .../NVPTX/atomicrmw-elementwise-sm70.ll       | 826 +++++++++---------
 .../NVPTX/atomicrmw-elementwise-sm90.ll       | 130 +--
 llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll     | 112 +--
 llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll     | 112 +--
 llvm/test/CodeGen/PowerPC/pr52894-32bit.ll    |  23 +-
 llvm/test/CodeGen/RISCV/rvv/pr165232.ll       | 182 ++--
 .../CodeGen/Thumb2/mve-gather-increment.ll    | 490 +++++------
 llvm/test/CodeGen/X86/combine-pmuldq.ll       |  80 +-
 .../X86/vector-shift-by-select-loop.ll        |  68 +-
 .../LoopStrengthReduce/X86/ivchain-X86.ll     |  20 +-
 17 files changed, 2159 insertions(+), 1545 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/licm-regpressure-peak.mir

diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index 50d910f3b74cab..727849b2e8eb24 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -163,12 +163,14 @@ namespace {
     // Track 'estimated' register pressure.
     SmallDenseSet<Register> RegSeen;
     SmallVector<unsigned, 8> RegPressure;
+    // Maximum estimate in the current block's scanned prefix.
+    SmallVector<unsigned, 8> PeakPressure;
 
     // Register pressure "limit" per register pressure set. If the pressure
     // is higher than the limit, then it's considered high.
     SmallVector<unsigned, 8> RegLimit;
 
-    // Register pressure on path leading from loop preheader to current BB.
+    // Peak register pressure estimates saved from enclosing scopes.
     SmallVector<SmallVector<unsigned, 8>, 16> BackTrace;
 
     // For each opcode per preheader, keep a list of potential CSE instructions.
@@ -195,6 +197,7 @@ namespace {
     void releaseMemory() {
       RegSeen.clear();
       RegPressure.clear();
+      PeakPressure.clear();
       RegLimit.clear();
       BackTrace.clear();
       CSEMap.clear();
@@ -408,6 +411,7 @@ bool MachineLICMImpl::run(MachineFunction &MF) {
     // Estimate register pressure during pre-regalloc pass.
     unsigned NumRPS = TRI->getNumRegPressureSets();
     RegPressure.resize(NumRPS);
+    PeakPressure.resize(NumRPS);
     llvm::fill(RegPressure, 0);
     RegLimit.resize(NumRPS);
     for (unsigned i = 0, e = NumRPS; i != e; ++i)
@@ -787,12 +791,14 @@ bool MachineLICMImpl::IsGuaranteedToExecute(MachineBasicBlock *BB,
 void MachineLICMImpl::EnterScope(MachineBasicBlock *MBB) {
   LLVM_DEBUG(dbgs() << "Entering " << printMBBReference(*MBB) << '\n');
 
-  // Remember livein register pressure.
-  BackTrace.push_back(RegPressure);
+  // Save the previous scope's peak, then start this block's prefix peak.
+  BackTrace.push_back(PeakPressure);
+  PeakPressure = RegPressure;
 }
 
 void MachineLICMImpl::ExitScope(MachineBasicBlock *MBB) {
   LLVM_DEBUG(dbgs() << "Exiting " << printMBBReference(*MBB) << '\n');
+  PeakPressure = std::move(BackTrace.back());
   BackTrace.pop_back();
 }
 
@@ -938,6 +944,10 @@ void MachineLICMImpl::InitRegPressure(MachineBasicBlock *BB) {
 
   for (const MachineInstr &MI : *BB)
     UpdateRegPressure(&MI, /*ConsiderUnseenAsDef=*/true);
+
+  // Seed the loop-entry peak. Hoists inserted before the preheader terminators
+  // do not extend live ranges across earlier preheader instructions.
+  PeakPressure = RegPressure;
 }
 
 /// Update estimate of register pressure after the specified instruction.
@@ -949,6 +959,8 @@ void MachineLICMImpl::UpdateRegPressure(const MachineInstr *MI,
       RegPressure[Class] = 0;
     else
       RegPressure[Class] += Weight;
+
+    PeakPressure[Class] = std::max(PeakPressure[Class], RegPressure[Class]);
   }
 }
 
@@ -1237,9 +1249,13 @@ bool MachineLICMImpl::CanCauseHighRegPressure(
     if (CheapInstr && !HoistCheapInsts)
       return true;
 
-    for (const auto &RP : BackTrace)
+    if (static_cast<int>(PeakPressure[Class]) + Weight >= Limit)
+      return true;
+
+    for (const SmallVector<unsigned, 8> &RP : BackTrace) {
       if (static_cast<int>(RP[Class]) + Weight >= Limit)
         return true;
+    }
   }
 
   return false;
@@ -1255,9 +1271,18 @@ void MachineLICMImpl::UpdateBackTraceRegPressure(const MachineInstr *MI) {
                                /*ConsiderUnseenAsDef=*/false);
 
   // Update register pressure of blocks from loop header to current block.
-  for (auto &RP : BackTrace)
-    for (const auto &[Class, Weight] : Cost)
-      RP[Class] += Weight;
+  for (const auto &[Class, Weight] : Cost) {
+    auto AdjustPeak = [Weight = Weight](unsigned &Peak) {
+      if (static_cast<int>(Peak) < -Weight)
+        Peak = 0;
+      else
+        Peak += Weight;
+    };
+
+    AdjustPeak(PeakPressure[Class]);
+    for (SmallVector<unsigned, 8> &RP : BackTrace)
+      AdjustPeak(RP[Class]);
+  }
 }
 
 /// Return true if it is potentially profitable to hoist the given loop
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index cd4aceed9df825..022ab83e6bcd32 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -4742,8 +4742,8 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7LESS-NEXT:    s_mov_b32 s6, -1
-; GFX7LESS-NEXT:    s_mov_b32 s4, s2
 ; GFX7LESS-NEXT:    s_mov_b32 s5, s3
+; GFX7LESS-NEXT:    s_mov_b32 s4, s2
 ; GFX7LESS-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v4, v0
@@ -6505,8 +6505,8 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, s4
 ; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX7LESS-NEXT:    s_mov_b32 s6, -1
-; GFX7LESS-NEXT:    s_mov_b32 s4, s2
 ; GFX7LESS-NEXT:    s_mov_b32 s5, s3
+; GFX7LESS-NEXT:    s_mov_b32 s4, s2
 ; GFX7LESS-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v0
@@ -7121,14 +7121,14 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, s4
 ; GFX7LESS-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX7LESS-NEXT:    s_mov_b32 s6, -1
-; GFX7LESS-NEXT:    s_mov_b32 s5, s3
-; GFX7LESS-NEXT:    s_mov_b32 s4, s2
 ; GFX7LESS-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v1
 ; GFX7LESS-NEXT:    v_subrev_i32_e32 v6, vcc, s14, v8
 ; GFX7LESS-NEXT:    v_subb_u32_e32 v7, vcc, v9, v5, vcc
+; GFX7LESS-NEXT:    s_mov_b32 s4, s2
+; GFX7LESS-NEXT:    s_mov_b32 s5, s3
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v0, v6
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v1, v7
@@ -7760,16 +7760,16 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX7LESS_ITERATIVE-NEXT:    s_cbranch_execz .LBB11_6
 ; GFX7LESS_ITERATIVE-NEXT:  ; %bb.3:
 ; GFX7LESS_ITERATIVE-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS_ITERATIVE-NEXT:    s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX7LESS_ITERATIVE-NEXT:    s_load_dwordx2 s[14:15], s[2:3], 0x0
 ; GFX7LESS_ITERATIVE-NEXT:    s_mov_b64 s[12:13], 0
 ; GFX7LESS_ITERATIVE-NEXT:    v_mov_b32_e32 v6, s9
+; GFX7LESS_ITERATIVE-NEXT:    s_mov_b32 s5, s3
 ; GFX7LESS_ITERATIVE-NEXT:    s_mov_b32 s7, 0xf000
-; GFX7LESS_ITERATIVE-NEXT:    s_mov_b32 s6, -1
 ; GFX7LESS_ITERATIVE-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS_ITERATIVE-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7LESS_ITERATIVE-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7LESS_ITERATIVE-NEXT:    v_mov_b32_e32 v0, s14
+; GFX7LESS_ITERATIVE-NEXT:    v_mov_b32_e32 v1, s15
+; GFX7LESS_ITERATIVE-NEXT:    s_mov_b32 s6, -1
 ; GFX7LESS_ITERATIVE-NEXT:    s_mov_b32 s4, s2
-; GFX7LESS_ITERATIVE-NEXT:    s_mov_b32 s5, s3
 ; GFX7LESS_ITERATIVE-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX7LESS_ITERATIVE-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS_ITERATIVE-NEXT:    v_mov_b32_e32 v9, v0
@@ -8453,8 +8453,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX8_DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8_DPP-NEXT:    v_mov_b32_e32 v7, s5
 ; GFX8_DPP-NEXT:    v_mov_b32_e32 v6, s4
-; GFX8_DPP-NEXT:    s_mov_b32 s4, s2
 ; GFX8_DPP-NEXT:    s_mov_b32 s5, s3
+; GFX8_DPP-NEXT:    s_mov_b32 s4, s2
 ; GFX8_DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX8_DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX8_DPP-NEXT:    v_mov_b32_e32 v10, v6
@@ -16100,8 +16100,8 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
 ; GFX7LESS-NEXT:    s_lshl_b32 s5, s5, 16
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s5
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s4
-; GFX7LESS-NEXT:    s_mov_b32 s4, s2
 ; GFX7LESS-NEXT:    s_mov_b32 s5, s3
+; GFX7LESS-NEXT:    s_mov_b32 s4, s2
 ; GFX7LESS-NEXT:  .LBB21_1: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/licm-regpressure-peak.mir b/llvm/test/CodeGen/AMDGPU/licm-regpressure-peak.mir
new file mode 100644
index 00000000000000..de95e1a3c1002c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/licm-regpressure-peak.mir
@@ -0,0 +1,604 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.00 -passes=early-machinelicm -stress-regalloc=6 -o - %s | FileCheck %s --check-prefixes=CHECK,LIMIT6
+# RUN: llc -mtriple=amdgpu9.00 -passes=early-machinelicm -stress-regalloc=8 -o - %s | FileCheck %s --check-prefixes=CHECK,LIMIT8
+# RUN: llc -mtriple=amdgpu9.00 -passes=early-machinelicm -stress-regalloc=9 -o - %s | FileCheck %s --check-prefixes=CHECK,LIMIT9
+
+# The load raises estimated VGPR pressure to 6. Its result dies before the
+# invariant COPY. The COPY stays at limit 6 and moves at limits 8 and 9.
+---
+name: earlier_pressure_peak
+tracksRegLiveness: true
+body: |
+  ; LIMIT6-LABEL: name: earlier_pressure_peak
+  ; LIMIT6: bb.0:
+  ; LIMIT6-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT6-NEXT:   liveins: $vgpr0, $sgpr0
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; LIMIT6-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT6-NEXT:   S_BRANCH %bb.1
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT: bb.1:
+  ; LIMIT6-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %3, %bb.1
+  ; LIMIT6-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT6-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT6-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; LIMIT6-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY2]], implicit $exec
+  ; LIMIT6-NEXT:   S_BRANCH %bb.1
+  ;
+  ; LIMIT8-LABEL: name: earlier_pressure_peak
+  ; LIMIT8: bb.0:
+  ; LIMIT8-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT8-NEXT:   liveins: $vgpr0, $sgpr0
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; LIMIT8-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT8-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; LIMIT8-NEXT:   S_BRANCH %bb.1
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT: bb.1:
+  ; LIMIT8-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %3, %bb.1
+  ; LIMIT8-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT8-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT8-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY2]], implicit $exec
+  ; LIMIT8-NEXT:   S_BRANCH %bb.1
+  ;
+  ; LIMIT9-LABEL: name: earlier_pressure_peak
+  ; LIMIT9: bb.0:
+  ; LIMIT9-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT9-NEXT:   liveins: $vgpr0, $sgpr0
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; LIMIT9-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT9-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; LIMIT9-NEXT:   S_BRANCH %bb.1
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT: bb.1:
+  ; LIMIT9-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %3, %bb.1
+  ; LIMIT9-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT9-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT9-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY2]], implicit $exec
+  ; LIMIT9-NEXT:   S_BRANCH %bb.1
+  bb.0:
+    liveins: $vgpr0, $sgpr0
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:sgpr_32 = COPY $sgpr0
+    S_BRANCH %bb.1
+
+  bb.1:
+    %2:vgpr_32 = PHI %0, %bb.0, %5, %bb.1
+    %3:vreg_128 = DS_READ_B128_gfx9 %2, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+    DS_WRITE_B128_gfx9 %2, %3, 16, 0, implicit $exec :: (store (s128), addrspace 3)
+    %4:vgpr_32 = COPY %1
+    %5:vgpr_32 = V_ADD_U32_e32 %2, %4, implicit $exec
+    S_BRANCH %bb.1
+...
+
+# The temporary dies in the loop header. The COPY's block starts at pressure 2,
+# but a hoist must also check the header's retained peak of 6. The COPY stays
+# at limit 6 and moves at limits 8 and 9.
+---
+name: ancestor_pressure_peak
+tracksRegLiveness: true
+body: |
+  ; LIMIT6-LABEL: name: ancestor_pressure_peak
+  ; LIMIT6: bb.0:
+  ; LIMIT6-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT6-NEXT:   liveins: $vgpr0, $sgpr0
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; LIMIT6-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT6-NEXT:   S_BRANCH %bb.1
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT: bb.1:
+  ; LIMIT6-NEXT:   successors: %bb.2(0x80000000)
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %3, %bb.2
+  ; LIMIT6-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT6-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT6-NEXT:   S_BRANCH %bb.2
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT: bb.2:
+  ; LIMIT6-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; LIMIT6-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY2]], implicit $exec
+  ; LIMIT6-NEXT:   S_BRANCH %bb.1
+  ;
+  ; LIMIT8-LABEL: name: ancestor_pressure_peak
+  ; LIMIT8: bb.0:
+  ; LIMIT8-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT8-NEXT:   liveins: $vgpr0, $sgpr0
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; LIMIT8-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT8-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; LIMIT8-NEXT:   S_BRANCH %bb.1
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT: bb.1:
+  ; LIMIT8-NEXT:   successors: %bb.2(0x80000000)
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %3, %bb.2
+  ; LIMIT8-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT8-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT8-NEXT:   S_BRANCH %bb.2
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT: bb.2:
+  ; LIMIT8-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY2]], implicit $exec
+  ; LIMIT8-NEXT:   S_BRANCH %bb.1
+  ;
+  ; LIMIT9-LABEL: name: ancestor_pressure_peak
+  ; LIMIT9: bb.0:
+  ; LIMIT9-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT9-NEXT:   liveins: $vgpr0, $sgpr0
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; LIMIT9-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT9-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; LIMIT9-NEXT:   S_BRANCH %bb.1
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT: bb.1:
+  ; LIMIT9-NEXT:   successors: %bb.2(0x80000000)
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %3, %bb.2
+  ; LIMIT9-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT9-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT9-NEXT:   S_BRANCH %bb.2
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT: bb.2:
+  ; LIMIT9-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY2]], implicit $exec
+  ; LIMIT9-NEXT:   S_BRANCH %bb.1
+  bb.0:
+    liveins: $vgpr0, $sgpr0
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:sgpr_32 = COPY $sgpr0
+    S_BRANCH %bb.1
+
+  bb.1:
+    %2:vgpr_32 = PHI %0, %bb.0, %5, %bb.2
+    %3:vreg_128 = DS_READ_B128_gfx9 %2, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+    DS_WRITE_B128_gfx9 %2, %3, 16, 0, implicit $exec :: (store (s128), addrspace 3)
+    S_BRANCH %bb.2
+
+  bb.2:
+    %4:vgpr_32 = COPY %1
+    %5:vgpr_32 = V_ADD_U32_e32 %2, %4, implicit $exec
+    S_BRANCH %bb.1
+...
+
+# bb.2 returns current pressure to the header's estimate before its scope is
+# popped. Its dead temporary has no later uses, so this does not need pressure
+# or RegSeen restoration between siblings. The COPY in bb.3 can then move.
+---
+name: sibling_pressure_peak
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: sibling_pressure_peak
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $vgpr0, $sgpr0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; CHECK-NEXT:   S_BRANCH %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, [[PHI]], %bb.2, %3, %bb.3
+  ; CHECK-NEXT:   S_CMP_EQ_U32 [[COPY1]], 0, implicit-def $scc
+  ; CHECK-NEXT:   S_CBRANCH_SCC1 %bb.2, implicit $scc
+  ; CHECK-NEXT:   S_BRANCH %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; CHECK-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; CHECK-NEXT:   S_BRANCH %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY2]], implicit $exec
+  ; CHECK-NEXT:   S_BRANCH %bb.1
+  bb.0:
+    liveins: $vgpr0, $sgpr0
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:sgpr_32 = COPY $sgpr0
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.2, %bb.3
+
+    %2:vgpr_32 = PHI %0, %bb.0, %2, %bb.2, %5, %bb.3
+    S_CMP_EQ_U32 %1, 0, implicit-def $scc
+    S_CBRANCH_SCC1 %bb.2, implicit $scc
+    S_BRANCH %bb.3
+
+  bb.2:
+    %3:vreg_128 = DS_READ_B128_gfx9 %2, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+    DS_WRITE_B128_gfx9 %2, %3, 16, 0, implicit $exec :: (store (s128), addrspace 3)
+    S_BRANCH %bb.1
+
+  bb.3:
+    %4:vgpr_32 = COPY %1
+    %5:vgpr_32 = V_ADD_U32_e32 %2, %4, implicit $exec
+    S_BRANCH %bb.1
+...
+
+# The preheader load raises VGPR pressure to 5, then the store lowers it to 1.
+# The COPY can move after the store, before the branch. Its result does not
+# cross the earlier preheader peak, so only the final estimate seeds the loop.
+---
+name: preheader_pressure_peak
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: preheader_pressure_peak
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $vgpr0, $sgpr0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; CHECK-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; CHECK-NEXT:   DS_WRITE_B128_gfx9 [[COPY]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; CHECK-NEXT:   S_BRANCH %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %4, %bb.1
+  ; CHECK-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY2]], implicit $exec
+  ; CHECK-NEXT:   S_BRANCH %bb.1
+  bb.0:
+    liveins: $vgpr0, $sgpr0
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:sgpr_32 = COPY $sgpr0
+    %2:vreg_128 = DS_READ_B128_gfx9 %0, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+    DS_WRITE_B128_gfx9 %0, %2, 16, 0, implicit $exec :: (store (s128), addrspace 3)
+    S_BRANCH %bb.1
+
+  bb.1:
+    %3:vgpr_32 = PHI %0, %bb.0, %5, %bb.1
+    %4:vgpr_32 = COPY %1
+    %5:vgpr_32 = V_ADD_U32_e32 %3, %4, implicit $exec
+    S_BRANCH %bb.1
+...
+
+# At limit 8, the first child's COPY can move, raising the ancestor's peak
+# from 6 to 7.
+# The later COPY must stay: its cost would bring the ancestor to the limit of 8.
+---
+name: ancestor_hoist_updates_peak
+tracksRegLiveness: true
+body: |
+  ; LIMIT6-LABEL: name: ancestor_hoist_updates_peak
+  ; LIMIT6: bb.0:
+  ; LIMIT6-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT6-NEXT:   liveins: $vgpr0, $sgpr0, $sgpr1
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; LIMIT6-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT6-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+  ; LIMIT6-NEXT:   S_BRANCH %bb.1
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT: bb.1:
+  ; LIMIT6-NEXT:   successors: %bb.2(0x80000000)
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %4, %bb.3
+  ; LIMIT6-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT6-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT6-NEXT:   S_BRANCH %bb.2
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT: bb.2:
+  ; LIMIT6-NEXT:   successors: %bb.3(0x80000000)
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; LIMIT6-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY3]], implicit $exec
+  ; LIMIT6-NEXT:   S_BRANCH %bb.3
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT: bb.3:
+  ; LIMIT6-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; LIMIT6-NEXT:   [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_]], [[COPY4]], implicit $exec
+  ; LIMIT6-NEXT:   S_BRANCH %bb.1
+  ;
+  ; LIMIT8-LABEL: name: ancestor_hoist_updates_peak
+  ; LIMIT8: bb.0:
+  ; LIMIT8-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT8-NEXT:   liveins: $vgpr0, $sgpr0, $sgpr1
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; LIMIT8-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT8-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+  ; LIMIT8-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; LIMIT8-NEXT:   S_BRANCH %bb.1
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT: bb.1:
+  ; LIMIT8-NEXT:   successors: %bb.2(0x80000000)
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %4, %bb.3
+  ; LIMIT8-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT8-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT8-NEXT:   S_BRANCH %bb.2
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT: bb.2:
+  ; LIMIT8-NEXT:   successors: %bb.3(0x80000000)
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY3]], implicit $exec
+  ; LIMIT8-NEXT:   S_BRANCH %bb.3
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT: bb.3:
+  ; LIMIT8-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; LIMIT8-NEXT:   [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_]], [[COPY4]], implicit $exec
+  ; LIMIT8-NEXT:   S_BRANCH %bb.1
+  ;
+  ; LIMIT9-LABEL: name: ancestor_hoist_updates_peak
+  ; LIMIT9: bb.0:
+  ; LIMIT9-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT9-NEXT:   liveins: $vgpr0, $sgpr0, $sgpr1
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; LIMIT9-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT9-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+  ; LIMIT9-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; LIMIT9-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; LIMIT9-NEXT:   S_BRANCH %bb.1
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT: bb.1:
+  ; LIMIT9-NEXT:   successors: %bb.2(0x80000000)
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %4, %bb.3
+  ; LIMIT9-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT9-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT9-NEXT:   S_BRANCH %bb.2
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT: bb.2:
+  ; LIMIT9-NEXT:   successors: %bb.3(0x80000000)
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY3]], implicit $exec
+  ; LIMIT9-NEXT:   S_BRANCH %bb.3
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT: bb.3:
+  ; LIMIT9-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT:   [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_]], [[COPY4]], implicit $exec
+  ; LIMIT9-NEXT:   S_BRANCH %bb.1
+  bb.0:
+    liveins: $vgpr0, $sgpr0, $sgpr1
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:sgpr_32 = COPY $sgpr0
+    %2:sgpr_32 = COPY $sgpr1
+    S_BRANCH %bb.1
+
+  bb.1:
+    %3:vgpr_32 = PHI %0, %bb.0, %8, %bb.3
+    %4:vreg_128 = DS_READ_B128_gfx9 %3, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+    DS_WRITE_B128_gfx9 %3, %4, 16, 0, implicit $exec :: (store (s128), addrspace 3)
+    S_BRANCH %bb.2
+
+  bb.2:
+    %5:vgpr_32 = COPY %1
+    %6:vgpr_32 = V_ADD_U32_e32 %3, %5, implicit $exec
+    S_BRANCH %bb.3
+
+  bb.3:
+    %7:vgpr_32 = COPY %2
+    %8:vgpr_32 = V_ADD_U32_e32 %6, %7, implicit $exec
+    S_BRANCH %bb.1
+...
+
+# At limit 9, current pressure falls from 6 to 2 before the COPYs. Two hoists
+# increase the saved peak to 7 and 8. The third would reach the limit of 9.
+# Both accepted hoists must be charged once even while current pressure is low.
+---
+name: multiple_hoists_raise_peak
+tracksRegLiveness: true
+body: |
+  ; LIMIT6-LABEL: name: multiple_hoists_raise_peak
+  ; LIMIT6: bb.0:
+  ; LIMIT6-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT6-NEXT:   liveins: $vgpr0, $sgpr0, $sgpr1, $sgpr2
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; LIMIT6-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT6-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+  ; LIMIT6-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+  ; LIMIT6-NEXT:   S_BRANCH %bb.1
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT: bb.1:
+  ; LIMIT6-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %5, %bb.1
+  ; LIMIT6-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT6-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT6-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; LIMIT6-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; LIMIT6-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; LIMIT6-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY4]], implicit $exec
+  ; LIMIT6-NEXT:   [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_]], [[COPY5]], implicit $exec
+  ; LIMIT6-NEXT:   [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_1]], [[COPY6]], implicit $exec
+  ; LIMIT6-NEXT:   S_BRANCH %bb.1
+  ;
+  ; LIMIT8-LABEL: name: multiple_hoists_raise_peak
+  ; LIMIT8: bb.0:
+  ; LIMIT8-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT8-NEXT:   liveins: $vgpr0, $sgpr0, $sgpr1, $sgpr2
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; LIMIT8-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT8-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+  ; LIMIT8-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+  ; LIMIT8-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; LIMIT8-NEXT:   S_BRANCH %bb.1
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT: bb.1:
+  ; LIMIT8-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %5, %bb.1
+  ; LIMIT8-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT8-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT8-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; LIMIT8-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; LIMIT8-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY4]], implicit $exec
+  ; LIMIT8-NEXT:   [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_]], [[COPY5]], implicit $exec
+  ; LIMIT8-NEXT:   [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_1]], [[COPY6]], implicit $exec
+  ; LIMIT8-NEXT:   S_BRANCH %bb.1
+  ;
+  ; LIMIT9-LABEL: name: multiple_hoists_raise_peak
+  ; LIMIT9: bb.0:
+  ; LIMIT9-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT9-NEXT:   liveins: $vgpr0, $sgpr0, $sgpr1, $sgpr2
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; LIMIT9-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT9-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+  ; LIMIT9-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+  ; LIMIT9-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; LIMIT9-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; LIMIT9-NEXT:   S_BRANCH %bb.1
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT: bb.1:
+  ; LIMIT9-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %5, %bb.1
+  ; LIMIT9-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT9-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT9-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+  ; LIMIT9-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY4]], implicit $exec
+  ; LIMIT9-NEXT:   [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_]], [[COPY5]], implicit $exec
+  ; LIMIT9-NEXT:   [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_1]], [[COPY6]], implicit $exec
+  ; LIMIT9-NEXT:   S_BRANCH %bb.1
+  bb.0:
+    liveins: $vgpr0, $sgpr0, $sgpr1, $sgpr2
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:sgpr_32 = COPY $sgpr0
+    %2:sgpr_32 = COPY $sgpr1
+    %3:sgpr_32 = COPY $sgpr2
+    S_BRANCH %bb.1
+
+  bb.1:
+    %4:vgpr_32 = PHI %0, %bb.0, %11, %bb.1
+    %5:vreg_128 = DS_READ_B128_gfx9 %4, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+    DS_WRITE_B128_gfx9 %4, %5, 16, 0, implicit $exec :: (store (s128), addrspace 3)
+    %6:vgpr_32 = COPY %1
+    %7:vgpr_32 = COPY %2
+    %8:vgpr_32 = COPY %3
+    %9:vgpr_32 = V_ADD_U32_e32 %4, %6, implicit $exec
+    %10:vgpr_32 = V_ADD_U32_e32 %9, %7, implicit $exec
+    %11:vgpr_32 = V_ADD_U32_e32 %10, %8, implicit $exec
+    S_BRANCH %bb.1
+...
+
+# The one-use vreg_128 input costs 4 even though the COPY selects one subregister.
+# Hoisting that COPY has net VGPR cost -3, lowering the peak from 10 to 7.
+# The following +1 COPY moves at limit 9 and stays at limits 6 and 8.
+---
+name: negative_cost_hoist_lowers_peak
+tracksRegLiveness: true
+body: |
+  ; LIMIT6-LABEL: name: negative_cost_hoist_lowers_peak
+  ; LIMIT6: bb.0:
+  ; LIMIT6-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT6-NEXT:   liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $sgpr0
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT:   [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+  ; LIMIT6-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; LIMIT6-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT6-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+  ; LIMIT6-NEXT:   S_BRANCH %bb.1
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT: bb.1:
+  ; LIMIT6-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT6-NEXT: {{  $}}
+  ; LIMIT6-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY1]], %bb.0, %4, %bb.1
+  ; LIMIT6-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT6-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT6-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; LIMIT6-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY3]], implicit $exec
+  ; LIMIT6-NEXT:   [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_]], [[COPY4]], implicit $exec
+  ; LIMIT6-NEXT:   S_BRANCH %bb.1
+  ;
+  ; LIMIT8-LABEL: name: negative_cost_hoist_lowers_peak
+  ; LIMIT8: bb.0:
+  ; LIMIT8-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT8-NEXT:   liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $sgpr0
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT:   [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+  ; LIMIT8-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; LIMIT8-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT8-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+  ; LIMIT8-NEXT:   S_BRANCH %bb.1
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT: bb.1:
+  ; LIMIT8-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT8-NEXT: {{  $}}
+  ; LIMIT8-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY1]], %bb.0, %4, %bb.1
+  ; LIMIT8-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT8-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT8-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; LIMIT8-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY3]], implicit $exec
+  ; LIMIT8-NEXT:   [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_]], [[COPY4]], implicit $exec
+  ; LIMIT8-NEXT:   S_BRANCH %bb.1
+  ;
+  ; LIMIT9-LABEL: name: negative_cost_hoist_lowers_peak
+  ; LIMIT9: bb.0:
+  ; LIMIT9-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT9-NEXT:   liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $sgpr0
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT:   [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+  ; LIMIT9-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; LIMIT9-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; LIMIT9-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+  ; LIMIT9-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; LIMIT9-NEXT:   S_BRANCH %bb.1
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT: bb.1:
+  ; LIMIT9-NEXT:   successors: %bb.1(0x80000000)
+  ; LIMIT9-NEXT: {{  $}}
+  ; LIMIT9-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY1]], %bb.0, %4, %bb.1
+  ; LIMIT9-NEXT:   [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[PHI]], 0, 0, implicit $exec :: (load (s128), addrspace 3)
+  ; LIMIT9-NEXT:   DS_WRITE_B128_gfx9 [[PHI]], [[DS_READ_B128_gfx9_]], 16, 0, implicit $exec :: (store (s128), addrspace 3)
+  ; LIMIT9-NEXT:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[PHI]], [[COPY3]], implicit $exec
+  ; LIMIT9-NEXT:   [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_]], [[COPY4]], implicit $exec
+  ; LIMIT9-NEXT:   S_BRANCH %bb.1
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $sgpr0
+
+    %0:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:vgpr_32 = COPY $vgpr4
+    %2:sgpr_32 = COPY $sgpr0
+    S_BRANCH %bb.1
+
+  bb.1:
+    %3:vgpr_32 = PHI %1, %bb.0, %8, %bb.1
+    %4:vreg_128 = DS_READ_B128_gfx9 %3, 0, 0, implicit $exec :: (load (s128), addrspace 3)
+    DS_WRITE_B128_gfx9 %3, %4, 16, 0, implicit $exec :: (store (s128), addrspace 3)
+    %5:vgpr_32 = COPY %0.sub0
+    %6:vgpr_32 = COPY %2
+    %7:vgpr_32 = V_ADD_U32_e32 %3, %5, implicit $exec
+    %8:vgpr_32 = V_ADD_U32_e32 %7, %6, implicit $exec
+    S_BRANCH %bb.1
+...
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll
index 3bd5a6e98c80ea..e721a34111affc 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll
@@ -19,7 +19,7 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    s_load_dword s8, s[4:5], 0x10
 ; CHECK-NEXT:    s_load_dwordx4 s[16:19], s[4:5], 0x20
 ; CHECK-NEXT:    s_load_dword s9, s[4:5], 0x30
-; CHECK-NEXT:    s_mov_b64 s[54:55], -1
+; CHECK-NEXT:    s_load_dword s28, s[4:5], 0x90
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_bitcmp1_b32 s21, 0
 ; CHECK-NEXT:    s_cselect_b64 s[2:3], -1, 0
@@ -36,24 +36,22 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; CHECK-NEXT:    s_bitcmp1_b32 s9, 0
 ; CHECK-NEXT:    s_cselect_b64 s[38:39], -1, 0
-; CHECK-NEXT:    s_mov_b64 s[52:53], 0
+; CHECK-NEXT:    s_mov_b64 s[54:55], -1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_bitcmp1_b32 s8, 0
 ; CHECK-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x60
-; CHECK-NEXT:    s_load_dword s28, s[4:5], 0x90
 ; CHECK-NEXT:    s_cselect_b64 s[40:41], -1, 0
-; CHECK-NEXT:    v_mov_b32_e32 v9, 0
-; CHECK-NEXT:    v_mov_b32_e32 v8, 0
-; CHECK-NEXT:    v_mov_b32_e32 v7, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_bitcmp1_b32 s28, 0
 ; CHECK-NEXT:    s_mov_b32 s28, 0
-; CHECK-NEXT:    s_cselect_b64 s[42:43], -1, 0
-; CHECK-NEXT:    s_lshl_b32 s21, s21, 1
 ; CHECK-NEXT:    s_mov_b32 s29, s28
 ; CHECK-NEXT:    s_mov_b32 s30, s28
 ; CHECK-NEXT:    s_mov_b32 s31, s28
-; CHECK-NEXT:    v_mov_b32_e32 v6, 0
+; CHECK-NEXT:    s_cselect_b64 s[42:43], -1, 0
+; CHECK-NEXT:    s_lshl_b32 s21, s21, 1
+; CHECK-NEXT:    s_mov_b64 s[52:53], 0
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    v_accvgpr_write_b32 a0, 0
 ; CHECK-NEXT:    v_accvgpr_write_b32 a1, 0
 ; CHECK-NEXT:    v_accvgpr_write_b32 a2, 0
@@ -90,8 +88,6 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    v_accvgpr_write_b32 a17, s29
 ; CHECK-NEXT:    v_accvgpr_write_b32 a18, s30
 ; CHECK-NEXT:    v_accvgpr_write_b32 a19, s31
-; CHECK-NEXT:    v_accvgpr_write_b32 a24, s21
-; CHECK-NEXT:    v_accvgpr_write_b32 a25, s22
 ; CHECK-NEXT:    s_mov_b64 s[60:61], 0
 ; CHECK-NEXT:    s_mov_b64 s[58:59], 0
 ; CHECK-NEXT:    s_mov_b64 s[56:57], 0
@@ -99,50 +95,52 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    s_mov_b32 s64, s28
 ; CHECK-NEXT:    s_mov_b64 s[48:49], 0
 ; CHECK-NEXT:    s_mov_b64 s[46:47], 0
-; CHECK-NEXT:    s_mov_b32 s21, s28
 ; CHECK-NEXT:    s_mov_b32 s33, s28
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
 ; CHECK-NEXT:    s_mov_b32 s62, s28
 ; CHECK-NEXT:    s_mov_b32 s63, s28
 ; CHECK-NEXT:    s_mov_b32 s65, s28
+; CHECK-NEXT:    s_mov_b32 s66, s28
 ; CHECK-NEXT:    v_mov_b32_e32 v4, 0
 ; CHECK-NEXT:  .LBB0_1: ; %bbl97
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    s_and_b64 s[52:53], s[52:53], exec
 ; CHECK-NEXT:    s_cselect_b32 s52, 0x3f803f80, 0
+; CHECK-NEXT:    v_mov_b32_e32 v10, s52
+; CHECK-NEXT:    v_mov_b32_e32 v11, s52
 ; CHECK-NEXT:    v_mov_b32_e32 v12, s52
 ; CHECK-NEXT:    v_mov_b32_e32 v13, s52
-; CHECK-NEXT:    v_mov_b32_e32 v14, s52
-; CHECK-NEXT:    v_mov_b32_e32 v15, s52
 ; CHECK-NEXT:    s_and_b64 s[52:53], s[60:61], exec
-; CHECK-NEXT:    v_accvgpr_read_b32 v0, a24
+; CHECK-NEXT:    v_mov_b32_e32 v5, s21
 ; CHECK-NEXT:    s_cselect_b32 s52, 0x3f803f80, 0
-; CHECK-NEXT:    ds_write_b128 v0, v[12:15]
+; CHECK-NEXT:    ds_write_b128 v5, v[10:13]
+; CHECK-NEXT:    v_mov_b32_e32 v10, s52
+; CHECK-NEXT:    v_mov_b32_e32 v11, s52
 ; CHECK-NEXT:    v_mov_b32_e32 v12, s52
 ; CHECK-NEXT:    v_mov_b32_e32 v13, s52
-; CHECK-NEXT:    v_mov_b32_e32 v14, s52
-; CHECK-NEXT:    v_mov_b32_e32 v15, s52
 ; CHECK-NEXT:    s_and_b64 s[52:53], s[58:59], exec
 ; CHECK-NEXT:    s_mov_b64 s[44:45], s[54:55]
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_cselect_b32 s52, s11, 0
 ; CHECK-NEXT:    s_cselect_b32 s53, s10, 0
 ; CHECK-NEXT:    s_cselect_b32 s54, s9, 0
 ; CHECK-NEXT:    s_cselect_b32 s55, s8, 0
-; CHECK-NEXT:    ds_write_b128 v4, v[12:15]
-; CHECK-NEXT:    v_mov_b32_e32 v12, s55
-; CHECK-NEXT:    v_mov_b32_e32 v13, s54
-; CHECK-NEXT:    v_mov_b32_e32 v14, s53
-; CHECK-NEXT:    v_mov_b32_e32 v15, s52
-; CHECK-NEXT:    v_accvgpr_read_b32 v5, a25
+; CHECK-NEXT:    ds_write_b128 v4, v[10:13]
+; CHECK-NEXT:    v_mov_b32_e32 v10, s55
+; CHECK-NEXT:    v_mov_b32_e32 v11, s54
+; CHECK-NEXT:    v_mov_b32_e32 v12, s53
+; CHECK-NEXT:    v_mov_b32_e32 v13, s52
+; CHECK-NEXT:    v_mov_b32_e32 v5, s22
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, v9, s[0:1]
-; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, v8, s[0:1]
-; CHECK-NEXT:    v_cndmask_b32_e64 v1, 0, v7, s[0:1]
-; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, v6, s[0:1]
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, v3, s[0:1]
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[0:1]
+; CHECK-NEXT:    v_cndmask_b32_e64 v1, 0, v1, s[0:1]
+; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, v0, s[0:1]
 ; CHECK-NEXT:    s_or_b32 s52, s64, s20
-; CHECK-NEXT:    ds_write_b128 v5, v[12:15]
+; CHECK-NEXT:    ds_write_b128 v5, v[10:13]
 ; CHECK-NEXT:    ds_write_b128 v4, v[0:3]
 ; CHECK-NEXT:    v_mov_b32_e32 v0, s52
-; CHECK-NEXT:    buffer_load_dwordx4 v[6:9], v0, s[28:31], 0 offen
+; CHECK-NEXT:    buffer_load_dwordx4 v[0:3], v0, s[28:31], 0 offen
 ; CHECK-NEXT:    v_mfma_f32_32x32x16_bf16 v[16:31], a[16:19], a[16:19], v[16:31]
 ; CHECK-NEXT:    s_and_b64 s[52:53], s[56:57], exec
 ; CHECK-NEXT:    s_cselect_b32 s52, s15, 0
@@ -150,41 +148,41 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    s_cselect_b32 s54, s13, 0
 ; CHECK-NEXT:    s_cselect_b32 s55, s12, 0
 ; CHECK-NEXT:    s_and_b64 vcc, s[2:3], exec
-; CHECK-NEXT:    v_mov_b32_e32 v12, s55
-; CHECK-NEXT:    v_mov_b32_e32 v13, s54
-; CHECK-NEXT:    v_mov_b32_e32 v14, s53
-; CHECK-NEXT:    v_mov_b32_e32 v15, s52
-; CHECK-NEXT:    s_cselect_b32 s52, s65, 0
-; CHECK-NEXT:    s_cselect_b32 s53, s63, 0
-; CHECK-NEXT:    s_cselect_b32 s54, s62, 0
-; CHECK-NEXT:    s_cselect_b32 s33, s33, 0
+; CHECK-NEXT:    v_mov_b32_e32 v10, s55
+; CHECK-NEXT:    v_mov_b32_e32 v11, s54
+; CHECK-NEXT:    v_mov_b32_e32 v12, s53
+; CHECK-NEXT:    v_mov_b32_e32 v13, s52
+; CHECK-NEXT:    s_cselect_b32 s52, s66, 0
+; CHECK-NEXT:    s_cselect_b32 s53, s65, 0
+; CHECK-NEXT:    s_cselect_b32 s54, s63, 0
+; CHECK-NEXT:    s_cselect_b32 s55, s62, 0
 ; CHECK-NEXT:    s_and_b64 s[50:51], s[50:51], exec
-; CHECK-NEXT:    ds_write_b128 v4, v[12:15]
-; CHECK-NEXT:    v_mov_b32_e32 v12, s33
-; CHECK-NEXT:    s_cselect_b32 s33, 0x3f803f80, 0
+; CHECK-NEXT:    ds_write_b128 v4, v[10:13]
+; CHECK-NEXT:    v_mov_b32_e32 v10, s55
+; CHECK-NEXT:    v_mov_b32_e32 v11, s54
+; CHECK-NEXT:    v_mov_b32_e32 v12, s53
+; CHECK-NEXT:    v_mov_b32_e32 v13, s52
+; CHECK-NEXT:    s_cselect_b32 s50, 0x3f803f80, 0
+; CHECK-NEXT:    ds_write_b128 v5, v[10:13]
+; CHECK-NEXT:    v_mov_b32_e32 v10, s50
+; CHECK-NEXT:    v_mov_b32_e32 v11, s50
+; CHECK-NEXT:    v_mov_b32_e32 v12, s50
+; CHECK-NEXT:    v_mov_b32_e32 v13, s50
 ; CHECK-NEXT:    s_and_b64 s[50:51], s[6:7], exec
 ; CHECK-NEXT:    v_mfma_f32_32x32x16_bf16 a[0:15], a[16:19], a[16:19], a[0:15]
-; CHECK-NEXT:    v_mov_b32_e32 v13, s54
-; CHECK-NEXT:    v_mov_b32_e32 v14, s53
-; CHECK-NEXT:    v_mov_b32_e32 v15, s52
-; CHECK-NEXT:    s_cselect_b32 s21, s21, 0
+; CHECK-NEXT:    s_cselect_b32 s33, s33, 0
 ; CHECK-NEXT:    s_and_b64 s[48:49], s[48:49], exec
-; CHECK-NEXT:    ds_write_b128 v5, v[12:15]
-; CHECK-NEXT:    v_mov_b32_e32 v12, s33
-; CHECK-NEXT:    v_mfma_f32_32x32x16_bf16 v[16:31], a[16:19], a[16:19], v[16:31]
-; CHECK-NEXT:    v_mov_b32_e32 v13, s33
-; CHECK-NEXT:    v_mov_b32_e32 v14, s33
-; CHECK-NEXT:    v_mov_b32_e32 v15, s33
-; CHECK-NEXT:    s_cselect_b32 s33, 0x3f803f80, 0
+; CHECK-NEXT:    s_cselect_b32 s48, 0x3f803f80, 0
+; CHECK-NEXT:    s_lshr_b32 s48, s48, 16
 ; CHECK-NEXT:    s_lshr_b32 s33, s33, 16
-; CHECK-NEXT:    s_lshr_b32 s21, s21, 16
 ; CHECK-NEXT:    s_and_b64 s[46:47], s[46:47], exec
-; CHECK-NEXT:    s_pack_ll_b32_b16 s21, s21, s33
-; CHECK-NEXT:    s_cselect_b32 s33, s19, 0
-; CHECK-NEXT:    s_lshr_b32 s33, s33, 16
-; CHECK-NEXT:    ds_write_b128 v4, v[12:15]
-; CHECK-NEXT:    v_mov_b32_e32 v12, s21
-; CHECK-NEXT:    v_mov_b32_e32 v13, s33
+; CHECK-NEXT:    s_cselect_b32 s46, s19, 0
+; CHECK-NEXT:    v_mfma_f32_32x32x16_bf16 v[16:31], a[16:19], a[16:19], v[16:31]
+; CHECK-NEXT:    s_pack_ll_b32_b16 s33, s33, s48
+; CHECK-NEXT:    s_lshr_b32 s46, s46, 16
+; CHECK-NEXT:    ds_write_b128 v4, v[10:13]
+; CHECK-NEXT:    v_mov_b32_e32 v10, s33
+; CHECK-NEXT:    v_mov_b32_e32 v11, s46
 ; CHECK-NEXT:    s_mov_b64 s[52:53], s[42:43]
 ; CHECK-NEXT:    s_mov_b64 s[60:61], s[40:41]
 ; CHECK-NEXT:    s_mov_b64 s[58:59], s[38:39]
@@ -194,12 +192,12 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    s_mov_b64 s[54:55], 0
 ; CHECK-NEXT:    s_mov_b64 s[48:49], s[34:35]
 ; CHECK-NEXT:    s_mov_b64 s[46:47], s[36:37]
-; CHECK-NEXT:    s_mov_b32 s21, s27
-; CHECK-NEXT:    s_mov_b32 s33, 1
+; CHECK-NEXT:    s_mov_b32 s33, s27
 ; CHECK-NEXT:    s_mov_b32 s62, 1
 ; CHECK-NEXT:    s_mov_b32 s63, 1
 ; CHECK-NEXT:    s_mov_b32 s65, 1
-; CHECK-NEXT:    ds_write_b64 v4, v[12:13]
+; CHECK-NEXT:    s_mov_b32 s66, 1
+; CHECK-NEXT:    ds_write_b64 v4, v[10:11]
 ; CHECK-NEXT:    s_cbranch_vccnz .LBB0_1
 ; CHECK-NEXT:  ; %bb.2: ; %bbl290
 ; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x80
@@ -207,9 +205,9 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
 ; CHECK-NEXT:    s_cselect_b32 s4, 0x3f803f80, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v4, s4
 ; CHECK-NEXT:    v_mov_b32_e32 v5, s4
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v6, s4
 ; CHECK-NEXT:    v_mov_b32_e32 v7, s4
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    ds_write_b128 v0, v[4:7]
 ; CHECK-NEXT:    v_mov_b32_e32 v4, s23
diff --git a/llvm/test/CodeGen/AVR/bug-81911.ll b/llvm/test/CodeGen/AVR/bug-81911.ll
index 2a22666a1ff927..fa331f0239d872 100644
--- a/llvm/test/CodeGen/AVR/bug-81911.ll
+++ b/llvm/test/CodeGen/AVR/bug-81911.ll
@@ -24,7 +24,7 @@ define internal i8 @main() {
 ; CHECK-NEXT:    push r29
 ; CHECK-NEXT:    in r28, 61
 ; CHECK-NEXT:    in r29, 62
-; CHECK-NEXT:    sbiw r28, 13
+; CHECK-NEXT:    sbiw r28, 11
 ; CHECK-NEXT:    in r0, 63
 ; CHECK-NEXT:    cli
 ; CHECK-NEXT:    out 62, r29
@@ -32,15 +32,11 @@ define internal i8 @main() {
 ; CHECK-NEXT:    out 61, r28
 ; CHECK-NEXT:    ldi r16, 0
 ; CHECK-NEXT:    ldi r17, 0
-; CHECK-NEXT:    ldi r18, -1
+; CHECK-NEXT:    ldi r25, -1
 ; CHECK-NEXT:    ;APP
 ; CHECK-NEXT:    ldi r24, 123
 ; CHECK-NEXT:    ;NO_APP
 ; CHECK-NEXT:    std Y+1, r24 ; 1-byte Folded Spill
-; CHECK-NEXT:    movw r24, r28
-; CHECK-NEXT:    adiw r24, 6
-; CHECK-NEXT:    std Y+3, r25 ; 2-byte Folded Spill
-; CHECK-NEXT:    std Y+2, r24 ; 2-byte Folded Spill
 ; CHECK-NEXT:    movw r8, r16
 ; CHECK-NEXT:    movw r6, r16
 ; CHECK-NEXT:    movw r4, r16
@@ -49,19 +45,19 @@ define internal i8 @main() {
 ; CHECK-NEXT:  .LBB0_1: ; %bb1
 ; CHECK-NEXT:    ; in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:    andi r30, 1
-; CHECK-NEXT:    ldd r31, Y+4 ; 1-byte Folded Reload
+; CHECK-NEXT:    ldd r31, Y+2 ; 1-byte Folded Reload
 ; CHECK-NEXT:    dec r31
 ; CHECK-NEXT:    cpi r30, 0
 ; CHECK-NEXT:    movw r8, r18
 ; CHECK-NEXT:    movw r6, r20
 ; CHECK-NEXT:    movw r4, r22
 ; CHECK-NEXT:    movw r2, r24
-; CHECK-NEXT:    mov r18, r31
+; CHECK-NEXT:    mov r25, r31
 ; CHECK-NEXT:    brne .LBB0_2
 ; CHECK-NEXT:    rjmp .LBB0_4
 ; CHECK-NEXT:  .LBB0_2: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    std Y+4, r18 ; 1-byte Folded Spill
+; CHECK-NEXT:    std Y+2, r25 ; 1-byte Folded Spill
 ; CHECK-NEXT:    movw r18, r8
 ; CHECK-NEXT:    movw r20, r6
 ; CHECK-NEXT:    movw r22, r4
@@ -72,16 +68,16 @@ define internal i8 @main() {
 ; CHECK-NEXT:    movw r12, r16
 ; CHECK-NEXT:    movw r14, r16
 ; CHECK-NEXT:    call __udivdi3
-; CHECK-NEXT:    std Y+13, r25
-; CHECK-NEXT:    std Y+12, r24
-; CHECK-NEXT:    std Y+11, r23
-; CHECK-NEXT:    std Y+10, r22
-; CHECK-NEXT:    std Y+9, r21
-; CHECK-NEXT:    std Y+8, r20
-; CHECK-NEXT:    std Y+7, r19
-; CHECK-NEXT:    std Y+6, r18
-; CHECK-NEXT:    ldd r30, Y+2 ; 2-byte Folded Reload
-; CHECK-NEXT:    ldd r31, Y+3 ; 2-byte Folded Reload
+; CHECK-NEXT:    std Y+11, r25
+; CHECK-NEXT:    std Y+10, r24
+; CHECK-NEXT:    std Y+9, r23
+; CHECK-NEXT:    std Y+8, r22
+; CHECK-NEXT:    std Y+7, r21
+; CHECK-NEXT:    std Y+6, r20
+; CHECK-NEXT:    std Y+5, r19
+; CHECK-NEXT:    std Y+4, r18
+; CHECK-NEXT:    movw r30, r28
+; CHECK-NEXT:    adiw r30, 4
 ; CHECK-NEXT:    ;APP
 ; CHECK-NEXT:    ;NO_APP
 ; CHECK-NEXT:    ldi r30, 1
@@ -101,13 +97,13 @@ define internal i8 @main() {
 ; CHECK-NEXT:    rjmp .LBB0_1
 ; CHECK-NEXT:  .LBB0_4: ; %bb3
 ; CHECK-NEXT:    ldd r24, Y+1 ; 1-byte Folded Reload
-; CHECK-NEXT:    std Y+5, r24
+; CHECK-NEXT:    std Y+3, r24
 ; CHECK-NEXT:    movw r24, r28
-; CHECK-NEXT:    adiw r24, 5
+; CHECK-NEXT:    adiw r24, 3
 ; CHECK-NEXT:    ;APP
 ; CHECK-NEXT:    ;NO_APP
-; CHECK-NEXT:    ldd r24, Y+5
-; CHECK-NEXT:    adiw r28, 13
+; CHECK-NEXT:    ldd r24, Y+3
+; CHECK-NEXT:    adiw r28, 11
 ; CHECK-NEXT:    in r0, 63
 ; CHECK-NEXT:    cli
 ; CHECK-NEXT:    out 62, r29
diff --git a/llvm/test/CodeGen/Hexagon/order-stack-object.ll b/llvm/test/CodeGen/Hexagon/order-stack-object.ll
index 920f95755ec5fc..f642b5d7e1263a 100644
--- a/llvm/test/CodeGen/Hexagon/order-stack-object.ll
+++ b/llvm/test/CodeGen/Hexagon/order-stack-object.ll
@@ -3,7 +3,7 @@
 ; Check that ordering objects on the stack from the largest to the smallest has
 ; decreased the space allocated on the stack by 512 Bytes.
 
-; CHECK: allocframe(r29,#2432):raw
+; CHECK: allocframe(r29,#2304):raw
 
 define void @test(ptr nocapture readonly %arg, ptr nocapture writeonly %arg1, i32 %arg2) local_unnamed_addr {
 bb:
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
index 976878ed7616db..f4ced938126ff9 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
@@ -506,27 +506,27 @@ define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM60-NEXT:    ld.volatile.global.b32 %r18, [%rd1];
 ; SM60-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM60-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM60-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM60-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM60-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM60-NEXT:  $L__BB18_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM60-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM60-NEXT:    add.s16 %rs3, %rs2, %rs1;
 ; SM60-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM60-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM60-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM60-NEXT:    add.s16 %rs6, %rs5, %rs4;
 ; SM60-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM60-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM60-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM60-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM60-NEXT:    add.s16 %rs9, %rs8, %rs7;
 ; SM60-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM60-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM60-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM60-NEXT:    add.s16 %rs12, %rs11, %rs10;
@@ -723,7 +723,6 @@ define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:  $L__BB22_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -731,6 +730,7 @@ define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM60-NEXT:    add.s16 %rs6, %rs3, %rs1;
 ; SM60-NEXT:    mov.b32 %r3, {%rs6, %rs5};
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
 ; SM60-NEXT:    add.s16 %rs11, %rs10, %rs8;
 ; SM60-NEXT:    add.s16 %rs12, %rs9, %rs7;
@@ -771,7 +771,6 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:  $L__BB23_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -779,6 +778,7 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    add.s16 %rs6, %rs3, %rs1;
 ; SM60-NEXT:    mov.b32 %r5, {%rs6, %rs5};
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
 ; SM60-NEXT:    add.s16 %rs11, %rs10, %rs8;
 ; SM60-NEXT:    add.s16 %rs12, %rs9, %rs7;
@@ -800,7 +800,6 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
 ; SM60-NEXT:    cvt.u32.u64 %r11, %rd12;
 ; SM60-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NEXT:  $L__BB23_3: // %atomicrmw.start2
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    mov.b32 {%rs15, %rs16}, %r11;
@@ -808,6 +807,7 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    add.s16 %rs18, %rs15, %rs13;
 ; SM60-NEXT:    mov.b32 %r7, {%rs18, %rs17};
 ; SM60-NEXT:    cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
 ; SM60-NEXT:    add.s16 %rs23, %rs22, %rs20;
 ; SM60-NEXT:    add.s16 %rs24, %rs21, %rs19;
@@ -1105,27 +1105,27 @@ define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM60-NEXT:    ld.volatile.global.b32 %r18, [%rd1];
 ; SM60-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM60-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM60-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM60-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM60-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM60-NEXT:  $L__BB34_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM60-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM60-NEXT:    sub.s16 %rs3, %rs2, %rs1;
 ; SM60-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM60-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM60-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM60-NEXT:    sub.s16 %rs6, %rs5, %rs4;
 ; SM60-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM60-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM60-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM60-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM60-NEXT:    sub.s16 %rs9, %rs8, %rs7;
 ; SM60-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM60-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM60-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM60-NEXT:    sub.s16 %rs12, %rs11, %rs10;
@@ -1322,7 +1322,6 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:  $L__BB38_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -1330,6 +1329,7 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM60-NEXT:    sub.s16 %rs6, %rs3, %rs1;
 ; SM60-NEXT:    mov.b32 %r3, {%rs6, %rs5};
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
 ; SM60-NEXT:    sub.s16 %rs11, %rs10, %rs8;
 ; SM60-NEXT:    sub.s16 %rs12, %rs9, %rs7;
@@ -1370,7 +1370,6 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:  $L__BB39_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -1378,6 +1377,7 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    sub.s16 %rs6, %rs3, %rs1;
 ; SM60-NEXT:    mov.b32 %r5, {%rs6, %rs5};
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
 ; SM60-NEXT:    sub.s16 %rs11, %rs10, %rs8;
 ; SM60-NEXT:    sub.s16 %rs12, %rs9, %rs7;
@@ -1399,7 +1399,6 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
 ; SM60-NEXT:    cvt.u32.u64 %r11, %rd12;
 ; SM60-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NEXT:  $L__BB39_3: // %atomicrmw.start2
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    mov.b32 {%rs15, %rs16}, %r11;
@@ -1407,6 +1406,7 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    sub.s16 %rs18, %rs15, %rs13;
 ; SM60-NEXT:    mov.b32 %r7, {%rs18, %rs17};
 ; SM60-NEXT:    cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
 ; SM60-NEXT:    sub.s16 %rs23, %rs22, %rs20;
 ; SM60-NEXT:    sub.s16 %rs24, %rs21, %rs19;
@@ -3839,11 +3839,11 @@ define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM60-NEXT:    membar.cta;
 ; SM60-NEXT:    ld.volatile.global.b32 %r26, [%rd1];
 ; SM60-NEXT:    prmt.b32 %r3, %r2, 0, 0x8880U;
-; SM60-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
 ; SM60-NEXT:  $L__BB114_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    prmt.b32 %r4, %r26, 0, 0x8880U;
 ; SM60-NEXT:    setp.gt.s32 %p1, %r4, %r3;
+; SM60-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
 ; SM60-NEXT:    prmt.b32 %r6, %r26, 0, 0x9991U;
 ; SM60-NEXT:    setp.gt.s32 %p2, %r6, %r5;
 ; SM60-NEXT:    prmt.b32 %r7, %r2, 0, 0xaaa2U;
@@ -4084,7 +4084,6 @@ define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:  $L__BB118_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -4096,6 +4095,7 @@ define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM60-NEXT:    max.s16 %rs6, %rs3, %rs1;
 ; SM60-NEXT:    mov.b32 %r3, {%rs6, %rs5};
 ; SM60-NEXT:    cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
 ; SM60-NEXT:    max.s16 %rs11, %rs10, %rs8;
 ; SM60-NEXT:    max.s16 %rs12, %rs9, %rs7;
@@ -4132,7 +4132,6 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:  $L__BB119_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -4144,6 +4143,7 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    max.s16 %rs6, %rs3, %rs1;
 ; SM60-NEXT:    mov.b32 %r5, {%rs6, %rs5};
 ; SM60-NEXT:    cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
 ; SM60-NEXT:    max.s16 %rs11, %rs10, %rs8;
 ; SM60-NEXT:    max.s16 %rs12, %rs9, %rs7;
@@ -4161,7 +4161,6 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
 ; SM60-NEXT:    cvt.u32.u64 %r11, %rd12;
 ; SM60-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NEXT:  $L__BB119_3: // %atomicrmw.start2
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd13, %r11;
@@ -4173,6 +4172,7 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    max.s16 %rs18, %rs15, %rs13;
 ; SM60-NEXT:    mov.b32 %r7, {%rs18, %rs17};
 ; SM60-NEXT:    cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
 ; SM60-NEXT:    max.s16 %rs23, %rs22, %rs20;
 ; SM60-NEXT:    max.s16 %rs24, %rs21, %rs19;
@@ -4471,11 +4471,11 @@ define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM60-NEXT:    membar.cta;
 ; SM60-NEXT:    ld.volatile.global.b32 %r26, [%rd1];
 ; SM60-NEXT:    prmt.b32 %r3, %r2, 0, 0x8880U;
-; SM60-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
 ; SM60-NEXT:  $L__BB130_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    prmt.b32 %r4, %r26, 0, 0x8880U;
 ; SM60-NEXT:    setp.le.s32 %p1, %r4, %r3;
+; SM60-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
 ; SM60-NEXT:    prmt.b32 %r6, %r26, 0, 0x9991U;
 ; SM60-NEXT:    setp.le.s32 %p2, %r6, %r5;
 ; SM60-NEXT:    prmt.b32 %r7, %r2, 0, 0xaaa2U;
@@ -4716,7 +4716,6 @@ define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:  $L__BB134_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -4728,6 +4727,7 @@ define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM60-NEXT:    min.s16 %rs6, %rs3, %rs1;
 ; SM60-NEXT:    mov.b32 %r3, {%rs6, %rs5};
 ; SM60-NEXT:    cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
 ; SM60-NEXT:    min.s16 %rs11, %rs10, %rs8;
 ; SM60-NEXT:    min.s16 %rs12, %rs9, %rs7;
@@ -4764,7 +4764,6 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:  $L__BB135_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -4776,6 +4775,7 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    min.s16 %rs6, %rs3, %rs1;
 ; SM60-NEXT:    mov.b32 %r5, {%rs6, %rs5};
 ; SM60-NEXT:    cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
 ; SM60-NEXT:    min.s16 %rs11, %rs10, %rs8;
 ; SM60-NEXT:    min.s16 %rs12, %rs9, %rs7;
@@ -4793,7 +4793,6 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
 ; SM60-NEXT:    cvt.u32.u64 %r11, %rd12;
 ; SM60-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NEXT:  $L__BB135_3: // %atomicrmw.start2
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd13, %r11;
@@ -4805,6 +4804,7 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM60-NEXT:    min.s16 %rs18, %rs15, %rs13;
 ; SM60-NEXT:    mov.b32 %r7, {%rs18, %rs17};
 ; SM60-NEXT:    cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
 ; SM60-NEXT:    min.s16 %rs23, %rs22, %rs20;
 ; SM60-NEXT:    min.s16 %rs24, %rs21, %rs19;
@@ -5059,7 +5059,6 @@ define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
 ; SM60-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM60-NEXT:    not.b32 %r2, %r7;
 ; SM60-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM60-NEXT:  $L__BB145_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    shr.u32 %r8, %r16, %r1;
@@ -5068,6 +5067,7 @@ define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
 ; SM60-NEXT:    mov.b32 %r9, {%rs3, %rs4};
 ; SM60-NEXT:    and.b32 %r10, %r9, 16711935;
 ; SM60-NEXT:    mov.b32 {%rs5, %rs6}, %r10;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM60-NEXT:    max.u16 %rs9, %rs5, %rs7;
 ; SM60-NEXT:    max.u16 %rs10, %rs6, %rs8;
 ; SM60-NEXT:    shl.b16 %rs11, %rs10, 8;
@@ -5102,11 +5102,11 @@ define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
 ; SM60-NEXT:    membar.cta;
 ; SM60-NEXT:    ld.volatile.global.b32 %r18, [%rd1];
 ; SM60-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM60-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM60-NEXT:  $L__BB146_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    prmt.b32 %r4, %r18, 0, 0x7770U;
 ; SM60-NEXT:    setp.gt.u32 %p1, %r4, %r3;
+; SM60-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM60-NEXT:    prmt.b32 %r6, %r18, 0, 0x7771U;
 ; SM60-NEXT:    setp.gt.u32 %p2, %r6, %r5;
 ; SM60-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -5323,7 +5323,6 @@ define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:  $L__BB150_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -5335,6 +5334,7 @@ define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
 ; SM60-NEXT:    max.u16 %rs6, %rs3, %rs1;
 ; SM60-NEXT:    mov.b32 %r3, {%rs6, %rs5};
 ; SM60-NEXT:    cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
 ; SM60-NEXT:    max.u16 %rs11, %rs10, %rs8;
 ; SM60-NEXT:    max.u16 %rs12, %rs9, %rs7;
@@ -5371,7 +5371,6 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:  $L__BB151_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -5383,6 +5382,7 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM60-NEXT:    max.u16 %rs6, %rs3, %rs1;
 ; SM60-NEXT:    mov.b32 %r5, {%rs6, %rs5};
 ; SM60-NEXT:    cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
 ; SM60-NEXT:    max.u16 %rs11, %rs10, %rs8;
 ; SM60-NEXT:    max.u16 %rs12, %rs9, %rs7;
@@ -5400,7 +5400,6 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
 ; SM60-NEXT:    cvt.u32.u64 %r11, %rd12;
 ; SM60-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NEXT:  $L__BB151_3: // %atomicrmw.start2
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd13, %r11;
@@ -5412,6 +5411,7 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM60-NEXT:    max.u16 %rs18, %rs15, %rs13;
 ; SM60-NEXT:    mov.b32 %r7, {%rs18, %rs17};
 ; SM60-NEXT:    cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
 ; SM60-NEXT:    max.u16 %rs23, %rs22, %rs20;
 ; SM60-NEXT:    max.u16 %rs24, %rs21, %rs19;
@@ -5666,7 +5666,6 @@ define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
 ; SM60-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM60-NEXT:    not.b32 %r2, %r7;
 ; SM60-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM60-NEXT:  $L__BB161_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    shr.u32 %r8, %r16, %r1;
@@ -5675,6 +5674,7 @@ define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
 ; SM60-NEXT:    mov.b32 %r9, {%rs3, %rs4};
 ; SM60-NEXT:    and.b32 %r10, %r9, 16711935;
 ; SM60-NEXT:    mov.b32 {%rs5, %rs6}, %r10;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM60-NEXT:    min.u16 %rs9, %rs5, %rs7;
 ; SM60-NEXT:    min.u16 %rs10, %rs6, %rs8;
 ; SM60-NEXT:    shl.b16 %rs11, %rs10, 8;
@@ -5709,11 +5709,11 @@ define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
 ; SM60-NEXT:    membar.cta;
 ; SM60-NEXT:    ld.volatile.global.b32 %r18, [%rd1];
 ; SM60-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM60-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM60-NEXT:  $L__BB162_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    prmt.b32 %r4, %r18, 0, 0x7770U;
 ; SM60-NEXT:    setp.le.u32 %p1, %r4, %r3;
+; SM60-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM60-NEXT:    prmt.b32 %r6, %r18, 0, 0x7771U;
 ; SM60-NEXT:    setp.le.u32 %p2, %r6, %r5;
 ; SM60-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -5930,7 +5930,6 @@ define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:  $L__BB166_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -5942,6 +5941,7 @@ define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
 ; SM60-NEXT:    min.u16 %rs6, %rs3, %rs1;
 ; SM60-NEXT:    mov.b32 %r3, {%rs6, %rs5};
 ; SM60-NEXT:    cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
 ; SM60-NEXT:    min.u16 %rs11, %rs10, %rs8;
 ; SM60-NEXT:    min.u16 %rs12, %rs9, %rs7;
@@ -5978,7 +5978,6 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:  $L__BB167_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -5990,6 +5989,7 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM60-NEXT:    min.u16 %rs6, %rs3, %rs1;
 ; SM60-NEXT:    mov.b32 %r5, {%rs6, %rs5};
 ; SM60-NEXT:    cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
 ; SM60-NEXT:    min.u16 %rs11, %rs10, %rs8;
 ; SM60-NEXT:    min.u16 %rs12, %rs9, %rs7;
@@ -6007,7 +6007,6 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
 ; SM60-NEXT:    cvt.u32.u64 %r11, %rd12;
 ; SM60-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NEXT:  $L__BB167_3: // %atomicrmw.start2
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd13, %r11;
@@ -6019,6 +6018,7 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM60-NEXT:    min.u16 %rs18, %rs15, %rs13;
 ; SM60-NEXT:    mov.b32 %r7, {%rs18, %rs17};
 ; SM60-NEXT:    cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
 ; SM60-NEXT:    min.u16 %rs23, %rs22, %rs20;
 ; SM60-NEXT:    min.u16 %rs24, %rs21, %rs19;
@@ -6276,7 +6276,6 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM60-NEXT:    not.b32 %r2, %r7;
 ; SM60-NEXT:    ld.volatile.global.b32 %r14, [%rd1];
-; SM60-NEXT:    mov.b32 {%rs8, %rs9}, %r4;
 ; SM60-NEXT:  $L__BB177_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    shr.u32 %r8, %r14, %r1;
@@ -6285,6 +6284,7 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NEXT:    shr.u16 %rs5, %rs3, 8;
 ; SM60-NEXT:    add.s16 %rs6, %rs5, 1;
 ; SM60-NEXT:    add.s16 %rs7, %rs3, 1;
+; SM60-NEXT:    mov.b32 {%rs8, %rs9}, %r4;
 ; SM60-NEXT:    setp.ge.u16 %p1, %rs5, %rs9;
 ; SM60-NEXT:    setp.ge.u16 %p2, %rs4, %rs8;
 ; SM60-NEXT:    selp.b16 %rs10, 0, %rs7, %p2;
@@ -6322,8 +6322,6 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i8_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
 ; SM60-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
-; SM60-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
 ; SM60-NEXT:  $L__BB178_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    prmt.b32 %r3, %r22, 0, 0x7770U;
@@ -6338,7 +6336,9 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NEXT:    prmt.b32 %r6, %r22, 0, 0x7773U;
 ; SM60-NEXT:    cvt.u16.u32 %rs7, %r6;
 ; SM60-NEXT:    add.s16 %rs8, %rs7, 1;
+; SM60-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
 ; SM60-NEXT:    setp.ge.u32 %p1, %r3, %r7;
+; SM60-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
 ; SM60-NEXT:    setp.ge.u32 %p2, %r4, %r8;
 ; SM60-NEXT:    prmt.b32 %r9, %r2, 0, 0x7772U;
 ; SM60-NEXT:    setp.ge.u32 %p3, %r5, %r9;
@@ -6523,12 +6523,12 @@ define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
 ; SM60-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i16_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
 ; SM60-NEXT:    ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NEXT:  $L__BB181_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; SM60-NEXT:    add.s16 %rs3, %rs1, 1;
 ; SM60-NEXT:    add.s16 %rs4, %rs2, 1;
+; SM60-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NEXT:    setp.ge.u16 %p1, %rs1, %rs5;
 ; SM60-NEXT:    setp.ge.u16 %p2, %rs2, %rs6;
 ; SM60-NEXT:    selp.b16 %rs7, 0, %rs4, %p2;
@@ -6561,8 +6561,6 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NEXT:    ld.volatile.global.b64 %rd2, [%rd1];
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
-; SM60-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM60-NEXT:  $L__BB182_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -6575,8 +6573,10 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NEXT:    mov.b32 {%rs5, %rs6}, %r5;
 ; SM60-NEXT:    add.s16 %rs7, %rs5, 1;
 ; SM60-NEXT:    add.s16 %rs8, %rs6, 1;
+; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM60-NEXT:    setp.ge.u16 %p1, %rs1, %rs9;
 ; SM60-NEXT:    setp.ge.u16 %p2, %rs2, %rs10;
+; SM60-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM60-NEXT:    setp.ge.u16 %p3, %rs5, %rs11;
 ; SM60-NEXT:    setp.ge.u16 %p4, %rs6, %rs12;
 ; SM60-NEXT:    selp.b16 %rs13, 0, %rs8, %p4;
@@ -6617,8 +6617,6 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NEXT:    ld.volatile.global.b64 %rd2, [%rd1];
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
-; SM60-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM60-NEXT:  $L__BB183_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -6631,8 +6629,10 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NEXT:    mov.b32 {%rs5, %rs6}, %r9;
 ; SM60-NEXT:    add.s16 %rs7, %rs5, 1;
 ; SM60-NEXT:    add.s16 %rs8, %rs6, 1;
+; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM60-NEXT:    setp.ge.u16 %p1, %rs1, %rs9;
 ; SM60-NEXT:    setp.ge.u16 %p2, %rs2, %rs10;
+; SM60-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM60-NEXT:    setp.ge.u16 %p3, %rs5, %rs11;
 ; SM60-NEXT:    setp.ge.u16 %p4, %rs6, %rs12;
 ; SM60-NEXT:    selp.b16 %rs13, 0, %rs8, %p4;
@@ -7088,7 +7088,6 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM60-NEXT:    not.b32 %r2, %r7;
 ; SM60-NEXT:    ld.volatile.global.b32 %r14, [%rd1];
-; SM60-NEXT:    mov.b32 {%rs8, %rs9}, %r4;
 ; SM60-NEXT:  $L__BB193_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    shr.u32 %r8, %r14, %r1;
@@ -7099,6 +7098,7 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NEXT:    add.s16 %rs7, %rs3, -1;
 ; SM60-NEXT:    setp.eq.b16 %p1, %rs5, 0;
 ; SM60-NEXT:    setp.eq.b16 %p2, %rs4, 0;
+; SM60-NEXT:    mov.b32 {%rs8, %rs9}, %r4;
 ; SM60-NEXT:    setp.gt.u16 %p3, %rs5, %rs9;
 ; SM60-NEXT:    setp.gt.u16 %p4, %rs4, %rs8;
 ; SM60-NEXT:    selp.b16 %rs10, %rs8, %rs7, %p4;
@@ -7138,8 +7138,6 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i8_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
 ; SM60-NEXT:    ld.volatile.global.b32 %r26, [%rd1];
-; SM60-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
-; SM60-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
 ; SM60-NEXT:  $L__BB194_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    prmt.b32 %r3, %r26, 0, 0x7770U;
@@ -7158,7 +7156,9 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NEXT:    setp.eq.b32 %p2, %r4, 0;
 ; SM60-NEXT:    setp.eq.b32 %p3, %r5, 0;
 ; SM60-NEXT:    setp.eq.b32 %p4, %r6, 0;
+; SM60-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
 ; SM60-NEXT:    setp.gt.u32 %p5, %r3, %r7;
+; SM60-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
 ; SM60-NEXT:    setp.gt.u32 %p6, %r4, %r8;
 ; SM60-NEXT:    prmt.b32 %r9, %r2, 0, 0x7772U;
 ; SM60-NEXT:    setp.gt.u32 %p7, %r5, %r9;
@@ -7373,7 +7373,6 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
 ; SM60-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i16_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
 ; SM60-NEXT:    ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NEXT:  $L__BB197_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
@@ -7381,6 +7380,7 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
 ; SM60-NEXT:    add.s16 %rs4, %rs2, -1;
 ; SM60-NEXT:    setp.eq.b16 %p1, %rs1, 0;
 ; SM60-NEXT:    setp.eq.b16 %p2, %rs2, 0;
+; SM60-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NEXT:    setp.gt.u16 %p3, %rs1, %rs5;
 ; SM60-NEXT:    setp.gt.u16 %p4, %rs2, %rs6;
 ; SM60-NEXT:    selp.b16 %rs7, %rs6, %rs4, %p4;
@@ -7415,8 +7415,6 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NEXT:    ld.volatile.global.b64 %rd2, [%rd1];
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
-; SM60-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM60-NEXT:  $L__BB198_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -7433,8 +7431,10 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NEXT:    setp.eq.b16 %p2, %rs2, 0;
 ; SM60-NEXT:    setp.eq.b16 %p3, %rs5, 0;
 ; SM60-NEXT:    setp.eq.b16 %p4, %rs6, 0;
+; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM60-NEXT:    setp.gt.u16 %p5, %rs1, %rs9;
 ; SM60-NEXT:    setp.gt.u16 %p6, %rs2, %rs10;
+; SM60-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM60-NEXT:    setp.gt.u16 %p7, %rs5, %rs11;
 ; SM60-NEXT:    setp.gt.u16 %p8, %rs6, %rs12;
 ; SM60-NEXT:    selp.b16 %rs13, %rs12, %rs8, %p8;
@@ -7479,8 +7479,6 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NEXT:    ld.volatile.global.b64 %rd2, [%rd1];
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
-; SM60-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM60-NEXT:  $L__BB199_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -7497,8 +7495,10 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NEXT:    setp.eq.b16 %p2, %rs2, 0;
 ; SM60-NEXT:    setp.eq.b16 %p3, %rs5, 0;
 ; SM60-NEXT:    setp.eq.b16 %p4, %rs6, 0;
+; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM60-NEXT:    setp.gt.u16 %p5, %rs1, %rs9;
 ; SM60-NEXT:    setp.gt.u16 %p6, %rs2, %rs10;
+; SM60-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM60-NEXT:    setp.gt.u16 %p7, %rs5, %rs11;
 ; SM60-NEXT:    setp.gt.u16 %p8, %rs6, %rs12;
 ; SM60-NEXT:    selp.b16 %rs13, %rs12, %rs8, %p8;
@@ -7994,13 +7994,13 @@ define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM60-NEXT:    not.b32 %r2, %r7;
 ; SM60-NEXT:    ld.volatile.global.b32 %r14, [%rd1];
-; SM60-NEXT:    mov.b32 {%rs6, %rs7}, %r4;
 ; SM60-NEXT:  $L__BB209_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    shr.u32 %r8, %r14, %r1;
 ; SM60-NEXT:    cvt.u16.u32 %rs3, %r8;
 ; SM60-NEXT:    and.b16 %rs4, %rs3, 255;
 ; SM60-NEXT:    shr.u16 %rs5, %rs3, 8;
+; SM60-NEXT:    mov.b32 {%rs6, %rs7}, %r4;
 ; SM60-NEXT:    setp.ge.u16 %p1, %rs5, %rs7;
 ; SM60-NEXT:    setp.ge.u16 %p2, %rs4, %rs6;
 ; SM60-NEXT:    sub.s16 %rs8, %rs5, %rs7;
@@ -8041,13 +8041,12 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NEXT:    membar.cta;
 ; SM60-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
 ; SM60-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM60-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM60-NEXT:    cvt.u16.u32 %rs3, %r5;
 ; SM60-NEXT:    cvt.u16.u32 %rs4, %r3;
 ; SM60-NEXT:  $L__BB210_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    prmt.b32 %r4, %r22, 0, 0x7770U;
 ; SM60-NEXT:    setp.ge.u32 %p1, %r4, %r3;
+; SM60-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM60-NEXT:    prmt.b32 %r6, %r22, 0, 0x7771U;
 ; SM60-NEXT:    setp.ge.u32 %p2, %r6, %r5;
 ; SM60-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -8058,6 +8057,7 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NEXT:    setp.ge.u32 %p4, %r10, %r9;
 ; SM60-NEXT:    cvt.u16.u32 %rs1, %r9;
 ; SM60-NEXT:    cvt.u16.u32 %rs2, %r7;
+; SM60-NEXT:    cvt.u16.u32 %rs3, %r5;
 ; SM60-NEXT:    cvt.u16.u32 %rs5, %r4;
 ; SM60-NEXT:    sub.s16 %rs6, %rs5, %rs4;
 ; SM60-NEXT:    cvt.u16.u32 %rs7, %r6;
@@ -8292,7 +8292,6 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM60-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; SM60-NEXT:  $L__BB214_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -8302,6 +8301,7 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NEXT:    mov.b32 {%rs3, %rs4}, %r6;
 ; SM60-NEXT:    setp.ge.u16 %p1, %rs3, %rs1;
 ; SM60-NEXT:    setp.ge.u16 %p2, %rs4, %rs2;
+; SM60-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r5;
 ; SM60-NEXT:    setp.ge.u16 %p3, %rs7, %rs5;
 ; SM60-NEXT:    setp.ge.u16 %p4, %rs8, %rs6;
@@ -8348,7 +8348,6 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM60-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; SM60-NEXT:  $L__BB215_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -8358,6 +8357,7 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM60-NEXT:    setp.ge.u16 %p1, %rs3, %rs1;
 ; SM60-NEXT:    setp.ge.u16 %p2, %rs4, %rs2;
+; SM60-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r9;
 ; SM60-NEXT:    setp.ge.u16 %p3, %rs7, %rs5;
 ; SM60-NEXT:    setp.ge.u16 %p4, %rs8, %rs6;
@@ -8989,7 +8989,6 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
 ; SM60-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM60-NEXT:    not.b32 %r2, %r7;
 ; SM60-NEXT:    ld.volatile.global.b32 %r20, [%rd1];
-; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM60-NEXT:  $L__BB225_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    shr.u32 %r8, %r20, %r1;
@@ -8998,6 +8997,7 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
 ; SM60-NEXT:    mov.b32 %r9, {%rs3, %rs4};
 ; SM60-NEXT:    and.b32 %r10, %r9, 16711935;
 ; SM60-NEXT:    mov.b32 {%rs5, %rs6}, %r10;
+; SM60-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM60-NEXT:    sub.s16 %rs9, %rs6, %rs8;
 ; SM60-NEXT:    setp.gt.u16 %p1, %rs9, %rs6;
 ; SM60-NEXT:    selp.b16 %rs10, -1, 0, %p1;
@@ -9043,12 +9043,6 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
 ; SM60-NEXT:    ld.volatile.global.b32 %r18, [%rd1];
 ; SM60-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM60-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM60-NEXT:    cvt.u16.u32 %rs5, %r6;
-; SM60-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM60-NEXT:    cvt.u16.u32 %rs9, %r10;
-; SM60-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM60-NEXT:    cvt.u16.u32 %rs13, %r13;
 ; SM60-NEXT:  $L__BB226_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
@@ -9056,17 +9050,23 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
 ; SM60-NEXT:    max.u16 %rs3, %rs2, %rs1;
 ; SM60-NEXT:    sub.s16 %rs4, %rs3, %rs1;
 ; SM60-NEXT:    cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT:    cvt.u16.u32 %rs5, %r6;
 ; SM60-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM60-NEXT:    cvt.u16.u32 %rs6, %r7;
 ; SM60-NEXT:    max.u16 %rs7, %rs6, %rs5;
 ; SM60-NEXT:    sub.s16 %rs8, %rs7, %rs5;
 ; SM60-NEXT:    cvt.u32.u16 %r8, %rs8;
 ; SM60-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT:    cvt.u16.u32 %rs9, %r10;
 ; SM60-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM60-NEXT:    cvt.u16.u32 %rs10, %r11;
 ; SM60-NEXT:    max.u16 %rs11, %rs10, %rs9;
 ; SM60-NEXT:    sub.s16 %rs12, %rs11, %rs9;
 ; SM60-NEXT:    cvt.u32.u16 %r12, %rs12;
+; SM60-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT:    cvt.u16.u32 %rs13, %r13;
 ; SM60-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM60-NEXT:    cvt.u16.u32 %rs14, %r14;
 ; SM60-NEXT:    max.u16 %rs15, %rs14, %rs13;
@@ -9283,7 +9283,6 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r11, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM60-NEXT:  $L__BB230_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    mov.b32 {%rs3, %rs4}, %r11;
@@ -9298,6 +9297,7 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NEXT:    xor.b32 %r5, %r4, -1;
 ; SM60-NEXT:    and.b32 %r6, %r3, %r5;
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r6;
+; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs11, %rs12}, %r12;
 ; SM60-NEXT:    sub.s16 %rs13, %rs12, %rs10;
 ; SM60-NEXT:    sub.s16 %rs14, %rs11, %rs9;
@@ -9345,7 +9345,6 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM60-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM60-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM60-NEXT:  $L__BB231_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
@@ -9360,6 +9359,7 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NEXT:    xor.b32 %r7, %r6, -1;
 ; SM60-NEXT:    and.b32 %r8, %r5, %r7;
 ; SM60-NEXT:    cvt.u64.u32 %rd3, %r8;
+; SM60-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM60-NEXT:    mov.b32 {%rs11, %rs12}, %r22;
 ; SM60-NEXT:    sub.s16 %rs13, %rs12, %rs10;
 ; SM60-NEXT:    sub.s16 %rs14, %rs11, %rs9;
@@ -17504,12 +17504,12 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB296_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    min.f32 %r10, %r8, %r9;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -17544,12 +17544,12 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB296_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-ALLOW-NEXT:    min.f32 %r10, %r8, %r9;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -17584,12 +17584,12 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB296_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r10, %r8, %r9;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -17624,12 +17624,12 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB296_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-ALLOW-NEXT:    min.ftz.f32 %r10, %r8, %r9;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -17664,13 +17664,13 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r10, [%rd1];
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB297_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    min.f32 %r5, %r4, %r3;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
 ; SM60-NOFTZ-DISALLOW-NEXT:    min.f32 %r8, %r7, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -17698,13 +17698,13 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r10, [%rd1];
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB297_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM60-NOFTZ-ALLOW-NEXT:    min.f32 %r5, %r4, %r3;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
 ; SM60-NOFTZ-ALLOW-NEXT:    min.f32 %r8, %r7, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -17732,13 +17732,13 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r10, [%rd1];
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB297_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM60-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r5, %r4, %r3;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
 ; SM60-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r8, %r7, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -17766,13 +17766,13 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r10, [%rd1];
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB297_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM60-FTZ-ALLOW-NEXT:    min.ftz.f32 %r5, %r4, %r3;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
 ; SM60-FTZ-ALLOW-NEXT:    min.ftz.f32 %r8, %r7, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -17805,11 +17805,9 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB298_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    min.f32 %r5, %r4, %r3;
@@ -17820,6 +17818,8 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r11, %rs10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    min.f32 %r12, %r11, %r10;
@@ -17861,11 +17861,9 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB298_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM60-NOFTZ-ALLOW-NEXT:    min.f32 %r5, %r4, %r3;
@@ -17876,6 +17874,8 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r11, %rs10;
 ; SM60-NOFTZ-ALLOW-NEXT:    min.f32 %r12, %r11, %r10;
@@ -17917,11 +17917,9 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB298_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM60-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r5, %r4, %r3;
@@ -17932,6 +17930,8 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r11, %rs10;
 ; SM60-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r12, %r11, %r10;
@@ -17973,11 +17973,9 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB298_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM60-FTZ-ALLOW-NEXT:    min.ftz.f32 %r5, %r4, %r3;
@@ -17988,6 +17986,8 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r11, %rs10;
 ; SM60-FTZ-ALLOW-NEXT:    min.ftz.f32 %r12, %r11, %r10;
@@ -18033,7 +18033,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB299_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r5, %rs2;
@@ -18047,6 +18046,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r12, %rs8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r13, %rs10;
@@ -18074,7 +18074,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB299_3: // %atomicrmw.start2
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r19, %rs14;
@@ -18088,6 +18087,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r26, %rs20;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r27, %rs22;
@@ -18130,7 +18130,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB299_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r5, %rs2;
@@ -18144,6 +18143,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r12, %rs8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r13, %rs10;
@@ -18171,7 +18171,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB299_3: // %atomicrmw.start2
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r19, %rs14;
@@ -18185,6 +18184,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r26, %rs20;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r27, %rs22;
@@ -18227,7 +18227,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB299_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r5, %rs2;
@@ -18241,6 +18240,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r12, %rs8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs10;
@@ -18268,7 +18268,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB299_3: // %atomicrmw.start2
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r19, %rs14;
@@ -18282,6 +18281,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r26, %rs20;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r27, %rs22;
@@ -18324,7 +18324,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB299_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r5, %rs2;
@@ -18338,6 +18337,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r12, %rs8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs10;
@@ -18365,7 +18365,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB299_3: // %atomicrmw.start2
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r19, %rs14;
@@ -18379,6 +18378,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r26, %rs20;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r27, %rs22;
@@ -18428,12 +18428,12 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB300_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    max.f32 %r10, %r8, %r9;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -18468,12 +18468,12 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB300_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-ALLOW-NEXT:    max.f32 %r10, %r8, %r9;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -18508,12 +18508,12 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB300_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r10, %r8, %r9;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -18548,12 +18548,12 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB300_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-ALLOW-NEXT:    max.ftz.f32 %r10, %r8, %r9;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -18588,13 +18588,13 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r10, [%rd1];
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB301_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    max.f32 %r5, %r4, %r3;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
 ; SM60-NOFTZ-DISALLOW-NEXT:    max.f32 %r8, %r7, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -18622,13 +18622,13 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r10, [%rd1];
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB301_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM60-NOFTZ-ALLOW-NEXT:    max.f32 %r5, %r4, %r3;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
 ; SM60-NOFTZ-ALLOW-NEXT:    max.f32 %r8, %r7, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -18656,13 +18656,13 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r10, [%rd1];
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB301_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM60-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r5, %r4, %r3;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
 ; SM60-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r8, %r7, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -18690,13 +18690,13 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r10, [%rd1];
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB301_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM60-FTZ-ALLOW-NEXT:    max.ftz.f32 %r5, %r4, %r3;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
 ; SM60-FTZ-ALLOW-NEXT:    max.ftz.f32 %r8, %r7, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -18729,11 +18729,9 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB302_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    max.f32 %r5, %r4, %r3;
@@ -18744,6 +18742,8 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r11, %rs10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    max.f32 %r12, %r11, %r10;
@@ -18785,11 +18785,9 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB302_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM60-NOFTZ-ALLOW-NEXT:    max.f32 %r5, %r4, %r3;
@@ -18800,6 +18798,8 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r11, %rs10;
 ; SM60-NOFTZ-ALLOW-NEXT:    max.f32 %r12, %r11, %r10;
@@ -18841,11 +18841,9 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB302_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM60-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r5, %r4, %r3;
@@ -18856,6 +18854,8 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r11, %rs10;
 ; SM60-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r12, %r11, %r10;
@@ -18897,11 +18897,9 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB302_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM60-FTZ-ALLOW-NEXT:    max.ftz.f32 %r5, %r4, %r3;
@@ -18912,6 +18910,8 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r11, %rs10;
 ; SM60-FTZ-ALLOW-NEXT:    max.ftz.f32 %r12, %r11, %r10;
@@ -18957,7 +18957,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB303_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r5, %rs2;
@@ -18971,6 +18970,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r12, %rs8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r13, %rs10;
@@ -18998,7 +18998,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB303_3: // %atomicrmw.start2
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r19, %rs14;
@@ -19012,6 +19011,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r26, %rs20;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r27, %rs22;
@@ -19054,7 +19054,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB303_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r5, %rs2;
@@ -19068,6 +19067,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r12, %rs8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r13, %rs10;
@@ -19095,7 +19095,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB303_3: // %atomicrmw.start2
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r19, %rs14;
@@ -19109,6 +19108,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r26, %rs20;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r27, %rs22;
@@ -19151,7 +19151,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB303_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r5, %rs2;
@@ -19165,6 +19164,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r12, %rs8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs10;
@@ -19192,7 +19192,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB303_3: // %atomicrmw.start2
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r19, %rs14;
@@ -19206,6 +19205,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r26, %rs20;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r27, %rs22;
@@ -19248,7 +19248,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB303_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r5, %rs2;
@@ -19262,6 +19261,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r12, %rs8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs10;
@@ -19289,7 +19289,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB303_3: // %atomicrmw.start2
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r19, %rs14;
@@ -19303,6 +19302,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r26, %rs20;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r27, %rs22;
@@ -19701,7 +19701,6 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB306_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -19723,6 +19722,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -19773,7 +19773,6 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB306_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -19795,6 +19794,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -19845,7 +19845,6 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB306_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -19867,6 +19866,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -19917,7 +19917,6 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB306_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -19939,6 +19938,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -19993,7 +19993,6 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB307_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -20015,6 +20014,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -20122,7 +20122,6 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB307_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -20144,6 +20143,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -20251,7 +20251,6 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB307_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -20273,6 +20272,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -20380,7 +20380,6 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB307_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -20402,6 +20401,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -20865,7 +20865,6 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB310_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -20887,6 +20886,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -20937,7 +20937,6 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB310_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -20959,6 +20958,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -21009,7 +21009,6 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB310_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -21031,6 +21030,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -21081,7 +21081,6 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB310_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -21103,6 +21102,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -21157,7 +21157,6 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB311_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -21179,6 +21178,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -21286,7 +21286,6 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB311_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -21308,6 +21307,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -21415,7 +21415,6 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB311_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -21437,6 +21436,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -21544,7 +21544,6 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB311_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -21566,6 +21565,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -21680,12 +21680,12 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB312_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -21725,12 +21725,12 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB312_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -21770,12 +21770,12 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB312_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -21815,12 +21815,12 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB312_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -21861,8 +21861,6 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB313_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -21875,6 +21873,8 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM60-NOFTZ-DISALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r17, %r16, %r14;
@@ -21909,8 +21909,6 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB313_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -21923,6 +21921,8 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM60-NOFTZ-ALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r17, %r16, %r14;
@@ -21957,8 +21957,6 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB313_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -21971,6 +21969,8 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM60-FTZ-DISALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r17, %r16, %r14;
@@ -22005,8 +22005,6 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB313_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -22019,6 +22017,8 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM60-FTZ-ALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r17, %r16, %r14;
@@ -22057,13 +22057,10 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB314_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -22087,6 +22084,9 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -22141,13 +22141,10 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB314_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -22171,6 +22168,9 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-NOFTZ-ALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -22225,13 +22225,10 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB314_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -22255,6 +22252,9 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-FTZ-DISALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -22309,13 +22309,10 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB314_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -22339,6 +22336,9 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-FTZ-ALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -22397,7 +22397,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB315_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -22425,6 +22424,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -22466,7 +22466,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB315_3: // %atomicrmw.start2
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -22494,6 +22493,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -22550,7 +22550,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB315_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -22578,6 +22577,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -22619,7 +22619,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB315_3: // %atomicrmw.start2
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -22647,6 +22646,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-NOFTZ-ALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -22703,7 +22703,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB315_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -22731,6 +22730,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -22772,7 +22772,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB315_3: // %atomicrmw.start2
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -22800,6 +22799,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-FTZ-DISALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -22856,7 +22856,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB315_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -22884,6 +22883,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -22925,7 +22925,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB315_3: // %atomicrmw.start2
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -22953,6 +22952,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-FTZ-ALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -23016,12 +23016,12 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB316_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -23061,12 +23061,12 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB316_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -23106,12 +23106,12 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB316_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -23151,12 +23151,12 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB316_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -23197,8 +23197,6 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB317_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -23211,6 +23209,8 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM60-NOFTZ-DISALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r17, %r16, %r14;
@@ -23245,8 +23245,6 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB317_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -23259,6 +23257,8 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM60-NOFTZ-ALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r17, %r16, %r14;
@@ -23293,8 +23293,6 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB317_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -23307,6 +23305,8 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM60-FTZ-DISALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r17, %r16, %r14;
@@ -23341,8 +23341,6 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB317_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -23355,6 +23353,8 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM60-FTZ-ALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r17, %r16, %r14;
@@ -23393,13 +23393,10 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB318_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -23423,6 +23420,9 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -23477,13 +23477,10 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB318_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -23507,6 +23504,9 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-NOFTZ-ALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -23561,13 +23561,10 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB318_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -23591,6 +23588,9 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-FTZ-DISALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -23645,13 +23645,10 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB318_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -23675,6 +23672,9 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-FTZ-ALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -23733,7 +23733,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB319_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -23761,6 +23760,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -23802,7 +23802,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB319_3: // %atomicrmw.start2
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -23830,6 +23829,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -23886,7 +23886,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB319_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -23914,6 +23913,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -23955,7 +23955,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB319_3: // %atomicrmw.start2
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -23983,6 +23982,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-NOFTZ-ALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -24039,7 +24039,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB319_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -24067,6 +24066,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -24108,7 +24108,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB319_3: // %atomicrmw.start2
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -24136,6 +24135,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-FTZ-DISALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -24192,7 +24192,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB319_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -24220,6 +24219,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -24261,7 +24261,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB319_3: // %atomicrmw.start2
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -24289,6 +24288,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-FTZ-ALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -24352,12 +24352,12 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB320_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    min.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -24397,12 +24397,12 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB320_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    min.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -24442,12 +24442,12 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB320_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -24487,12 +24487,12 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB320_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    min.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -24533,8 +24533,6 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB321_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -24547,6 +24545,8 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM60-NOFTZ-DISALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    min.f32 %r17, %r16, %r14;
@@ -24581,8 +24581,6 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB321_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -24595,6 +24593,8 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM60-NOFTZ-ALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    min.f32 %r17, %r16, %r14;
@@ -24629,8 +24629,6 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB321_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -24643,6 +24641,8 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM60-FTZ-DISALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r17, %r16, %r14;
@@ -24677,8 +24677,6 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB321_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -24691,6 +24689,8 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM60-FTZ-ALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-FTZ-ALLOW-NEXT:    min.ftz.f32 %r17, %r16, %r14;
@@ -24729,13 +24729,10 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB322_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -24759,6 +24756,9 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -24813,13 +24813,10 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB322_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -24843,6 +24840,9 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-NOFTZ-ALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -24897,13 +24897,10 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB322_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -24927,7 +24924,10 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-FTZ-DISALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r28, %r27, %r25;
@@ -24981,13 +24981,10 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB322_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -25011,6 +25008,9 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-FTZ-ALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -25069,7 +25069,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB323_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -25097,6 +25096,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -25138,7 +25138,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB323_3: // %atomicrmw.start2
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -25166,6 +25165,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -25222,7 +25222,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB323_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -25250,6 +25249,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -25291,7 +25291,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB323_3: // %atomicrmw.start2
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -25319,6 +25318,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-NOFTZ-ALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -25375,7 +25375,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB323_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -25403,6 +25402,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -25444,7 +25444,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB323_3: // %atomicrmw.start2
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -25472,6 +25471,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-FTZ-DISALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -25528,7 +25528,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB323_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -25556,6 +25555,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -25597,7 +25597,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB323_3: // %atomicrmw.start2
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -25625,6 +25624,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-FTZ-ALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -25688,12 +25688,12 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB324_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    max.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -25733,12 +25733,12 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB324_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    max.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -25778,12 +25778,12 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB324_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -25823,12 +25823,12 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB324_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    max.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -25869,8 +25869,6 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB325_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -25883,6 +25881,8 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM60-NOFTZ-DISALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    max.f32 %r17, %r16, %r14;
@@ -25917,8 +25917,6 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB325_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -25931,6 +25929,8 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM60-NOFTZ-ALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    max.f32 %r17, %r16, %r14;
@@ -25965,8 +25965,6 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB325_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -25979,6 +25977,8 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM60-FTZ-DISALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r17, %r16, %r14;
@@ -26013,8 +26013,6 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB325_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r24;
@@ -26027,6 +26025,8 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM60-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM60-FTZ-ALLOW-NEXT:    or.b32 %r11, %r7, 4194304;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r14, %r13, 16;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r16, %r15, 16;
 ; SM60-FTZ-ALLOW-NEXT:    max.ftz.f32 %r17, %r16, %r14;
@@ -26065,13 +26065,10 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB326_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -26095,6 +26092,9 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -26149,13 +26149,10 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB326_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -26179,6 +26176,9 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-NOFTZ-ALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -26233,13 +26233,10 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB326_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -26263,6 +26260,9 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-FTZ-DISALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -26317,13 +26317,10 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB326_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r45;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -26347,6 +26344,9 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
 ; SM60-FTZ-ALLOW-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r23;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r24, %rs6;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r25, %r24, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r46;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs8;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
@@ -26405,7 +26405,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB327_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -26433,6 +26432,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -26474,7 +26474,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB327_3: // %atomicrmw.start2
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -26502,6 +26501,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -26558,7 +26558,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB327_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -26586,6 +26585,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -26627,7 +26627,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB327_3: // %atomicrmw.start2
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -26655,6 +26654,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-NOFTZ-ALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -26711,7 +26711,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB327_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -26739,6 +26738,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -26780,7 +26780,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB327_3: // %atomicrmw.start2
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -26808,6 +26807,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-FTZ-DISALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -26864,7 +26864,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r89, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB327_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -26892,6 +26891,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM60-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r14, 0x7632U;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r90;
@@ -26933,7 +26933,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r91, %rd12;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB327_3: // %atomicrmw.start2
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r47, %rs10;
@@ -26961,6 +26960,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM60-FTZ-ALLOW-NEXT:    selp.b32 %r66, %r65, %r64, %p7;
 ; SM60-FTZ-ALLOW-NEXT:    prmt.b32 %r67, %r66, %r56, 0x7632U;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r67;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r68, %rs14;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r69, %r68, 16;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r92;
@@ -27024,8 +27024,6 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB328_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -27033,6 +27031,8 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p2, %r8, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -27071,8 +27071,6 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB328_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -27080,6 +27078,8 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p2, %r8, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -27118,8 +27118,6 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB328_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -27127,6 +27125,8 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r8, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -27165,8 +27165,6 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB328_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -27174,6 +27172,8 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r8, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -27213,8 +27213,6 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB329_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -27230,6 +27228,8 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p6, %r10, %r8;
@@ -27263,8 +27263,6 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB329_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -27280,6 +27278,8 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p6, %r10, %r8;
@@ -27313,8 +27313,6 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB329_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -27330,6 +27328,8 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p6, %r10, %r8;
@@ -27363,8 +27363,6 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB329_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -27380,6 +27378,8 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p6, %r10, %r8;
@@ -27417,16 +27417,13 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB330_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p2, %r6, %r4;
@@ -27449,9 +27446,12 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p10, %r15, %r13;
@@ -27505,16 +27505,13 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB330_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p2, %r6, %r4;
@@ -27537,9 +27534,12 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p10, %r15, %r13;
@@ -27593,16 +27593,13 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB330_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r6, %r4;
@@ -27625,9 +27622,12 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p10, %r15, %r13;
@@ -27681,16 +27681,13 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB330_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM60-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r6, %r4;
@@ -27713,9 +27710,12 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM60-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p10, %r15, %r13;
@@ -27773,7 +27773,6 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB331_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -27803,6 +27802,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -27934,7 +27934,6 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB331_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -27964,6 +27963,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -28095,7 +28095,6 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB331_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -28125,6 +28124,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -28256,7 +28256,6 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB331_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -28286,6 +28285,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -28424,8 +28424,6 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB332_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -28433,6 +28431,8 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p2, %r8, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -28471,8 +28471,6 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB332_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -28480,6 +28478,8 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p2, %r8, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -28518,8 +28518,6 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB332_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -28527,6 +28525,8 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r8, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -28565,8 +28565,6 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB332_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -28574,6 +28572,8 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r8, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -28613,8 +28613,6 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB333_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -28630,6 +28628,8 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p6, %r10, %r8;
@@ -28663,8 +28663,6 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB333_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -28680,6 +28678,8 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p6, %r10, %r8;
@@ -28713,8 +28713,6 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB333_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -28730,6 +28728,8 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p6, %r10, %r8;
@@ -28763,8 +28763,6 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB333_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -28780,6 +28778,8 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p6, %r10, %r8;
@@ -28817,16 +28817,13 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB334_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p2, %r6, %r4;
@@ -28849,9 +28846,12 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p10, %r15, %r13;
@@ -28905,16 +28905,13 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB334_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p2, %r6, %r4;
@@ -28937,9 +28934,12 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p10, %r15, %r13;
@@ -28993,16 +28993,13 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB334_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r6, %r4;
@@ -29025,9 +29022,12 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p10, %r15, %r13;
@@ -29081,16 +29081,13 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB334_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM60-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r6, %r4;
@@ -29113,9 +29110,12 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM60-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p10, %r15, %r13;
@@ -29173,7 +29173,6 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB335_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -29203,6 +29202,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -29334,7 +29334,6 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB335_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -29364,6 +29363,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -29495,7 +29495,6 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB335_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -29525,6 +29524,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -29656,7 +29656,6 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB335_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -29686,6 +29685,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM60-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -29901,27 +29901,27 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
 ; SM60-NEXT:    ld.volatile.global.b32 %r18, [%rd1];
 ; SM60-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM60-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM60-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM60-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM60-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM60-NEXT:  $L__BB338_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM60-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM60-NEXT:    add.s16 %rs3, %rs2, %rs1;
 ; SM60-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM60-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM60-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM60-NEXT:    add.s16 %rs6, %rs5, %rs4;
 ; SM60-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM60-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM60-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM60-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM60-NEXT:    add.s16 %rs9, %rs8, %rs7;
 ; SM60-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM60-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM60-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM60-NEXT:    add.s16 %rs12, %rs11, %rs10;
@@ -30127,27 +30127,27 @@ define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM60-NEXT:    ld.volatile.global.b32 %r18, [%rd1];
 ; SM60-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM60-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM60-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM60-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM60-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM60-NEXT:  $L__BB342_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM60-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM60-NEXT:    add.s16 %rs3, %rs2, %rs1;
 ; SM60-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM60-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM60-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM60-NEXT:    add.s16 %rs6, %rs5, %rs4;
 ; SM60-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM60-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM60-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM60-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM60-NEXT:    add.s16 %rs9, %rs8, %rs7;
 ; SM60-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM60-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM60-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM60-NEXT:    add.s16 %rs12, %rs11, %rs10;
@@ -30356,27 +30356,27 @@ define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM60-NEXT:    ld.volatile.global.b32 %r18, [%rd1];
 ; SM60-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM60-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM60-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM60-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM60-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM60-NEXT:  $L__BB346_1: // %atomicrmw.start
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM60-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM60-NEXT:    add.s16 %rs3, %rs2, %rs1;
 ; SM60-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM60-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM60-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM60-NEXT:    add.s16 %rs6, %rs5, %rs4;
 ; SM60-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM60-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM60-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM60-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM60-NEXT:    add.s16 %rs9, %rs8, %rs7;
 ; SM60-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM60-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM60-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM60-NEXT:    add.s16 %rs12, %rs11, %rs10;
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
index 52441453ce093a..cf668b57451435 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
@@ -496,27 +496,27 @@ define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM70-NEXT:  $L__BB18_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM70-NEXT:    add.s16 %rs3, %rs2, %rs1;
 ; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM70-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM70-NEXT:    add.s16 %rs6, %rs5, %rs4;
 ; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM70-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM70-NEXT:    add.s16 %rs9, %rs8, %rs7;
 ; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM70-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM70-NEXT:    add.s16 %rs12, %rs11, %rs10;
@@ -713,7 +713,6 @@ define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:  $L__BB22_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -721,6 +720,7 @@ define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM70-NEXT:    add.s16 %rs6, %rs3, %rs1;
 ; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
 ; SM70-NEXT:    add.s16 %rs11, %rs10, %rs8;
 ; SM70-NEXT:    add.s16 %rs12, %rs9, %rs7;
@@ -761,7 +761,6 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:  $L__BB23_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -769,6 +768,7 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    add.s16 %rs6, %rs3, %rs1;
 ; SM70-NEXT:    mov.b32 %r5, {%rs6, %rs5};
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
 ; SM70-NEXT:    add.s16 %rs11, %rs10, %rs8;
 ; SM70-NEXT:    add.s16 %rs12, %rs9, %rs7;
@@ -790,7 +790,6 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
 ; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
 ; SM70-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NEXT:  $L__BB23_3: // %atomicrmw.start2
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    mov.b32 {%rs15, %rs16}, %r11;
@@ -798,6 +797,7 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    add.s16 %rs18, %rs15, %rs13;
 ; SM70-NEXT:    mov.b32 %r7, {%rs18, %rs17};
 ; SM70-NEXT:    cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
 ; SM70-NEXT:    add.s16 %rs23, %rs22, %rs20;
 ; SM70-NEXT:    add.s16 %rs24, %rs21, %rs19;
@@ -1095,27 +1095,27 @@ define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM70-NEXT:  $L__BB34_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM70-NEXT:    sub.s16 %rs3, %rs2, %rs1;
 ; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM70-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM70-NEXT:    sub.s16 %rs6, %rs5, %rs4;
 ; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM70-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM70-NEXT:    sub.s16 %rs9, %rs8, %rs7;
 ; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM70-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM70-NEXT:    sub.s16 %rs12, %rs11, %rs10;
@@ -1312,7 +1312,6 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:  $L__BB38_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -1320,6 +1319,7 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM70-NEXT:    sub.s16 %rs6, %rs3, %rs1;
 ; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
 ; SM70-NEXT:    sub.s16 %rs11, %rs10, %rs8;
 ; SM70-NEXT:    sub.s16 %rs12, %rs9, %rs7;
@@ -1360,7 +1360,6 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:  $L__BB39_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -1368,6 +1367,7 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    sub.s16 %rs6, %rs3, %rs1;
 ; SM70-NEXT:    mov.b32 %r5, {%rs6, %rs5};
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
 ; SM70-NEXT:    sub.s16 %rs11, %rs10, %rs8;
 ; SM70-NEXT:    sub.s16 %rs12, %rs9, %rs7;
@@ -1389,7 +1389,6 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
 ; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
 ; SM70-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NEXT:  $L__BB39_3: // %atomicrmw.start2
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    mov.b32 {%rs15, %rs16}, %r11;
@@ -1397,6 +1396,7 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    sub.s16 %rs18, %rs15, %rs13;
 ; SM70-NEXT:    mov.b32 %r7, {%rs18, %rs17};
 ; SM70-NEXT:    cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
 ; SM70-NEXT:    sub.s16 %rs23, %rs22, %rs20;
 ; SM70-NEXT:    sub.s16 %rs24, %rs21, %rs19;
@@ -3829,11 +3829,11 @@ define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM70-NEXT:    fence.acq_rel.cta;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
 ; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x8880U;
-; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
 ; SM70-NEXT:  $L__BB114_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    prmt.b32 %r4, %r26, 0, 0x8880U;
 ; SM70-NEXT:    setp.gt.s32 %p1, %r4, %r3;
+; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
 ; SM70-NEXT:    prmt.b32 %r6, %r26, 0, 0x9991U;
 ; SM70-NEXT:    setp.gt.s32 %p2, %r6, %r5;
 ; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0xaaa2U;
@@ -4074,7 +4074,6 @@ define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:  $L__BB118_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -4086,6 +4085,7 @@ define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM70-NEXT:    max.s16 %rs6, %rs3, %rs1;
 ; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
 ; SM70-NEXT:    cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
 ; SM70-NEXT:    max.s16 %rs11, %rs10, %rs8;
 ; SM70-NEXT:    max.s16 %rs12, %rs9, %rs7;
@@ -4122,7 +4122,6 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:  $L__BB119_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -4134,6 +4133,7 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    max.s16 %rs6, %rs3, %rs1;
 ; SM70-NEXT:    mov.b32 %r5, {%rs6, %rs5};
 ; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
 ; SM70-NEXT:    max.s16 %rs11, %rs10, %rs8;
 ; SM70-NEXT:    max.s16 %rs12, %rs9, %rs7;
@@ -4151,7 +4151,6 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
 ; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
 ; SM70-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NEXT:  $L__BB119_3: // %atomicrmw.start2
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd13, %r11;
@@ -4163,6 +4162,7 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    max.s16 %rs18, %rs15, %rs13;
 ; SM70-NEXT:    mov.b32 %r7, {%rs18, %rs17};
 ; SM70-NEXT:    cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
 ; SM70-NEXT:    max.s16 %rs23, %rs22, %rs20;
 ; SM70-NEXT:    max.s16 %rs24, %rs21, %rs19;
@@ -4461,11 +4461,11 @@ define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM70-NEXT:    fence.acq_rel.cta;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
 ; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x8880U;
-; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
 ; SM70-NEXT:  $L__BB130_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    prmt.b32 %r4, %r26, 0, 0x8880U;
 ; SM70-NEXT:    setp.le.s32 %p1, %r4, %r3;
+; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
 ; SM70-NEXT:    prmt.b32 %r6, %r26, 0, 0x9991U;
 ; SM70-NEXT:    setp.le.s32 %p2, %r6, %r5;
 ; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0xaaa2U;
@@ -4706,7 +4706,6 @@ define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:  $L__BB134_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -4718,6 +4717,7 @@ define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM70-NEXT:    min.s16 %rs6, %rs3, %rs1;
 ; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
 ; SM70-NEXT:    cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
 ; SM70-NEXT:    min.s16 %rs11, %rs10, %rs8;
 ; SM70-NEXT:    min.s16 %rs12, %rs9, %rs7;
@@ -4754,7 +4754,6 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:  $L__BB135_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -4766,6 +4765,7 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    min.s16 %rs6, %rs3, %rs1;
 ; SM70-NEXT:    mov.b32 %r5, {%rs6, %rs5};
 ; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
 ; SM70-NEXT:    min.s16 %rs11, %rs10, %rs8;
 ; SM70-NEXT:    min.s16 %rs12, %rs9, %rs7;
@@ -4783,7 +4783,6 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
 ; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
 ; SM70-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NEXT:  $L__BB135_3: // %atomicrmw.start2
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd13, %r11;
@@ -4795,6 +4794,7 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM70-NEXT:    min.s16 %rs18, %rs15, %rs13;
 ; SM70-NEXT:    mov.b32 %r7, {%rs18, %rs17};
 ; SM70-NEXT:    cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
 ; SM70-NEXT:    min.s16 %rs23, %rs22, %rs20;
 ; SM70-NEXT:    min.s16 %rs24, %rs21, %rs19;
@@ -5049,7 +5049,6 @@ define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
 ; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM70-NEXT:    not.b32 %r2, %r7;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM70-NEXT:  $L__BB145_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    shr.u32 %r8, %r16, %r1;
@@ -5058,6 +5057,7 @@ define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
 ; SM70-NEXT:    mov.b32 %r9, {%rs3, %rs4};
 ; SM70-NEXT:    and.b32 %r10, %r9, 16711935;
 ; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r10;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM70-NEXT:    max.u16 %rs9, %rs5, %rs7;
 ; SM70-NEXT:    max.u16 %rs10, %rs6, %rs8;
 ; SM70-NEXT:    shl.b16 %rs11, %rs10, 8;
@@ -5092,11 +5092,11 @@ define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
 ; SM70-NEXT:    fence.acq_rel.cta;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM70-NEXT:  $L__BB146_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7770U;
 ; SM70-NEXT:    setp.gt.u32 %p1, %r4, %r3;
+; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM70-NEXT:    prmt.b32 %r6, %r18, 0, 0x7771U;
 ; SM70-NEXT:    setp.gt.u32 %p2, %r6, %r5;
 ; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -5313,7 +5313,6 @@ define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:  $L__BB150_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -5325,6 +5324,7 @@ define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
 ; SM70-NEXT:    max.u16 %rs6, %rs3, %rs1;
 ; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
 ; SM70-NEXT:    cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
 ; SM70-NEXT:    max.u16 %rs11, %rs10, %rs8;
 ; SM70-NEXT:    max.u16 %rs12, %rs9, %rs7;
@@ -5361,7 +5361,6 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:  $L__BB151_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -5373,6 +5372,7 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM70-NEXT:    max.u16 %rs6, %rs3, %rs1;
 ; SM70-NEXT:    mov.b32 %r5, {%rs6, %rs5};
 ; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
 ; SM70-NEXT:    max.u16 %rs11, %rs10, %rs8;
 ; SM70-NEXT:    max.u16 %rs12, %rs9, %rs7;
@@ -5390,7 +5390,6 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
 ; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
 ; SM70-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NEXT:  $L__BB151_3: // %atomicrmw.start2
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd13, %r11;
@@ -5402,6 +5401,7 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM70-NEXT:    max.u16 %rs18, %rs15, %rs13;
 ; SM70-NEXT:    mov.b32 %r7, {%rs18, %rs17};
 ; SM70-NEXT:    cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
 ; SM70-NEXT:    max.u16 %rs23, %rs22, %rs20;
 ; SM70-NEXT:    max.u16 %rs24, %rs21, %rs19;
@@ -5656,7 +5656,6 @@ define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
 ; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM70-NEXT:    not.b32 %r2, %r7;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM70-NEXT:  $L__BB161_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    shr.u32 %r8, %r16, %r1;
@@ -5665,6 +5664,7 @@ define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
 ; SM70-NEXT:    mov.b32 %r9, {%rs3, %rs4};
 ; SM70-NEXT:    and.b32 %r10, %r9, 16711935;
 ; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r10;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM70-NEXT:    min.u16 %rs9, %rs5, %rs7;
 ; SM70-NEXT:    min.u16 %rs10, %rs6, %rs8;
 ; SM70-NEXT:    shl.b16 %rs11, %rs10, 8;
@@ -5699,11 +5699,11 @@ define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
 ; SM70-NEXT:    fence.acq_rel.cta;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM70-NEXT:  $L__BB162_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7770U;
 ; SM70-NEXT:    setp.le.u32 %p1, %r4, %r3;
+; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM70-NEXT:    prmt.b32 %r6, %r18, 0, 0x7771U;
 ; SM70-NEXT:    setp.le.u32 %p2, %r6, %r5;
 ; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -5920,7 +5920,6 @@ define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:  $L__BB166_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -5932,6 +5931,7 @@ define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
 ; SM70-NEXT:    min.u16 %rs6, %rs3, %rs1;
 ; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
 ; SM70-NEXT:    cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
 ; SM70-NEXT:    min.u16 %rs11, %rs10, %rs8;
 ; SM70-NEXT:    min.u16 %rs12, %rs9, %rs7;
@@ -5968,7 +5968,6 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:  $L__BB167_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -5980,6 +5979,7 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM70-NEXT:    min.u16 %rs6, %rs3, %rs1;
 ; SM70-NEXT:    mov.b32 %r5, {%rs6, %rs5};
 ; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
 ; SM70-NEXT:    min.u16 %rs11, %rs10, %rs8;
 ; SM70-NEXT:    min.u16 %rs12, %rs9, %rs7;
@@ -5997,7 +5997,6 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
 ; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
 ; SM70-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NEXT:  $L__BB167_3: // %atomicrmw.start2
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd13, %r11;
@@ -6009,6 +6008,7 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
 ; SM70-NEXT:    min.u16 %rs18, %rs15, %rs13;
 ; SM70-NEXT:    mov.b32 %r7, {%rs18, %rs17};
 ; SM70-NEXT:    cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
 ; SM70-NEXT:    min.u16 %rs23, %rs22, %rs20;
 ; SM70-NEXT:    min.u16 %rs24, %rs21, %rs19;
@@ -6266,7 +6266,6 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM70-NEXT:    not.b32 %r2, %r7;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-NEXT:    mov.b32 {%rs8, %rs9}, %r4;
 ; SM70-NEXT:  $L__BB177_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    shr.u32 %r8, %r14, %r1;
@@ -6275,6 +6274,7 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NEXT:    shr.u16 %rs5, %rs3, 8;
 ; SM70-NEXT:    add.s16 %rs6, %rs5, 1;
 ; SM70-NEXT:    add.s16 %rs7, %rs3, 1;
+; SM70-NEXT:    mov.b32 {%rs8, %rs9}, %r4;
 ; SM70-NEXT:    setp.ge.u16 %p1, %rs5, %rs9;
 ; SM70-NEXT:    setp.ge.u16 %p2, %rs4, %rs8;
 ; SM70-NEXT:    selp.b16 %rs10, 0, %rs7, %p2;
@@ -6312,8 +6312,6 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i8_global_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r22, [%rd1];
-; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
-; SM70-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
 ; SM70-NEXT:  $L__BB178_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    prmt.b32 %r3, %r22, 0, 0x7770U;
@@ -6328,7 +6326,9 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NEXT:    prmt.b32 %r6, %r22, 0, 0x7773U;
 ; SM70-NEXT:    cvt.u16.u32 %rs7, %r6;
 ; SM70-NEXT:    add.s16 %rs8, %rs7, 1;
+; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
 ; SM70-NEXT:    setp.ge.u32 %p1, %r3, %r7;
+; SM70-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
 ; SM70-NEXT:    setp.ge.u32 %p2, %r4, %r8;
 ; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0x7772U;
 ; SM70-NEXT:    setp.ge.u32 %p3, %r5, %r9;
@@ -6513,12 +6513,12 @@ define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
 ; SM70-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i16_global_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NEXT:  $L__BB181_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; SM70-NEXT:    add.s16 %rs3, %rs1, 1;
 ; SM70-NEXT:    add.s16 %rs4, %rs2, 1;
+; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NEXT:    setp.ge.u16 %p1, %rs1, %rs5;
 ; SM70-NEXT:    setp.ge.u16 %p2, %rs2, %rs6;
 ; SM70-NEXT:    selp.b16 %rs7, 0, %rs4, %p2;
@@ -6551,8 +6551,6 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
-; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM70-NEXT:  $L__BB182_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -6565,8 +6563,10 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r5;
 ; SM70-NEXT:    add.s16 %rs7, %rs5, 1;
 ; SM70-NEXT:    add.s16 %rs8, %rs6, 1;
+; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM70-NEXT:    setp.ge.u16 %p1, %rs1, %rs9;
 ; SM70-NEXT:    setp.ge.u16 %p2, %rs2, %rs10;
+; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM70-NEXT:    setp.ge.u16 %p3, %rs5, %rs11;
 ; SM70-NEXT:    setp.ge.u16 %p4, %rs6, %rs12;
 ; SM70-NEXT:    selp.b16 %rs13, 0, %rs8, %p4;
@@ -6607,8 +6607,6 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
-; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM70-NEXT:  $L__BB183_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -6621,8 +6619,10 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r9;
 ; SM70-NEXT:    add.s16 %rs7, %rs5, 1;
 ; SM70-NEXT:    add.s16 %rs8, %rs6, 1;
+; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM70-NEXT:    setp.ge.u16 %p1, %rs1, %rs9;
 ; SM70-NEXT:    setp.ge.u16 %p2, %rs2, %rs10;
+; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM70-NEXT:    setp.ge.u16 %p3, %rs5, %rs11;
 ; SM70-NEXT:    setp.ge.u16 %p4, %rs6, %rs12;
 ; SM70-NEXT:    selp.b16 %rs13, 0, %rs8, %p4;
@@ -7078,7 +7078,6 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM70-NEXT:    not.b32 %r2, %r7;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-NEXT:    mov.b32 {%rs8, %rs9}, %r4;
 ; SM70-NEXT:  $L__BB193_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    shr.u32 %r8, %r14, %r1;
@@ -7089,6 +7088,7 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NEXT:    add.s16 %rs7, %rs3, -1;
 ; SM70-NEXT:    setp.eq.b16 %p1, %rs5, 0;
 ; SM70-NEXT:    setp.eq.b16 %p2, %rs4, 0;
+; SM70-NEXT:    mov.b32 {%rs8, %rs9}, %r4;
 ; SM70-NEXT:    setp.gt.u16 %p3, %rs5, %rs9;
 ; SM70-NEXT:    setp.gt.u16 %p4, %rs4, %rs8;
 ; SM70-NEXT:    selp.b16 %rs10, %rs8, %rs7, %p4;
@@ -7128,8 +7128,6 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i8_global_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
-; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
-; SM70-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
 ; SM70-NEXT:  $L__BB194_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    prmt.b32 %r3, %r26, 0, 0x7770U;
@@ -7148,7 +7146,9 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NEXT:    setp.eq.b32 %p2, %r4, 0;
 ; SM70-NEXT:    setp.eq.b32 %p3, %r5, 0;
 ; SM70-NEXT:    setp.eq.b32 %p4, %r6, 0;
+; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
 ; SM70-NEXT:    setp.gt.u32 %p5, %r3, %r7;
+; SM70-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
 ; SM70-NEXT:    setp.gt.u32 %p6, %r4, %r8;
 ; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0x7772U;
 ; SM70-NEXT:    setp.gt.u32 %p7, %r5, %r9;
@@ -7363,7 +7363,6 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
 ; SM70-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i16_global_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NEXT:  $L__BB197_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
@@ -7371,6 +7370,7 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
 ; SM70-NEXT:    add.s16 %rs4, %rs2, -1;
 ; SM70-NEXT:    setp.eq.b16 %p1, %rs1, 0;
 ; SM70-NEXT:    setp.eq.b16 %p2, %rs2, 0;
+; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NEXT:    setp.gt.u16 %p3, %rs1, %rs5;
 ; SM70-NEXT:    setp.gt.u16 %p4, %rs2, %rs6;
 ; SM70-NEXT:    selp.b16 %rs7, %rs6, %rs4, %p4;
@@ -7405,8 +7405,6 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
-; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM70-NEXT:  $L__BB198_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -7423,8 +7421,10 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NEXT:    setp.eq.b16 %p2, %rs2, 0;
 ; SM70-NEXT:    setp.eq.b16 %p3, %rs5, 0;
 ; SM70-NEXT:    setp.eq.b16 %p4, %rs6, 0;
+; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM70-NEXT:    setp.gt.u16 %p5, %rs1, %rs9;
 ; SM70-NEXT:    setp.gt.u16 %p6, %rs2, %rs10;
+; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM70-NEXT:    setp.gt.u16 %p7, %rs5, %rs11;
 ; SM70-NEXT:    setp.gt.u16 %p8, %rs6, %rs12;
 ; SM70-NEXT:    selp.b16 %rs13, %rs12, %rs8, %p8;
@@ -7469,8 +7469,6 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
-; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM70-NEXT:  $L__BB199_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -7487,8 +7485,10 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NEXT:    setp.eq.b16 %p2, %rs2, 0;
 ; SM70-NEXT:    setp.eq.b16 %p3, %rs5, 0;
 ; SM70-NEXT:    setp.eq.b16 %p4, %rs6, 0;
+; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM70-NEXT:    setp.gt.u16 %p5, %rs1, %rs9;
 ; SM70-NEXT:    setp.gt.u16 %p6, %rs2, %rs10;
+; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM70-NEXT:    setp.gt.u16 %p7, %rs5, %rs11;
 ; SM70-NEXT:    setp.gt.u16 %p8, %rs6, %rs12;
 ; SM70-NEXT:    selp.b16 %rs13, %rs12, %rs8, %p8;
@@ -7984,13 +7984,13 @@ define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM70-NEXT:    not.b32 %r2, %r7;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-NEXT:    mov.b32 {%rs6, %rs7}, %r4;
 ; SM70-NEXT:  $L__BB209_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    shr.u32 %r8, %r14, %r1;
 ; SM70-NEXT:    cvt.u16.u32 %rs3, %r8;
 ; SM70-NEXT:    and.b16 %rs4, %rs3, 255;
 ; SM70-NEXT:    shr.u16 %rs5, %rs3, 8;
+; SM70-NEXT:    mov.b32 {%rs6, %rs7}, %r4;
 ; SM70-NEXT:    setp.ge.u16 %p1, %rs5, %rs7;
 ; SM70-NEXT:    setp.ge.u16 %p2, %rs4, %rs6;
 ; SM70-NEXT:    sub.s16 %rs8, %rs5, %rs7;
@@ -8031,13 +8031,12 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NEXT:    fence.acq_rel.cta;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r22, [%rd1];
 ; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM70-NEXT:    cvt.u16.u32 %rs3, %r5;
 ; SM70-NEXT:    cvt.u16.u32 %rs4, %r3;
 ; SM70-NEXT:  $L__BB210_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    prmt.b32 %r4, %r22, 0, 0x7770U;
 ; SM70-NEXT:    setp.ge.u32 %p1, %r4, %r3;
+; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM70-NEXT:    prmt.b32 %r6, %r22, 0, 0x7771U;
 ; SM70-NEXT:    setp.ge.u32 %p2, %r6, %r5;
 ; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -8048,6 +8047,7 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NEXT:    setp.ge.u32 %p4, %r10, %r9;
 ; SM70-NEXT:    cvt.u16.u32 %rs1, %r9;
 ; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
+; SM70-NEXT:    cvt.u16.u32 %rs3, %r5;
 ; SM70-NEXT:    cvt.u16.u32 %rs5, %r4;
 ; SM70-NEXT:    sub.s16 %rs6, %rs5, %rs4;
 ; SM70-NEXT:    cvt.u16.u32 %rs7, %r6;
@@ -8282,7 +8282,6 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; SM70-NEXT:  $L__BB214_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -8292,6 +8291,7 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r6;
 ; SM70-NEXT:    setp.ge.u16 %p1, %rs3, %rs1;
 ; SM70-NEXT:    setp.ge.u16 %p2, %rs4, %rs2;
+; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r5;
 ; SM70-NEXT:    setp.ge.u16 %p3, %rs7, %rs5;
 ; SM70-NEXT:    setp.ge.u16 %p4, %rs8, %rs6;
@@ -8338,7 +8338,6 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; SM70-NEXT:  $L__BB215_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
@@ -8348,6 +8347,7 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM70-NEXT:    setp.ge.u16 %p1, %rs3, %rs1;
 ; SM70-NEXT:    setp.ge.u16 %p2, %rs4, %rs2;
+; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r9;
 ; SM70-NEXT:    setp.ge.u16 %p3, %rs7, %rs5;
 ; SM70-NEXT:    setp.ge.u16 %p4, %rs8, %rs6;
@@ -8979,7 +8979,6 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
 ; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM70-NEXT:    not.b32 %r2, %r7;
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r20, [%rd1];
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM70-NEXT:  $L__BB225_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    shr.u32 %r8, %r20, %r1;
@@ -8988,6 +8987,7 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
 ; SM70-NEXT:    mov.b32 %r9, {%rs3, %rs4};
 ; SM70-NEXT:    and.b32 %r10, %r9, 16711935;
 ; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r10;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM70-NEXT:    sub.s16 %rs9, %rs6, %rs8;
 ; SM70-NEXT:    setp.gt.u16 %p1, %rs9, %rs6;
 ; SM70-NEXT:    selp.b16 %rs10, -1, 0, %p1;
@@ -9033,12 +9033,6 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM70-NEXT:    cvt.u16.u32 %rs5, %r6;
-; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM70-NEXT:    cvt.u16.u32 %rs9, %r10;
-; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM70-NEXT:    cvt.u16.u32 %rs13, %r13;
 ; SM70-NEXT:  $L__BB226_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
@@ -9046,17 +9040,23 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
 ; SM70-NEXT:    max.u16 %rs3, %rs2, %rs1;
 ; SM70-NEXT:    sub.s16 %rs4, %rs3, %rs1;
 ; SM70-NEXT:    cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT:    cvt.u16.u32 %rs5, %r6;
 ; SM70-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM70-NEXT:    cvt.u16.u32 %rs6, %r7;
 ; SM70-NEXT:    max.u16 %rs7, %rs6, %rs5;
 ; SM70-NEXT:    sub.s16 %rs8, %rs7, %rs5;
 ; SM70-NEXT:    cvt.u32.u16 %r8, %rs8;
 ; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT:    cvt.u16.u32 %rs9, %r10;
 ; SM70-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM70-NEXT:    cvt.u16.u32 %rs10, %r11;
 ; SM70-NEXT:    max.u16 %rs11, %rs10, %rs9;
 ; SM70-NEXT:    sub.s16 %rs12, %rs11, %rs9;
 ; SM70-NEXT:    cvt.u32.u16 %r12, %rs12;
+; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT:    cvt.u16.u32 %rs13, %r13;
 ; SM70-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM70-NEXT:    cvt.u16.u32 %rs14, %r14;
 ; SM70-NEXT:    max.u16 %rs15, %rs14, %rs13;
@@ -9273,7 +9273,6 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r11, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM70-NEXT:  $L__BB230_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r11;
@@ -9288,6 +9287,7 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NEXT:    xor.b32 %r5, %r4, -1;
 ; SM70-NEXT:    and.b32 %r6, %r3, %r5;
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r6;
+; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r12;
 ; SM70-NEXT:    sub.s16 %rs13, %rs12, %rs10;
 ; SM70-NEXT:    sub.s16 %rs14, %rs11, %rs9;
@@ -9335,7 +9335,6 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM70-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM70-NEXT:  $L__BB231_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
@@ -9350,6 +9349,7 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NEXT:    xor.b32 %r7, %r6, -1;
 ; SM70-NEXT:    and.b32 %r8, %r5, %r7;
 ; SM70-NEXT:    cvt.u64.u32 %rd3, %r8;
+; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r22;
 ; SM70-NEXT:    sub.s16 %rs13, %rs12, %rs10;
 ; SM70-NEXT:    sub.s16 %rs14, %rs11, %rs9;
@@ -17089,12 +17089,12 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB296_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r10, %r8, %r9;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -17129,12 +17129,12 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB296_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r10, %r8, %r9;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -17169,12 +17169,12 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB296_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r10, %r8, %r9;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -17209,12 +17209,12 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB296_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r10, %r8, %r9;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -17249,13 +17249,13 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB297_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r5, %r4, %r3;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
 ; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r8, %r7, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -17283,13 +17283,13 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB297_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r5, %r4, %r3;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
 ; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r8, %r7, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -17317,13 +17317,13 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB297_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r5, %r4, %r3;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
 ; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r8, %r7, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -17351,13 +17351,13 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB297_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r5, %r4, %r3;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
 ; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r8, %r7, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -17390,11 +17390,9 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB298_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r5, %r4, %r3;
@@ -17405,6 +17403,8 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r11, %rs10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r12, %r11, %r10;
@@ -17446,11 +17446,9 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB298_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r5, %r4, %r3;
@@ -17461,6 +17459,8 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r11, %rs10;
 ; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r12, %r11, %r10;
@@ -17502,11 +17502,9 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB298_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r5, %r4, %r3;
@@ -17517,6 +17515,8 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r11, %rs10;
 ; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r12, %r11, %r10;
@@ -17558,11 +17558,9 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB298_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r5, %r4, %r3;
@@ -17573,6 +17571,8 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r11, %rs10;
 ; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r12, %r11, %r10;
@@ -17618,7 +17618,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB299_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r5, %rs2;
@@ -17632,6 +17631,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r12, %rs8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r13, %rs10;
@@ -17659,7 +17659,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB299_3: // %atomicrmw.start2
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r19, %rs14;
@@ -17673,6 +17672,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r26, %rs20;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r27, %rs22;
@@ -17715,7 +17715,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB299_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r5, %rs2;
@@ -17729,6 +17728,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r12, %rs8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r13, %rs10;
@@ -17756,7 +17756,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB299_3: // %atomicrmw.start2
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r19, %rs14;
@@ -17770,6 +17769,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r26, %rs20;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r27, %rs22;
@@ -17812,7 +17812,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB299_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r5, %rs2;
@@ -17826,6 +17825,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r12, %rs8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs10;
@@ -17853,7 +17853,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB299_3: // %atomicrmw.start2
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r19, %rs14;
@@ -17867,6 +17866,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r26, %rs20;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r27, %rs22;
@@ -17909,7 +17909,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB299_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r5, %rs2;
@@ -17923,6 +17922,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r12, %rs8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs10;
@@ -17950,7 +17950,6 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB299_3: // %atomicrmw.start2
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r19, %rs14;
@@ -17964,6 +17963,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r26, %rs20;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r27, %rs22;
@@ -18013,12 +18013,12 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB300_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r10, %r8, %r9;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -18053,12 +18053,12 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB300_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r10, %r8, %r9;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -18093,12 +18093,12 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB300_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r10, %r8, %r9;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -18133,12 +18133,12 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB300_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r10, %r8, %r9;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -18173,13 +18173,13 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB301_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r5, %r4, %r3;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
 ; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r8, %r7, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -18207,13 +18207,13 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB301_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r5, %r4, %r3;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
 ; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r8, %r7, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -18241,13 +18241,13 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB301_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r5, %r4, %r3;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
 ; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r8, %r7, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -18275,13 +18275,13 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB301_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r5, %r4, %r3;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
+; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
 ; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r8, %r7, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
@@ -18314,11 +18314,9 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB302_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r5, %r4, %r3;
@@ -18329,6 +18327,8 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r11, %rs10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r12, %r11, %r10;
@@ -18370,11 +18370,9 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB302_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
 ; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r5, %r4, %r3;
@@ -18385,6 +18383,8 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r11, %rs10;
 ; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r12, %r11, %r10;
@@ -18426,11 +18426,9 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB302_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r5, %r4, %r3;
@@ -18441,6 +18439,8 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r11, %rs10;
 ; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r12, %r11, %r10;
@@ -18482,11 +18482,9 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
-; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB302_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
 ; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r5, %r4, %r3;
@@ -18497,6 +18495,8 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r11, %rs10;
 ; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r12, %r11, %r10;
@@ -18542,7 +18542,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB303_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r5, %rs2;
@@ -18556,6 +18555,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r12, %rs8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r13, %rs10;
@@ -18583,7 +18583,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB303_3: // %atomicrmw.start2
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r19, %rs14;
@@ -18597,6 +18596,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r26, %rs20;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r27, %rs22;
@@ -18639,7 +18639,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB303_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r5, %rs2;
@@ -18653,6 +18652,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r12, %rs8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r13, %rs10;
@@ -18680,7 +18680,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB303_3: // %atomicrmw.start2
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r19, %rs14;
@@ -18694,6 +18693,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r26, %rs20;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r27, %rs22;
@@ -18736,7 +18736,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB303_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r5, %rs2;
@@ -18750,6 +18749,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r12, %rs8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs10;
@@ -18777,7 +18777,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB303_3: // %atomicrmw.start2
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r19, %rs14;
@@ -18791,6 +18790,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r26, %rs20;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r27, %rs22;
@@ -18833,7 +18833,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB303_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r5, %rs2;
@@ -18847,6 +18846,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r12, %rs8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs10;
@@ -18874,7 +18874,6 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB303_3: // %atomicrmw.start2
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r19, %rs14;
@@ -18888,6 +18887,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r26, %rs20;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r27, %rs22;
@@ -19286,7 +19286,6 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB306_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -19308,6 +19307,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -19358,7 +19358,6 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB306_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -19380,6 +19379,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -19430,7 +19430,6 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB306_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -19452,6 +19451,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -19502,7 +19502,6 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB306_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -19524,6 +19523,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -19578,7 +19578,6 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB307_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -19600,6 +19599,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -19707,7 +19707,6 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB307_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -19729,6 +19728,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -19836,7 +19836,6 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB307_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -19858,6 +19857,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -19965,7 +19965,6 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB307_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -19987,6 +19986,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -20450,7 +20450,6 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB310_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -20472,6 +20471,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -20522,7 +20522,6 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB310_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -20544,6 +20543,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -20594,7 +20594,6 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB310_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -20616,6 +20615,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -20666,7 +20666,6 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB310_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -20688,6 +20687,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -20742,7 +20742,6 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB311_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -20764,6 +20763,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -20871,7 +20871,6 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB311_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -20893,6 +20892,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -21000,7 +21000,6 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB311_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -21022,6 +21021,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -21129,7 +21129,6 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB311_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -21151,6 +21150,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -21265,12 +21265,12 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB312_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -21311,12 +21311,12 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB312_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -21357,12 +21357,12 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB312_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -21403,12 +21403,12 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB312_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -21450,8 +21450,6 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB313_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -21465,6 +21463,8 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r18, %r17, %r15;
@@ -21500,8 +21500,6 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB313_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -21515,6 +21513,8 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r18, %r17, %r15;
@@ -21550,8 +21550,6 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB313_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -21565,6 +21563,8 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r18, %r17, %r15;
@@ -21600,8 +21600,6 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB313_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -21615,6 +21613,8 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM70-FTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r18, %r17, %r15;
@@ -21654,13 +21654,10 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB314_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -21686,6 +21683,9 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -21742,13 +21742,10 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB314_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -21774,6 +21771,9 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -21830,13 +21830,10 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB314_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -21862,6 +21859,9 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -21918,13 +21918,10 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB314_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -21950,6 +21947,9 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -22010,7 +22010,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB315_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -22040,6 +22039,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -22083,7 +22083,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB315_3: // %atomicrmw.start2
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -22113,6 +22112,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -22171,7 +22171,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB315_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -22201,6 +22200,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -22244,7 +22244,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB315_3: // %atomicrmw.start2
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -22274,6 +22273,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -22332,7 +22332,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB315_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -22362,6 +22361,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -22405,7 +22405,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB315_3: // %atomicrmw.start2
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -22435,6 +22434,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -22493,7 +22493,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB315_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -22523,6 +22522,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -22566,7 +22566,6 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB315_3: // %atomicrmw.start2
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -22596,6 +22595,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -22661,12 +22661,12 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB316_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -22707,12 +22707,12 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB316_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -22753,12 +22753,12 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB316_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -22799,12 +22799,12 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB316_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -22846,8 +22846,6 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB317_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -22861,6 +22859,8 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r18, %r17, %r15;
@@ -22896,8 +22896,6 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB317_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -22911,6 +22909,8 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r18, %r17, %r15;
@@ -22946,8 +22946,6 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB317_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -22961,6 +22959,8 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r18, %r17, %r15;
@@ -22996,8 +22996,6 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB317_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -23011,6 +23009,8 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM70-FTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r18, %r17, %r15;
@@ -23050,13 +23050,10 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB318_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -23082,6 +23079,9 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -23138,13 +23138,10 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB318_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -23170,6 +23167,9 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -23226,13 +23226,10 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB318_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -23258,6 +23255,9 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -23314,13 +23314,10 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB318_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -23346,6 +23343,9 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -23406,7 +23406,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB319_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -23436,6 +23435,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -23479,7 +23479,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB319_3: // %atomicrmw.start2
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -23509,6 +23508,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -23567,7 +23567,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB319_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -23597,6 +23596,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -23640,7 +23640,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB319_3: // %atomicrmw.start2
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -23670,6 +23669,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -23728,7 +23728,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB319_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -23758,6 +23757,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -23801,7 +23801,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB319_3: // %atomicrmw.start2
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -23831,6 +23830,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -23889,7 +23889,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB319_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -23919,6 +23918,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -23962,7 +23962,6 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB319_3: // %atomicrmw.start2
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -23992,6 +23991,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -24057,12 +24057,12 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB320_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -24103,12 +24103,12 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB320_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -24149,12 +24149,12 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB320_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -24195,12 +24195,12 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB320_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -24242,8 +24242,6 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB321_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -24257,6 +24255,8 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r18, %r17, %r15;
@@ -24292,8 +24292,6 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB321_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -24307,6 +24305,8 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r18, %r17, %r15;
@@ -24342,8 +24342,6 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB321_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -24357,6 +24355,8 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r18, %r17, %r15;
@@ -24392,8 +24392,6 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB321_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -24407,6 +24405,8 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM70-FTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r18, %r17, %r15;
@@ -24446,13 +24446,10 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB322_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -24478,6 +24475,9 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -24534,13 +24534,10 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB322_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -24566,6 +24563,9 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -24622,13 +24622,10 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB322_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -24654,7 +24651,10 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r30, %r29, %r27;
@@ -24710,13 +24710,10 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB322_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -24742,6 +24739,9 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -24802,7 +24802,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB323_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -24832,6 +24831,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -24875,7 +24875,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB323_3: // %atomicrmw.start2
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -24905,6 +24904,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -24963,7 +24963,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB323_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -24993,6 +24992,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -25036,7 +25036,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB323_3: // %atomicrmw.start2
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -25066,6 +25065,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -25124,7 +25124,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB323_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -25154,6 +25153,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -25197,7 +25197,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB323_3: // %atomicrmw.start2
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -25227,6 +25226,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -25285,7 +25285,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB323_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -25315,6 +25314,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -25358,7 +25358,6 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB323_3: // %atomicrmw.start2
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -25388,6 +25387,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -25453,12 +25453,12 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB324_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -25499,12 +25499,12 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB324_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -25545,12 +25545,12 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB324_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -25591,12 +25591,12 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB324_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -25638,8 +25638,6 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB325_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -25653,6 +25651,8 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r18, %r17, %r15;
@@ -25688,8 +25688,6 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB325_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -25703,6 +25701,8 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
 ; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r18, %r17, %r15;
@@ -25738,8 +25738,6 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB325_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -25753,6 +25751,8 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r18, %r17, %r15;
@@ -25788,8 +25788,6 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB325_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
@@ -25803,6 +25801,8 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
 ; SM70-FTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
 ; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r18, %r17, %r15;
@@ -25842,13 +25842,10 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB326_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -25874,6 +25871,9 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -25930,13 +25930,10 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB326_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -25962,6 +25959,9 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -26018,13 +26018,10 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB326_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -26050,6 +26047,9 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -26106,13 +26106,10 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB326_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
@@ -26138,6 +26135,9 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
 ; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
@@ -26198,7 +26198,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB327_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -26228,6 +26227,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -26271,7 +26271,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB327_3: // %atomicrmw.start2
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -26301,6 +26300,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -26359,7 +26359,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB327_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -26389,6 +26388,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -26432,7 +26432,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB327_3: // %atomicrmw.start2
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -26462,6 +26461,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -26520,7 +26520,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB327_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -26550,6 +26549,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -26593,7 +26593,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB327_3: // %atomicrmw.start2
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -26623,6 +26622,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -26681,7 +26681,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB327_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
@@ -26711,6 +26710,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
 ; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
@@ -26754,7 +26754,6 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB327_3: // %atomicrmw.start2
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
@@ -26784,6 +26783,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
 ; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
@@ -26849,8 +26849,6 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB328_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -26858,6 +26856,8 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p2, %r8, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -26896,8 +26896,6 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB328_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -26905,6 +26903,8 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p2, %r8, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -26943,8 +26943,6 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB328_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -26952,6 +26950,8 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r8, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -26990,8 +26990,6 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB328_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -26999,6 +26997,8 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r8, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -27038,8 +27038,6 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB329_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -27055,6 +27053,8 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p6, %r10, %r8;
@@ -27088,8 +27088,6 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB329_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -27105,6 +27103,8 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p6, %r10, %r8;
@@ -27138,8 +27138,6 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB329_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -27155,6 +27153,8 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p6, %r10, %r8;
@@ -27188,8 +27188,6 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB329_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -27205,6 +27203,8 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p6, %r10, %r8;
@@ -27242,16 +27242,13 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB330_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p2, %r6, %r4;
@@ -27274,9 +27271,12 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p10, %r15, %r13;
@@ -27330,16 +27330,13 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB330_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p2, %r6, %r4;
@@ -27362,9 +27359,12 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p10, %r15, %r13;
@@ -27418,16 +27418,13 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB330_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r6, %r4;
@@ -27450,9 +27447,12 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p10, %r15, %r13;
@@ -27506,16 +27506,13 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB330_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r6, %r4;
@@ -27538,9 +27535,12 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p10, %r15, %r13;
@@ -27598,7 +27598,6 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB331_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -27628,6 +27627,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -27759,7 +27759,6 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB331_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -27789,6 +27788,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -27920,7 +27920,6 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB331_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -27950,6 +27949,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -28081,7 +28081,6 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB331_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -28111,6 +28110,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -28249,8 +28249,6 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB332_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -28258,6 +28256,8 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p2, %r8, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -28296,8 +28296,6 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB332_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -28305,6 +28303,8 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p2, %r8, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -28343,8 +28343,6 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB332_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -28352,6 +28350,8 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r8, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -28390,8 +28390,6 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB332_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -28399,6 +28397,8 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r8, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -28438,8 +28438,6 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB333_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -28455,6 +28453,8 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p6, %r10, %r8;
@@ -28488,8 +28488,6 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB333_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -28505,6 +28503,8 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p6, %r10, %r8;
@@ -28538,8 +28538,6 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB333_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -28555,6 +28553,8 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p6, %r10, %r8;
@@ -28588,8 +28588,6 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB333_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
@@ -28605,6 +28603,8 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p6, %r10, %r8;
@@ -28642,16 +28642,13 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB334_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p2, %r6, %r4;
@@ -28674,9 +28671,12 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p10, %r15, %r13;
@@ -28730,16 +28730,13 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB334_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p2, %r6, %r4;
@@ -28762,9 +28759,12 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p10, %r15, %r13;
@@ -28818,16 +28818,13 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB334_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r6, %r4;
@@ -28850,9 +28847,12 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p10, %r15, %r13;
@@ -28906,16 +28906,13 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB334_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r6, %r4;
@@ -28938,9 +28935,12 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
 ; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p10, %r15, %r13;
@@ -28998,7 +28998,6 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB335_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -29028,6 +29027,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -29159,7 +29159,6 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB335_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -29189,6 +29188,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -29320,7 +29320,6 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB335_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -29350,6 +29349,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -29481,7 +29481,6 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB335_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
@@ -29511,6 +29510,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
+; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
 ; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
@@ -29726,27 +29726,27 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM70-NEXT:  $L__BB338_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM70-NEXT:    add.s16 %rs3, %rs2, %rs1;
 ; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM70-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM70-NEXT:    add.s16 %rs6, %rs5, %rs4;
 ; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM70-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM70-NEXT:    add.s16 %rs9, %rs8, %rs7;
 ; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM70-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM70-NEXT:    add.s16 %rs12, %rs11, %rs10;
@@ -29952,27 +29952,27 @@ define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM70-NEXT:  $L__BB342_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM70-NEXT:    add.s16 %rs3, %rs2, %rs1;
 ; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM70-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM70-NEXT:    add.s16 %rs6, %rs5, %rs4;
 ; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM70-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM70-NEXT:    add.s16 %rs9, %rs8, %rs7;
 ; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM70-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM70-NEXT:    add.s16 %rs12, %rs11, %rs10;
@@ -30181,27 +30181,27 @@ define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM70-NEXT:  $L__BB346_1: // %atomicrmw.start
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM70-NEXT:    add.s16 %rs3, %rs2, %rs1;
 ; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM70-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM70-NEXT:    add.s16 %rs6, %rs5, %rs4;
 ; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM70-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM70-NEXT:    add.s16 %rs9, %rs8, %rs7;
 ; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM70-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM70-NEXT:    add.s16 %rs12, %rs11, %rs10;
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
index b8d3eb503531ac..c7a4dc37093f1d 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
@@ -508,27 +508,27 @@ define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM90-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM90-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM90-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM90-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM90-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM90-NEXT:  $L__BB18_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM90-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM90-NEXT:    add.s16 %rs3, %rs2, %rs1;
 ; SM90-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM90-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM90-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM90-NEXT:    add.s16 %rs6, %rs5, %rs4;
 ; SM90-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM90-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM90-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM90-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM90-NEXT:    add.s16 %rs9, %rs8, %rs7;
 ; SM90-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM90-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM90-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM90-NEXT:    add.s16 %rs12, %rs11, %rs10;
@@ -1080,27 +1080,27 @@ define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM90-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM90-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM90-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM90-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM90-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM90-NEXT:  $L__BB34_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM90-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM90-NEXT:    sub.s16 %rs3, %rs2, %rs1;
 ; SM90-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM90-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM90-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM90-NEXT:    sub.s16 %rs6, %rs5, %rs4;
 ; SM90-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM90-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM90-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM90-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM90-NEXT:    sub.s16 %rs9, %rs8, %rs7;
 ; SM90-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM90-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM90-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM90-NEXT:    sub.s16 %rs12, %rs11, %rs10;
@@ -1297,7 +1297,6 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM90-NEXT:    mov.b64 {_, %r6}, %rd2;
 ; SM90-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM90-NEXT:  $L__BB38_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
@@ -1305,6 +1304,7 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
 ; SM90-NEXT:    sub.s16 %rs6, %rs3, %rs1;
 ; SM90-NEXT:    mov.b32 %r3, {%rs6, %rs5};
 ; SM90-NEXT:    cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM90-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
 ; SM90-NEXT:    sub.s16 %rs11, %rs10, %rs8;
 ; SM90-NEXT:    sub.s16 %rs12, %rs9, %rs7;
@@ -1349,7 +1349,6 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM90-NEXT:    mov.b64 {_, %r10}, %rd2;
 ; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM90-NEXT:  $L__BB39_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
@@ -1357,6 +1356,7 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
 ; SM90-NEXT:    sub.s16 %rs6, %rs3, %rs1;
 ; SM90-NEXT:    mov.b32 %r5, {%rs6, %rs5};
 ; SM90-NEXT:    cvt.u64.u32 %rd4, %r5;
+; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM90-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
 ; SM90-NEXT:    sub.s16 %rs11, %rs10, %rs8;
 ; SM90-NEXT:    sub.s16 %rs12, %rs9, %rs7;
@@ -3828,7 +3828,6 @@ define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
 ; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
 ; SM90-NEXT:    cvt.s16.s8 %rs7, %rs6;
 ; SM90-NEXT:    cvt.s16.s8 %rs8, %rs5;
-; SM90-NEXT:    mov.b32 %r11, {%rs8, %rs7};
 ; SM90-NEXT:  $L__BB113_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    shr.u32 %r8, %r18, %r1;
@@ -3836,6 +3835,7 @@ define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
 ; SM90-NEXT:    bfe.s32 %r9, %r8, 8, 8;
 ; SM90-NEXT:    cvt.s8.s32 %rs4, %r9;
 ; SM90-NEXT:    mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT:    mov.b32 %r11, {%rs8, %rs7};
 ; SM90-NEXT:    max.s16x2 %r12, %r10, %r11;
 ; SM90-NEXT:    mov.b32 {%rs9, %rs10}, %r12;
 ; SM90-NEXT:    shl.b16 %rs11, %rs10, 8;
@@ -3871,11 +3871,11 @@ define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM90-NEXT:    fence.release.cta;
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
 ; SM90-NEXT:    prmt.b32 %r3, %r2, 0, 0x8880U;
-; SM90-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
 ; SM90-NEXT:  $L__BB114_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    prmt.b32 %r4, %r26, 0, 0x8880U;
 ; SM90-NEXT:    setp.gt.s32 %p1, %r4, %r3;
+; SM90-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
 ; SM90-NEXT:    prmt.b32 %r6, %r26, 0, 0x9991U;
 ; SM90-NEXT:    setp.gt.s32 %p2, %r6, %r5;
 ; SM90-NEXT:    prmt.b32 %r7, %r2, 0, 0xaaa2U;
@@ -4435,7 +4435,6 @@ define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
 ; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
 ; SM90-NEXT:    cvt.s16.s8 %rs7, %rs6;
 ; SM90-NEXT:    cvt.s16.s8 %rs8, %rs5;
-; SM90-NEXT:    mov.b32 %r11, {%rs8, %rs7};
 ; SM90-NEXT:  $L__BB129_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    shr.u32 %r8, %r18, %r1;
@@ -4443,6 +4442,7 @@ define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
 ; SM90-NEXT:    bfe.s32 %r9, %r8, 8, 8;
 ; SM90-NEXT:    cvt.s8.s32 %rs4, %r9;
 ; SM90-NEXT:    mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT:    mov.b32 %r11, {%rs8, %rs7};
 ; SM90-NEXT:    min.s16x2 %r12, %r10, %r11;
 ; SM90-NEXT:    mov.b32 {%rs9, %rs10}, %r12;
 ; SM90-NEXT:    shl.b16 %rs11, %rs10, 8;
@@ -4478,11 +4478,11 @@ define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM90-NEXT:    fence.release.cta;
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
 ; SM90-NEXT:    prmt.b32 %r3, %r2, 0, 0x8880U;
-; SM90-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
 ; SM90-NEXT:  $L__BB130_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    prmt.b32 %r4, %r26, 0, 0x8880U;
 ; SM90-NEXT:    setp.le.s32 %p1, %r4, %r3;
+; SM90-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
 ; SM90-NEXT:    prmt.b32 %r6, %r26, 0, 0x9991U;
 ; SM90-NEXT:    setp.le.s32 %p2, %r6, %r5;
 ; SM90-NEXT:    prmt.b32 %r7, %r2, 0, 0xaaa2U;
@@ -5080,11 +5080,11 @@ define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
 ; SM90-NEXT:    fence.release.cta;
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM90-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM90-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM90-NEXT:  $L__BB146_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    prmt.b32 %r4, %r18, 0, 0x7770U;
 ; SM90-NEXT:    setp.gt.u32 %p1, %r4, %r3;
+; SM90-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM90-NEXT:    prmt.b32 %r6, %r18, 0, 0x7771U;
 ; SM90-NEXT:    setp.gt.u32 %p2, %r6, %r5;
 ; SM90-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -5658,11 +5658,11 @@ define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
 ; SM90-NEXT:    fence.release.cta;
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM90-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM90-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM90-NEXT:  $L__BB162_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    prmt.b32 %r4, %r18, 0, 0x7770U;
 ; SM90-NEXT:    setp.le.u32 %p1, %r4, %r3;
+; SM90-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM90-NEXT:    prmt.b32 %r6, %r18, 0, 0x7771U;
 ; SM90-NEXT:    setp.le.u32 %p2, %r6, %r5;
 ; SM90-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -6198,7 +6198,6 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM90-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM90-NEXT:    not.b32 %r2, %r7;
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM90-NEXT:    mov.b32 {%rs6, %rs7}, %r4;
 ; SM90-NEXT:  $L__BB177_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    shr.u32 %r8, %r17, %r1;
@@ -6208,6 +6207,7 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM90-NEXT:    and.b16 %rs5, %rs3, 255;
 ; SM90-NEXT:    mov.b32 %r10, 65537;
 ; SM90-NEXT:    add.s16x2 %r11, %r9, %r10;
+; SM90-NEXT:    mov.b32 {%rs6, %rs7}, %r4;
 ; SM90-NEXT:    setp.ge.u16 %p1, %rs4, %rs7;
 ; SM90-NEXT:    setp.ge.u16 %p2, %rs5, %rs6;
 ; SM90-NEXT:    mov.b32 {%rs8, %rs9}, %r11;
@@ -6246,8 +6246,6 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM90-NEXT:    ld.param::func.b64 %rd1, [uinc_wrap_acq_rel_v4i8_global_cta_param_0];
 ; SM90-NEXT:    fence.release.cta;
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r22, [%rd1];
-; SM90-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
-; SM90-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
 ; SM90-NEXT:  $L__BB178_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    prmt.b32 %r3, %r22, 0, 0x7770U;
@@ -6262,7 +6260,9 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM90-NEXT:    prmt.b32 %r6, %r22, 0, 0x7773U;
 ; SM90-NEXT:    cvt.u16.u32 %rs7, %r6;
 ; SM90-NEXT:    add.s16 %rs8, %rs7, 1;
+; SM90-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
 ; SM90-NEXT:    setp.ge.u32 %p1, %r3, %r7;
+; SM90-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
 ; SM90-NEXT:    setp.ge.u32 %p2, %r4, %r8;
 ; SM90-NEXT:    prmt.b32 %r9, %r2, 0, 0x7772U;
 ; SM90-NEXT:    setp.ge.u32 %p3, %r5, %r9;
@@ -6487,7 +6487,6 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM90-NEXT:    mov.b64 {_, %r9}, %rd2;
 ; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
 ; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; SM90-NEXT:  $L__BB182_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    cvt.u64.u32 %rd3, %r8;
@@ -6500,6 +6499,7 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
 ; SM90-NEXT:    setp.ge.u16 %p1, %rs3, %rs1;
 ; SM90-NEXT:    setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r8;
 ; SM90-NEXT:    setp.ge.u16 %p3, %rs7, %rs5;
 ; SM90-NEXT:    setp.ge.u16 %p4, %rs8, %rs6;
@@ -6548,7 +6548,6 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM90-NEXT:    mov.b64 {_, %r15}, %rd2;
 ; SM90-NEXT:    cvt.u32.u64 %r14, %rd2;
 ; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
-; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
 ; SM90-NEXT:  $L__BB183_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    cvt.u64.u32 %rd4, %r14;
@@ -6567,6 +6566,7 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
 ; SM90-NEXT:    setp.ge.u16 %p1, %rs3, %rs1;
 ; SM90-NEXT:    setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
 ; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r16;
 ; SM90-NEXT:    setp.ge.u16 %p3, %rs7, %rs5;
 ; SM90-NEXT:    setp.ge.u16 %p4, %rs8, %rs6;
@@ -7043,7 +7043,6 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM90-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM90-NEXT:    not.b32 %r2, %r7;
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-NEXT:    mov.b32 {%rs8, %rs9}, %r4;
 ; SM90-NEXT:  $L__BB193_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    shr.u32 %r8, %r14, %r1;
@@ -7054,6 +7053,7 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM90-NEXT:    add.s16 %rs7, %rs3, -1;
 ; SM90-NEXT:    setp.eq.b16 %p1, %rs5, 0;
 ; SM90-NEXT:    setp.eq.b16 %p2, %rs4, 0;
+; SM90-NEXT:    mov.b32 {%rs8, %rs9}, %r4;
 ; SM90-NEXT:    setp.gt.u16 %p3, %rs5, %rs9;
 ; SM90-NEXT:    setp.gt.u16 %p4, %rs4, %rs8;
 ; SM90-NEXT:    selp.b16 %rs10, %rs8, %rs7, %p4;
@@ -7093,8 +7093,6 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM90-NEXT:    ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v4i8_global_cta_param_0];
 ; SM90-NEXT:    fence.release.cta;
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
-; SM90-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
-; SM90-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
 ; SM90-NEXT:  $L__BB194_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    prmt.b32 %r3, %r26, 0, 0x7770U;
@@ -7113,7 +7111,9 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM90-NEXT:    setp.eq.b32 %p2, %r4, 0;
 ; SM90-NEXT:    setp.eq.b32 %p3, %r5, 0;
 ; SM90-NEXT:    setp.eq.b32 %p4, %r6, 0;
+; SM90-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
 ; SM90-NEXT:    setp.gt.u32 %p5, %r3, %r7;
+; SM90-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
 ; SM90-NEXT:    setp.gt.u32 %p6, %r4, %r8;
 ; SM90-NEXT:    prmt.b32 %r9, %r2, 0, 0x7772U;
 ; SM90-NEXT:    setp.gt.u32 %p7, %r5, %r9;
@@ -7328,7 +7328,6 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
 ; SM90-NEXT:    ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v2i16_global_cta_param_0];
 ; SM90-NEXT:    fence.release.cta;
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM90-NEXT:  $L__BB197_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
@@ -7336,6 +7335,7 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
 ; SM90-NEXT:    add.s16 %rs4, %rs2, -1;
 ; SM90-NEXT:    setp.eq.b16 %p1, %rs1, 0;
 ; SM90-NEXT:    setp.eq.b16 %p2, %rs2, 0;
+; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
 ; SM90-NEXT:    setp.gt.u16 %p3, %rs1, %rs5;
 ; SM90-NEXT:    setp.gt.u16 %p4, %rs2, %rs6;
 ; SM90-NEXT:    selp.b16 %rs7, %rs6, %rs4, %p4;
@@ -7370,8 +7370,6 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM90-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
 ; SM90-NEXT:    mov.b64 {_, %r6}, %rd2;
 ; SM90-NEXT:    cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
-; SM90-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM90-NEXT:  $L__BB198_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -7388,8 +7386,10 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM90-NEXT:    setp.eq.b16 %p2, %rs2, 0;
 ; SM90-NEXT:    setp.eq.b16 %p3, %rs5, 0;
 ; SM90-NEXT:    setp.eq.b16 %p4, %rs6, 0;
+; SM90-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM90-NEXT:    setp.gt.u16 %p5, %rs1, %rs9;
 ; SM90-NEXT:    setp.gt.u16 %p6, %rs2, %rs10;
+; SM90-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
 ; SM90-NEXT:    setp.gt.u16 %p7, %rs5, %rs11;
 ; SM90-NEXT:    setp.gt.u16 %p8, %rs6, %rs12;
 ; SM90-NEXT:    selp.b16 %rs13, %rs12, %rs8, %p8;
@@ -7438,8 +7438,6 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM90-NEXT:    cvt.u32.u64 %r11, %rd3;
 ; SM90-NEXT:    mov.b64 {_, %r10}, %rd2;
 ; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    mov.b32 {%rs17, %rs18}, %r4;
-; SM90-NEXT:    mov.b32 {%rs19, %rs20}, %r3;
 ; SM90-NEXT:  $L__BB199_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    cvt.u64.u32 %rd4, %r9;
@@ -7470,8 +7468,10 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM90-NEXT:    setp.eq.b16 %p6, %rs10, 0;
 ; SM90-NEXT:    setp.eq.b16 %p7, %rs13, 0;
 ; SM90-NEXT:    setp.eq.b16 %p8, %rs14, 0;
+; SM90-NEXT:    mov.b32 {%rs17, %rs18}, %r4;
 ; SM90-NEXT:    setp.gt.u16 %p9, %rs1, %rs17;
 ; SM90-NEXT:    setp.gt.u16 %p10, %rs2, %rs18;
+; SM90-NEXT:    mov.b32 {%rs19, %rs20}, %r3;
 ; SM90-NEXT:    setp.gt.u16 %p11, %rs5, %rs19;
 ; SM90-NEXT:    setp.gt.u16 %p12, %rs6, %rs20;
 ; SM90-NEXT:    mov.b32 {%rs21, %rs22}, %r2;
@@ -7977,13 +7977,13 @@ define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
 ; SM90-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM90-NEXT:    not.b32 %r2, %r7;
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-NEXT:    mov.b32 {%rs6, %rs7}, %r4;
 ; SM90-NEXT:  $L__BB209_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    shr.u32 %r8, %r14, %r1;
 ; SM90-NEXT:    cvt.u16.u32 %rs3, %r8;
 ; SM90-NEXT:    and.b16 %rs4, %rs3, 255;
 ; SM90-NEXT:    shr.u16 %rs5, %rs3, 8;
+; SM90-NEXT:    mov.b32 {%rs6, %rs7}, %r4;
 ; SM90-NEXT:    setp.ge.u16 %p1, %rs5, %rs7;
 ; SM90-NEXT:    setp.ge.u16 %p2, %rs4, %rs6;
 ; SM90-NEXT:    sub.s16 %rs8, %rs5, %rs7;
@@ -8024,13 +8024,12 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM90-NEXT:    fence.release.cta;
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r22, [%rd1];
 ; SM90-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM90-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM90-NEXT:    cvt.u16.u32 %rs3, %r5;
 ; SM90-NEXT:    cvt.u16.u32 %rs4, %r3;
 ; SM90-NEXT:  $L__BB210_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    prmt.b32 %r4, %r22, 0, 0x7770U;
 ; SM90-NEXT:    setp.ge.u32 %p1, %r4, %r3;
+; SM90-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
 ; SM90-NEXT:    prmt.b32 %r6, %r22, 0, 0x7771U;
 ; SM90-NEXT:    setp.ge.u32 %p2, %r6, %r5;
 ; SM90-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -8041,6 +8040,7 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM90-NEXT:    setp.ge.u32 %p4, %r10, %r9;
 ; SM90-NEXT:    cvt.u16.u32 %rs1, %r9;
 ; SM90-NEXT:    cvt.u16.u32 %rs2, %r7;
+; SM90-NEXT:    cvt.u16.u32 %rs3, %r5;
 ; SM90-NEXT:    cvt.u16.u32 %rs5, %r4;
 ; SM90-NEXT:    sub.s16 %rs6, %rs5, %rs4;
 ; SM90-NEXT:    cvt.u16.u32 %rs7, %r6;
@@ -8275,7 +8275,6 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM90-NEXT:    mov.b64 {_, %r6}, %rd2;
 ; SM90-NEXT:    cvt.u32.u64 %r5, %rd2;
 ; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; SM90-NEXT:  $L__BB214_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    cvt.u64.u32 %rd3, %r5;
@@ -8285,6 +8284,7 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
 ; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r6;
 ; SM90-NEXT:    setp.ge.u16 %p1, %rs3, %rs1;
 ; SM90-NEXT:    setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r5;
 ; SM90-NEXT:    setp.ge.u16 %p3, %rs7, %rs5;
 ; SM90-NEXT:    setp.ge.u16 %p4, %rs8, %rs6;
@@ -8335,7 +8335,6 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM90-NEXT:    mov.b64 {_, %r10}, %rd2;
 ; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
 ; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
-; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
 ; SM90-NEXT:  $L__BB215_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    cvt.u64.u32 %rd4, %r9;
@@ -8349,6 +8348,7 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
 ; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
 ; SM90-NEXT:    setp.ge.u16 %p1, %rs3, %rs1;
 ; SM90-NEXT:    setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
 ; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r11;
 ; SM90-NEXT:    setp.ge.u16 %p3, %rs7, %rs5;
 ; SM90-NEXT:    setp.ge.u16 %p4, %rs8, %rs6;
@@ -9011,7 +9011,6 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
 ; SM90-NEXT:    shl.b32 %r7, 65535, %r1;
 ; SM90-NEXT:    not.b32 %r2, %r7;
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM90-NEXT:  $L__BB225_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    shr.u32 %r8, %r17, %r1;
@@ -9021,6 +9020,7 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
 ; SM90-NEXT:    and.b32 %r10, %r9, 16711935;
 ; SM90-NEXT:    max.u16x2 %r11, %r10, %r4;
 ; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
 ; SM90-NEXT:    sub.s16 %rs9, %rs5, %rs7;
 ; SM90-NEXT:    sub.s16 %rs10, %rs6, %rs8;
 ; SM90-NEXT:    shl.b16 %rs11, %rs10, 8;
@@ -9057,12 +9057,6 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM90-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM90-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM90-NEXT:    cvt.u16.u32 %rs5, %r6;
-; SM90-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM90-NEXT:    cvt.u16.u32 %rs9, %r10;
-; SM90-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM90-NEXT:    cvt.u16.u32 %rs13, %r13;
 ; SM90-NEXT:  $L__BB226_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
@@ -9070,17 +9064,23 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
 ; SM90-NEXT:    max.u16 %rs3, %rs2, %rs1;
 ; SM90-NEXT:    sub.s16 %rs4, %rs3, %rs1;
 ; SM90-NEXT:    cvt.u32.u16 %r5, %rs4;
+; SM90-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT:    cvt.u16.u32 %rs5, %r6;
 ; SM90-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM90-NEXT:    cvt.u16.u32 %rs6, %r7;
 ; SM90-NEXT:    max.u16 %rs7, %rs6, %rs5;
 ; SM90-NEXT:    sub.s16 %rs8, %rs7, %rs5;
 ; SM90-NEXT:    cvt.u32.u16 %r8, %rs8;
 ; SM90-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT:    cvt.u16.u32 %rs9, %r10;
 ; SM90-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM90-NEXT:    cvt.u16.u32 %rs10, %r11;
 ; SM90-NEXT:    max.u16 %rs11, %rs10, %rs9;
 ; SM90-NEXT:    sub.s16 %rs12, %rs11, %rs9;
 ; SM90-NEXT:    cvt.u32.u16 %r12, %rs12;
+; SM90-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT:    cvt.u16.u32 %rs13, %r13;
 ; SM90-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM90-NEXT:    cvt.u16.u32 %rs14, %r14;
 ; SM90-NEXT:    max.u16 %rs15, %rs14, %rs13;
@@ -9291,7 +9291,6 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM90-NEXT:    mov.b64 {_, %r8}, %rd2;
 ; SM90-NEXT:    cvt.u32.u64 %r7, %rd2;
 ; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM90-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM90-NEXT:  $L__BB230_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    max.u16x2 %r3, %r7, %r1;
@@ -9302,6 +9301,7 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
 ; SM90-NEXT:    cvt.u64.u32 %rd3, %r4;
 ; SM90-NEXT:    max.u16x2 %r5, %r8, %r2;
 ; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r5;
+; SM90-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM90-NEXT:    sub.s16 %rs11, %rs8, %rs10;
 ; SM90-NEXT:    sub.s16 %rs12, %rs7, %rs9;
 ; SM90-NEXT:    mov.b32 %r6, {%rs12, %rs11};
@@ -9345,7 +9345,6 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM90-NEXT:    mov.b64 {_, %r14}, %rd2;
 ; SM90-NEXT:    cvt.u32.u64 %r13, %rd2;
 ; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM90-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM90-NEXT:  $L__BB231_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    max.u16x2 %r5, %r13, %r1;
@@ -9356,6 +9355,7 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
 ; SM90-NEXT:    cvt.u64.u32 %rd4, %r6;
 ; SM90-NEXT:    max.u16x2 %r7, %r14, %r2;
 ; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r7;
+; SM90-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
 ; SM90-NEXT:    sub.s16 %rs11, %rs8, %rs10;
 ; SM90-NEXT:    sub.s16 %rs12, %rs7, %rs9;
 ; SM90-NEXT:    mov.b32 %r8, {%rs12, %rs11};
@@ -20196,27 +20196,27 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM90-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM90-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM90-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM90-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM90-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM90-NEXT:  $L__BB338_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM90-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM90-NEXT:    add.s16 %rs3, %rs2, %rs1;
 ; SM90-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM90-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM90-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM90-NEXT:    add.s16 %rs6, %rs5, %rs4;
 ; SM90-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM90-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM90-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM90-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM90-NEXT:    add.s16 %rs9, %rs8, %rs7;
 ; SM90-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM90-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM90-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM90-NEXT:    add.s16 %rs12, %rs11, %rs10;
@@ -20422,27 +20422,27 @@ define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM90-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM90-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM90-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM90-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM90-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM90-NEXT:  $L__BB342_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM90-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM90-NEXT:    add.s16 %rs3, %rs2, %rs1;
 ; SM90-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM90-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM90-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM90-NEXT:    add.s16 %rs6, %rs5, %rs4;
 ; SM90-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM90-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM90-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM90-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM90-NEXT:    add.s16 %rs9, %rs8, %rs7;
 ; SM90-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM90-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM90-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM90-NEXT:    add.s16 %rs12, %rs11, %rs10;
@@ -20651,27 +20651,27 @@ define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
 ; SM90-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
 ; SM90-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
 ; SM90-NEXT:    cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM90-NEXT:    cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM90-NEXT:    cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM90-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM90-NEXT:  $L__BB346_1: // %atomicrmw.start
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM90-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
 ; SM90-NEXT:    cvt.u16.u32 %rs2, %r4;
 ; SM90-NEXT:    add.s16 %rs3, %rs2, %rs1;
 ; SM90-NEXT:    cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT:    cvt.u16.u32 %rs4, %r6;
 ; SM90-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
 ; SM90-NEXT:    cvt.u16.u32 %rs5, %r7;
 ; SM90-NEXT:    add.s16 %rs6, %rs5, %rs4;
 ; SM90-NEXT:    cvt.u32.u16 %r8, %rs6;
 ; SM90-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT:    cvt.u16.u32 %rs7, %r10;
 ; SM90-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
 ; SM90-NEXT:    cvt.u16.u32 %rs8, %r11;
 ; SM90-NEXT:    add.s16 %rs9, %rs8, %rs7;
 ; SM90-NEXT:    cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT:    cvt.u16.u32 %rs10, %r13;
 ; SM90-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
 ; SM90-NEXT:    cvt.u16.u32 %rs11, %r14;
 ; SM90-NEXT:    add.s16 %rs12, %rs11, %rs10;
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
index f614491de0368d..6dfaab754e8db3 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
@@ -2811,12 +2811,12 @@ define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB74_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    min.f32 %r10, %r8, %r9;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -2851,12 +2851,12 @@ define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB74_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-ALLOW-NEXT:    min.f32 %r10, %r8, %r9;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -2891,12 +2891,12 @@ define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB74_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r10, %r8, %r9;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -2931,12 +2931,12 @@ define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB74_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-ALLOW-NEXT:    min.ftz.f32 %r10, %r8, %r9;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -2975,12 +2975,12 @@ define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB75_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    max.f32 %r10, %r8, %r9;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -3015,12 +3015,12 @@ define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB75_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM60-NOFTZ-ALLOW-NEXT:    max.f32 %r10, %r8, %r9;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -3055,12 +3055,12 @@ define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB75_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r10, %r8, %r9;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -3095,12 +3095,12 @@ define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB75_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM60-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM60-FTZ-ALLOW-NEXT:    max.ftz.f32 %r10, %r8, %r9;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -3499,12 +3499,12 @@ define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB78_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -3544,12 +3544,12 @@ define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB78_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -3589,12 +3589,12 @@ define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB78_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -3634,12 +3634,12 @@ define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB78_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -3683,12 +3683,12 @@ define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB79_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -3728,12 +3728,12 @@ define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB79_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -3773,12 +3773,12 @@ define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB79_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -3818,12 +3818,12 @@ define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB79_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -3867,12 +3867,12 @@ define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB80_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    min.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -3912,12 +3912,12 @@ define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB80_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    min.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -3957,12 +3957,12 @@ define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB80_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -4002,12 +4002,12 @@ define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB80_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    min.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -4051,12 +4051,12 @@ define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB81_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    max.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -4096,12 +4096,12 @@ define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB81_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    max.f32 %r11, %r8, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-NOFTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -4141,12 +4141,12 @@ define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB81_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -4186,12 +4186,12 @@ define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r22, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB81_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r22, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    max.ftz.f32 %r11, %r8, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    bfe.u32 %r12, %r11, 16, 1;
 ; SM60-FTZ-ALLOW-NEXT:    add.s32 %r13, %r12, %r11;
@@ -4235,8 +4235,6 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB82_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4244,6 +4242,8 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p2, %r8, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -4282,8 +4282,6 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB82_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4291,6 +4289,8 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p2, %r8, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -4329,8 +4329,6 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB82_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4338,6 +4336,8 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r8, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -4376,8 +4376,6 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB82_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4385,6 +4383,8 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r8, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -4427,8 +4427,6 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:  $L__BB83_1: // %atomicrmw.start
 ; SM60-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4436,6 +4434,8 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p2, %r8, %r10;
 ; SM60-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -4474,8 +4474,6 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-NOFTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:  $L__BB83_1: // %atomicrmw.start
 ; SM60-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4483,6 +4481,8 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p2, %r8, %r10;
 ; SM60-NOFTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -4521,8 +4521,6 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-DISALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:  $L__BB83_1: // %atomicrmw.start
 ; SM60-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4530,6 +4528,8 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r8, %r10;
 ; SM60-FTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -4568,8 +4568,6 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM60-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM60-FTZ-ALLOW-NEXT:    ld.volatile.global.b32 %r16, [%rd1];
-; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:  $L__BB83_1: // %atomicrmw.start
 ; SM60-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM60-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4577,6 +4575,8 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM60-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM60-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM60-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM60-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r8, %r10;
 ; SM60-FTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM60-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
index 588adf128d3719..87fda1ef854402 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
@@ -2604,12 +2604,12 @@ define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB74_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r10, %r8, %r9;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -2644,12 +2644,12 @@ define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB74_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r10, %r8, %r9;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -2684,12 +2684,12 @@ define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB74_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r10, %r8, %r9;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -2724,12 +2724,12 @@ define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB74_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r10, %r8, %r9;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -2768,12 +2768,12 @@ define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB75_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r10, %r8, %r9;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -2808,12 +2808,12 @@ define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB75_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
 ; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r10, %r8, %r9;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -2848,12 +2848,12 @@ define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB75_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r10, %r8, %r9;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -2888,12 +2888,12 @@ define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB75_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
+; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
 ; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r10, %r8, %r9;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
@@ -3292,12 +3292,12 @@ define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB78_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3338,12 +3338,12 @@ define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB78_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3384,12 +3384,12 @@ define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB78_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3430,12 +3430,12 @@ define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB78_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3480,12 +3480,12 @@ define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB79_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3526,12 +3526,12 @@ define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB79_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3572,12 +3572,12 @@ define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB79_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3618,12 +3618,12 @@ define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB79_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3668,12 +3668,12 @@ define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB80_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3714,12 +3714,12 @@ define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB80_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3760,12 +3760,12 @@ define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB80_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3806,12 +3806,12 @@ define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB80_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3856,12 +3856,12 @@ define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB81_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3902,12 +3902,12 @@ define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB81_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r11, %r8, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3948,12 +3948,12 @@ define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB81_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -3994,12 +3994,12 @@ define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB81_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r11, %r8, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
 ; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
@@ -4044,8 +4044,6 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB82_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4053,6 +4051,8 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p2, %r8, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -4091,8 +4091,6 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB82_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4100,6 +4098,8 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p2, %r8, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -4138,8 +4138,6 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB82_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4147,6 +4145,8 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r8, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -4185,8 +4185,6 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB82_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4194,6 +4192,8 @@ define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r8, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -4236,8 +4236,6 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB83_1: // %atomicrmw.start
 ; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4245,6 +4243,8 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p2, %r8, %r10;
 ; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -4283,8 +4283,6 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:  $L__BB83_1: // %atomicrmw.start
 ; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4292,6 +4290,8 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p2, %r8, %r10;
 ; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
@@ -4330,8 +4330,6 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:  $L__BB83_1: // %atomicrmw.start
 ; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4339,6 +4337,8 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r8, %r10;
 ; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
@@ -4377,8 +4377,6 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
 ; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
 ; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
-; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:  $L__BB83_1: // %atomicrmw.start
 ; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
 ; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
@@ -4386,6 +4384,8 @@ define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat
 ; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
 ; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
 ; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r8, %r10;
 ; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
 ; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
diff --git a/llvm/test/CodeGen/PowerPC/pr52894-32bit.ll b/llvm/test/CodeGen/PowerPC/pr52894-32bit.ll
index ed6de3ea482f3b..72f44583e5458e 100644
--- a/llvm/test/CodeGen/PowerPC/pr52894-32bit.ll
+++ b/llvm/test/CodeGen/PowerPC/pr52894-32bit.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -verify-machineinstrs -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
-; RUN:   -mcpu=pwr8 -mtriple=powerpcle < %s | FileCheck %s
+; RUN:   -mcpu=pwr8 -mtriple=powerpcle < %s | FileCheck %s --check-prefixes=CHECK,LE
 ; RUN: llc -verify-machineinstrs -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
-; RUN:   -mcpu=pwr8 -mtriple=powerpc < %s | FileCheck %s
+; RUN:   -mcpu=pwr8 -mtriple=powerpc < %s | FileCheck %s --check-prefixes=CHECK,BE
 %struct.d = type { [131072 x i32] }
 
 @a = dso_local local_unnamed_addr global [4096 x i32] zeroinitializer, align 4
@@ -9,12 +9,19 @@
 ; Function Attrs: mustprogress uwtable
 define dso_local void @_Z1g1dILi17EE(ptr nocapture noundef readnone byval(%struct.d) align 4 %0) local_unnamed_addr #0 {
 ; CHECK-LABEL: _Z1g1dILi17EE:
-; CHECK-NOT:    mtfprwz f0, r4
-; CHECK-NOT:    stwx r3, r1, r4
-; CHECK-NOT:    mffprwz r4, f0
-; CHECK-NOT:    mtfprwz f0, r4
-; CHECK-NOT:    lwzx r3, r1, r4
-; CHECK-NOT:    mffprwz r4, f0
+; Check the GPR save in a VSR when a large stack offset needs a temporary GPR.
+; LE:    mtfprwz f0, r4
+; LE:    stwx [[REG:r[0-9]+]], r1, r4
+; LE:    mffprwz r4, f0
+; LE:    mtfprwz f0, r4
+; LE:    lwzx [[REG]], r1, r4
+; LE:    mffprwz r4, f0
+; BE-NOT:    mtfprwz f0, r4
+; BE-NOT:    stwx r3, r1, r4
+; BE-NOT:    mffprwz r4, f0
+; BE-NOT:    mtfprwz f0, r4
+; BE-NOT:    lwzx r3, r1, r4
+; BE-NOT:    mffprwz r4, f0
 entry:
   %c = alloca %struct.d, align 4
   call void @llvm.lifetime.start.p0(i64 524288, ptr nonnull %c) #3
diff --git a/llvm/test/CodeGen/RISCV/rvv/pr165232.ll b/llvm/test/CodeGen/RISCV/rvv/pr165232.ll
index 3a12ea323c7cf5..94e02266646da8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/pr165232.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/pr165232.ll
@@ -10,14 +10,12 @@ define i1 @main(ptr %var_117, ptr %arrayinit.element3045, ptr %arrayinit.element
 ; CHECK-NEXT:    addi sp, sp, -16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    csrr t0, vlenb
-; CHECK-NEXT:    slli t1, t0, 4
-; CHECK-NEXT:    add t0, t1, t0
+; CHECK-NEXT:    slli t0, t0, 4
 ; CHECK-NEXT:    sub sp, sp, t0
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x11, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 17 * vlenb
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
 ; CHECK-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli t0, a0, 3
-; CHECK-NEXT:    add a0, t0, a0
+; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs4r.v v12, (a0) # vscale x 32-byte Folded Spill
@@ -26,155 +24,141 @@ define i1 @main(ptr %var_117, ptr %arrayinit.element3045, ptr %arrayinit.element
 ; CHECK-NEXT:    add a0, a0, t0
 ; CHECK-NEXT:    vs4r.v v16, (a0) # vscale x 32-byte Folded Spill
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli t0, a0, 2
-; CHECK-NEXT:    add a0, t0, a0
+; CHECK-NEXT:    slli a0, a0, 2
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
 ; CHECK-NEXT:    vsetvli t0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v10, 0
+; CHECK-NEXT:    # kill: def $v11 killed $v10 killed $vtype
+; CHECK-NEXT:    vmv.v.i v12, 0
 ; CHECK-NEXT:    vmv.v.i v13, 0
-; CHECK-NEXT:    # kill: def $v14 killed $v13 killed $vtype
+; CHECK-NEXT:    vmv.v.i v14, 0
 ; CHECK-NEXT:    vmv.v.i v15, 0
 ; CHECK-NEXT:    vmv.v.i v16, 0
-; CHECK-NEXT:    vmv.v.i v17, 0
-; CHECK-NEXT:    vsetvli t0, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vmv.v.i v20, 0
-; CHECK-NEXT:    vmv1r.v v18, v13
-; CHECK-NEXT:    vmv.v.i v22, 0
-; CHECK-NEXT:    vmv1r.v v19, v13
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli t0, a0, 4
-; CHECK-NEXT:    add a0, t0, a0
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    ld t0, 56(a0)
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli t1, a0, 4
-; CHECK-NEXT:    add a0, t1, a0
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    ld t1, 48(a0)
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli t2, a0, 4
-; CHECK-NEXT:    add a0, t2, a0
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    ld t2, 40(a0)
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli t3, a0, 4
-; CHECK-NEXT:    add a0, t3, a0
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    ld t3, 32(a0)
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli t4, a0, 4
-; CHECK-NEXT:    add a0, t4, a0
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    ld t4, 16(a0)
-; CHECK-NEXT:    vmv.v.i v24, 0
-; CHECK-NEXT:    vmv1r.v v9, v13
-; CHECK-NEXT:    vmv1r.v v10, v13
+; CHECK-NEXT:    vmv1r.v v21, v10
 ; CHECK-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
-; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vmv.v.i v20, 0
+; CHECK-NEXT:    vmv1r.v v22, v10
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli t5, a0, 4
-; CHECK-NEXT:    add a0, t5, a0
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    ld t5, 24(a0)
-; CHECK-NEXT:    vmv1r.v v11, v13
-; CHECK-NEXT:    vmv1r.v v12, v13
-; CHECK-NEXT:    csrr t6, vlenb
-; CHECK-NEXT:    add t6, sp, t6
-; CHECK-NEXT:    addi t6, t6, 16
-; CHECK-NEXT:    vs1r.v v9, (t6) # vscale x 8-byte Folded Spill
+; CHECK-NEXT:    vmv1r.v v23, v10
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v18, 0
+; CHECK-NEXT:    vmv1r.v v24, v10
+; CHECK-NEXT:    addi t6, sp, 16
+; CHECK-NEXT:    vs1r.v v21, (t6) # vscale x 8-byte Folded Spill
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    add t6, t6, a0
-; CHECK-NEXT:    vs2r.v v10, (t6) # vscale x 16-byte Folded Spill
+; CHECK-NEXT:    vs2r.v v22, (t6) # vscale x 16-byte Folded Spill
 ; CHECK-NEXT:    slli a0, a0, 1
 ; CHECK-NEXT:    add t6, t6, a0
 ; CHECK-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
-; CHECK-NEXT:    vs1r.v v12, (t6) # vscale x 8-byte Folded Spill
-; CHECK-NEXT:    vmclr.m v9
-; CHECK-NEXT:    addi t6, sp, 16
-; CHECK-NEXT:    vs1r.v v9, (t6) # vscale x 8-byte Folded Spill
-; CHECK-NEXT:    vmv2r.v v26, v20
+; CHECK-NEXT:    vs1r.v v24, (t6) # vscale x 8-byte Folded Spill
 ; CHECK-NEXT:    li t6, 1023
 ; CHECK-NEXT:    slli t6, t6, 52
 ; CHECK-NEXT:    sd zero, 0(a0)
 ; CHECK-NEXT:    sd t6, 0(t5)
 ; CHECK-NEXT:  .LBB0_1: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vmv.v.i v10, 0
-; CHECK-NEXT:    vmv.v.i v30, 0
-; CHECK-NEXT:    vmv1r.v v12, v8
+; CHECK-NEXT:    vmv2r.v v22, v18
+; CHECK-NEXT:    vmv1r.v v21, v20
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, tu, ma
-; CHECK-NEXT:    vle64.v v30, (a4)
-; CHECK-NEXT:    vmv1r.v v28, v8
-; CHECK-NEXT:    vle32.v v12, (t4)
-; CHECK-NEXT:    vmv1r.v v29, v8
-; CHECK-NEXT:    vle32.v v28, (t1)
-; CHECK-NEXT:    vmv1r.v v9, v8
-; CHECK-NEXT:    vle32.v v29, (t2)
-; CHECK-NEXT:    vmv1r.v v10, v8
-; CHECK-NEXT:    vle32.v v9, (t3)
-; CHECK-NEXT:    vmv1r.v v9, v8
-; CHECK-NEXT:    vle32.v v10, (a6)
-; CHECK-NEXT:    vmv1r.v v10, v8
-; CHECK-NEXT:    vle32.v v9, (a7)
-; CHECK-NEXT:    vmv1r.v v11, v8
-; CHECK-NEXT:    vle32.v v10, (a5)
-; CHECK-NEXT:    vmv1r.v v10, v8
-; CHECK-NEXT:    vle32.v v11, (a2)
-; CHECK-NEXT:    vle32.v v10, (a3)
-; CHECK-NEXT:    csrr t5, vlenb
-; CHECK-NEXT:    add t5, sp, t5
-; CHECK-NEXT:    addi t5, t5, 16
-; CHECK-NEXT:    vl1r.v v1, (t5) # vscale x 8-byte Folded Reload
+; CHECK-NEXT:    vle64.v v22, (a4)
+; CHECK-NEXT:    vmv1r.v v24, v20
+; CHECK-NEXT:    vle32.v v21, (t4)
+; CHECK-NEXT:    vmv1r.v v25, v20
+; CHECK-NEXT:    vle32.v v24, (t1)
+; CHECK-NEXT:    vmv1r.v v24, v20
+; CHECK-NEXT:    vle32.v v25, (t2)
+; CHECK-NEXT:    vmv1r.v v25, v20
+; CHECK-NEXT:    vle32.v v24, (t3)
+; CHECK-NEXT:    vmv1r.v v24, v20
+; CHECK-NEXT:    vle32.v v25, (a6)
+; CHECK-NEXT:    vmv1r.v v25, v20
+; CHECK-NEXT:    vle32.v v24, (a7)
+; CHECK-NEXT:    vmv1r.v v30, v20
+; CHECK-NEXT:    vle32.v v25, (a5)
+; CHECK-NEXT:    vmv1r.v v9, v20
+; CHECK-NEXT:    vle32.v v30, (a2)
+; CHECK-NEXT:    vle32.v v9, (a3)
+; CHECK-NEXT:    addi t5, sp, 16
+; CHECK-NEXT:    vl1r.v v25, (t5) # vscale x 8-byte Folded Reload
 ; CHECK-NEXT:    csrr t6, vlenb
 ; CHECK-NEXT:    add t5, t5, t6
-; CHECK-NEXT:    vl2r.v v2, (t5) # vscale x 16-byte Folded Reload
+; CHECK-NEXT:    vl2r.v v26, (t5) # vscale x 16-byte Folded Reload
 ; CHECK-NEXT:    slli t6, t6, 1
 ; CHECK-NEXT:    add t5, t5, t6
-; CHECK-NEXT:    vl1r.v v4, (t5) # vscale x 8-byte Folded Reload
+; CHECK-NEXT:    vl1r.v v28, (t5) # vscale x 8-byte Folded Reload
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vsseg4e32.v v1, (zero)
-; CHECK-NEXT:    vmv4r.v v0, v12
-; CHECK-NEXT:    vmv4r.v v4, v16
-; CHECK-NEXT:    vmv1r.v v2, v9
-; CHECK-NEXT:    vmv1r.v v0, v12
-; CHECK-NEXT:    vsseg8e32.v v0, (a1)
-; CHECK-NEXT:    addi t5, sp, 16
-; CHECK-NEXT:    vl1r.v v12, (t5) # vscale x 8-byte Folded Reload
-; CHECK-NEXT:    vmv1r.v v9, v12
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vsseg4e32.v v25, (zero)
+; CHECK-NEXT:    vmv2r.v v2, v10
+; CHECK-NEXT:    vmv4r.v v4, v12
+; CHECK-NEXT:    vmv1r.v v8, v16
+; CHECK-NEXT:    vmv1r.v v3, v24
+; CHECK-NEXT:    vmv1r.v v1, v21
+; CHECK-NEXT:    vsseg8e32.v v1, (a1)
+; CHECK-NEXT:    vmclr.m v8
+; CHECK-NEXT:    vmv.v.v v17, v8
+; CHECK-NEXT:    vmv.v.v v0, v8
 ; CHECK-NEXT:    csrr t5, vlenb
-; CHECK-NEXT:    slli t6, t5, 2
-; CHECK-NEXT:    add t5, t6, t5
+; CHECK-NEXT:    slli t5, t5, 2
 ; CHECK-NEXT:    add t5, sp, t5
 ; CHECK-NEXT:    addi t5, t5, 16
-; CHECK-NEXT:    vl2r.v v2, (t5) # vscale x 16-byte Folded Reload
-; CHECK-NEXT:    csrr t6, vlenb
-; CHECK-NEXT:    slli t6, t6, 1
-; CHECK-NEXT:    add t5, t5, t6
-; CHECK-NEXT:    vl2r.v v4, (t5) # vscale x 16-byte Folded Reload
+; CHECK-NEXT:    vl4r.v v24, (t5) # vscale x 32-byte Folded Reload
 ; CHECK-NEXT:    vsetvli zero, t0, e64, m2, ta, ma
-; CHECK-NEXT:    vsseg2e64.v v2, (zero)
+; CHECK-NEXT:    vsseg2e64.v v24, (zero)
+; CHECK-NEXT:    vsetvli t5, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v24, 0
 ; CHECK-NEXT:    vsetivli zero, 0, e64, m2, ta, mu
-; CHECK-NEXT:    vmv.v.i v28, 0
-; CHECK-NEXT:    vmflt.vv v9, v30, v28, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmflt.vv v17, v22, v18, v0.t
+; CHECK-NEXT:    vmv2r.v v26, v24
+; CHECK-NEXT:    vmv1r.v v0, v17
+; CHECK-NEXT:    vmv2r.v v28, v24
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, tu, mu
-; CHECK-NEXT:    vssub.vv v10, v8, v11, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v12
+; CHECK-NEXT:    vssub.vv v9, v20, v30, v0.t
+; CHECK-NEXT:    vmv2r.v v30, v24
+; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vsseg4e64.v v20, (zero), v0.t
+; CHECK-NEXT:    vsseg4e64.v v24, (zero), v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vsseg8e32.v v10, (a0)
+; CHECK-NEXT:    vsseg8e32.v v9, (a0)
 ; CHECK-NEXT:    csrr t5, vlenb
-; CHECK-NEXT:    slli t6, t5, 3
-; CHECK-NEXT:    add t5, t6, t5
+; CHECK-NEXT:    slli t5, t5, 3
 ; CHECK-NEXT:    add t5, sp, t5
 ; CHECK-NEXT:    addi t5, t5, 16
-; CHECK-NEXT:    vl8r.v v0, (t5) # vscale x 64-byte Folded Reload
+; CHECK-NEXT:    vl2r.v v22, (t5) # vscale x 16-byte Folded Reload
+; CHECK-NEXT:    csrr t6, vlenb
+; CHECK-NEXT:    slli t6, t6, 1
+; CHECK-NEXT:    add t5, t5, t6
+; CHECK-NEXT:    vl4r.v v24, (t5) # vscale x 32-byte Folded Reload
+; CHECK-NEXT:    slli t6, t6, 1
+; CHECK-NEXT:    add t5, t5, t6
+; CHECK-NEXT:    vl2r.v v28, (t5) # vscale x 16-byte Folded Reload
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vsseg4e64.v v0, (zero)
+; CHECK-NEXT:    vsseg4e64.v v22, (zero)
 ; CHECK-NEXT:    j .LBB0_1
 entry:
   store double 0.000000e+00, ptr %var_117, align 8
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll b/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
index ed04e9896a2cc3..89561d275c1bd4 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
@@ -724,147 +724,144 @@ define arm_aapcs_vfpcc void @gather_inc_v8i16_complex(ptr noalias nocapture read
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    .pad #120
-; CHECK-NEXT:    sub sp, #120
+; CHECK-NEXT:    .pad #104
+; CHECK-NEXT:    sub sp, #104
 ; CHECK-NEXT:    cmp r2, #1
-; CHECK-NEXT:    strd r1, r2, [sp, #60] @ 8-byte Folded Spill
+; CHECK-NEXT:    strd r1, r2, [sp, #44] @ 8-byte Folded Spill
 ; CHECK-NEXT:    blt.w .LBB15_5
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph.preheader
-; CHECK-NEXT:    ldr r1, [sp, #64] @ 4-byte Reload
-; CHECK-NEXT:    adr r3, .LCPI15_2
-; CHECK-NEXT:    vldrw.u32 q0, [r3]
+; CHECK-NEXT:    ldr r1, [sp, #48] @ 4-byte Reload
 ; CHECK-NEXT:    movs r2, #1
+; CHECK-NEXT:    add r3, sp, #88
+; CHECK-NEXT:    add r6, sp, #56
 ; CHECK-NEXT:    bic r1, r1, #7
-; CHECK-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    str r1, [sp, #40] @ 4-byte Spill
 ; CHECK-NEXT:    subs r1, #8
-; CHECK-NEXT:    vstrw.32 q0, [sp, #40] @ 16-byte Spill
-; CHECK-NEXT:    add r3, sp, #104
-; CHECK-NEXT:    add r7, sp, #72
+; CHECK-NEXT:    mov r12, r0
 ; CHECK-NEXT:    add.w r1, r2, r1, lsr #3
-; CHECK-NEXT:    str r1, [sp, #56] @ 4-byte Spill
+; CHECK-NEXT:    adr r2, .LCPI15_2
+; CHECK-NEXT:    vldrw.u32 q0, [r2]
+; CHECK-NEXT:    str r1, [sp, #36] @ 4-byte Spill
 ; CHECK-NEXT:    adr r1, .LCPI15_0
-; CHECK-NEXT:    adr r2, .LCPI15_1
+; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vstrw.32 q0, [sp, #24] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q0, [r2]
-; CHECK-NEXT:    vstrw.32 q0, [sp, #8] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
 ; CHECK-NEXT:  .LBB15_2: @ %vector.ph
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB15_3 Depth 2
-; CHECK-NEXT:    ldr r1, [sp, #56] @ 4-byte Reload
+; CHECK-NEXT:    ldr r1, [sp, #36] @ 4-byte Reload
 ; CHECK-NEXT:    dls lr, r1
-; CHECK-NEXT:    ldr.w r12, [sp, #60] @ 4-byte Reload
-; CHECK-NEXT:    vldrw.u32 q4, [sp, #24] @ 16-byte Reload
-; CHECK-NEXT:    vldrw.u32 q5, [sp, #40] @ 16-byte Reload
-; CHECK-NEXT:    vldrw.u32 q6, [sp, #8] @ 16-byte Reload
+; CHECK-NEXT:    adr r1, .LCPI15_1
+; CHECK-NEXT:    ldr.w r9, [sp, #44] @ 4-byte Reload
+; CHECK-NEXT:    vldrw.u32 q3, [r1]
+; CHECK-NEXT:    vldrw.u32 q4, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q5, [sp, #16] @ 16-byte Reload
 ; CHECK-NEXT:  .LBB15_3: @ %vector.body
 ; CHECK-NEXT:    @ Parent Loop BB15_2 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    vstrw.32 q5, [r3]
-; CHECK-NEXT:    mov r8, r3
 ; CHECK-NEXT:    vldrh.s32 q1, [r3, #8]
 ; CHECK-NEXT:    vshl.i32 q1, q1, #1
-; CHECK-NEXT:    vadd.i32 q1, q1, r4
-; CHECK-NEXT:    vmov r1, r2, d2
-; CHECK-NEXT:    vmov r5, r6, d3
+; CHECK-NEXT:    vadd.i32 q1, q1, r12
+; CHECK-NEXT:    vmov r1, r4, d2
+; CHECK-NEXT:    vmov r2, r5, d3
 ; CHECK-NEXT:    vldrh.s32 q1, [r3]
 ; CHECK-NEXT:    vshl.i32 q1, q1, #1
-; CHECK-NEXT:    vadd.i32 q2, q1, r4
-; CHECK-NEXT:    vmov r3, r9, d4
-; CHECK-NEXT:    ldrh r0, [r2]
-; CHECK-NEXT:    ldrh.w r11, [r1]
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    str r0, [sp, #68] @ 4-byte Spill
-; CHECK-NEXT:    add r0, sp, #88
-; CHECK-NEXT:    ldrh.w r10, [r5]
-; CHECK-NEXT:    ldrh r2, [r6]
-; CHECK-NEXT:    ldrh r3, [r3]
-; CHECK-NEXT:    ldrh.w r5, [r9]
-; CHECK-NEXT:    vstrw.32 q6, [r7]
-; CHECK-NEXT:    vldrh.s32 q1, [r7]
-; CHECK-NEXT:    vmov.16 q7[0], r3
-; CHECK-NEXT:    vmov.16 q7[1], r5
-; CHECK-NEXT:    vldrh.s32 q3, [r1, #8]
-; CHECK-NEXT:    vshl.i32 q1, q1, #1
-; CHECK-NEXT:    vadd.i32 q1, q1, r4
-; CHECK-NEXT:    vshl.i32 q3, q3, #1
-; CHECK-NEXT:    vmov r6, r7, d2
-; CHECK-NEXT:    vadd.i32 q3, q3, r4
-; CHECK-NEXT:    vmov r3, r5, d3
+; CHECK-NEXT:    vadd.i32 q1, q1, r12
+; CHECK-NEXT:    vmov r10, r8, d2
+; CHECK-NEXT:    ldrh r3, [r1]
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    ldrh r0, [r5]
+; CHECK-NEXT:    ldrh.w r11, [r2]
+; CHECK-NEXT:    str r0, [sp, #52] @ 4-byte Spill
+; CHECK-NEXT:    add r0, sp, #72
+; CHECK-NEXT:    ldrh r4, [r4]
+; CHECK-NEXT:    ldrh.w r5, [r10]
+; CHECK-NEXT:    ldrh.w r2, [r8]
+; CHECK-NEXT:    vstrw.32 q3, [r6]
+; CHECK-NEXT:    vldrh.s32 q2, [r6]
+; CHECK-NEXT:    vmov.16 q6[0], r5
+; CHECK-NEXT:    vmov.16 q6[1], r2
+; CHECK-NEXT:    vshl.i32 q2, q2, #1
+; CHECK-NEXT:    vadd.i32 q2, q2, r12
+; CHECK-NEXT:    vmov r6, r7, d4
+; CHECK-NEXT:    vmov r2, r5, d5
+; CHECK-NEXT:    vldrh.s32 q2, [r1, #8]
+; CHECK-NEXT:    vshl.i32 q2, q2, #1
+; CHECK-NEXT:    vadd.i32 q2, q2, r12
 ; CHECK-NEXT:    ldrh r6, [r6]
-; CHECK-NEXT:    ldrh r3, [r3]
-; CHECK-NEXT:    vmov.16 q1[0], r6
+; CHECK-NEXT:    ldrh r2, [r2]
+; CHECK-NEXT:    vmov.16 q7[0], r6
 ; CHECK-NEXT:    ldrh r6, [r7]
 ; CHECK-NEXT:    ldrh r5, [r5]
-; CHECK-NEXT:    vmov.16 q1[1], r6
-; CHECK-NEXT:    vmov.16 q1[2], r3
-; CHECK-NEXT:    vmov r3, r6, d6
-; CHECK-NEXT:    vmov.16 q1[3], r5
-; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.16 q7[1], r6
+; CHECK-NEXT:    vmov.16 q7[2], r2
+; CHECK-NEXT:    vmov r2, r6, d4
+; CHECK-NEXT:    vmov.16 q7[3], r5
+; CHECK-NEXT:    ldrh r2, [r2]
 ; CHECK-NEXT:    ldrh r6, [r6]
-; CHECK-NEXT:    vmov.16 q1[4], r3
-; CHECK-NEXT:    vmov r3, r5, d7
-; CHECK-NEXT:    vmov.16 q1[5], r6
-; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.16 q7[4], r2
+; CHECK-NEXT:    vmov r2, r5, d5
+; CHECK-NEXT:    vmov.16 q7[5], r6
+; CHECK-NEXT:    ldrh r2, [r2]
 ; CHECK-NEXT:    ldrh r5, [r5]
 ; CHECK-NEXT:    vstrw.32 q4, [r0]
-; CHECK-NEXT:    vldrh.s32 q3, [r0]
-; CHECK-NEXT:    vmov.16 q1[6], r3
-; CHECK-NEXT:    vmov.16 q1[7], r5
-; CHECK-NEXT:    vshl.i32 q3, q3, #1
-; CHECK-NEXT:    vadd.i32 q0, q3, r4
-; CHECK-NEXT:    vmov r3, r5, d0
+; CHECK-NEXT:    vldrh.s32 q2, [r0]
+; CHECK-NEXT:    vmov.16 q7[6], r2
+; CHECK-NEXT:    vmov.16 q7[7], r5
+; CHECK-NEXT:    vshl.i32 q2, q2, #1
+; CHECK-NEXT:    vadd.i32 q0, q2, r12
+; CHECK-NEXT:    vmov r2, r5, d0
 ; CHECK-NEXT:    vmov r6, r7, d1
 ; CHECK-NEXT:    vldrh.s32 q0, [r0, #8]
-; CHECK-NEXT:    ldr r0, [sp, #68] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #52] @ 4-byte Reload
 ; CHECK-NEXT:    vshl.i32 q0, q0, #1
-; CHECK-NEXT:    vadd.i32 q0, q0, r4
-; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vadd.i32 q0, q0, r12
+; CHECK-NEXT:    ldrh r2, [r2]
 ; CHECK-NEXT:    ldrh r5, [r5]
-; CHECK-NEXT:    vmov.16 q3[0], r3
+; CHECK-NEXT:    vmov.16 q2[0], r2
 ; CHECK-NEXT:    ldrh r6, [r6]
-; CHECK-NEXT:    vmov.16 q3[1], r5
+; CHECK-NEXT:    vmov.16 q2[1], r5
 ; CHECK-NEXT:    ldrh r7, [r7]
-; CHECK-NEXT:    vmov.16 q3[2], r6
-; CHECK-NEXT:    vmov r3, r5, d5
-; CHECK-NEXT:    vmov.16 q3[3], r7
+; CHECK-NEXT:    vmov.16 q2[2], r6
+; CHECK-NEXT:    vmov r2, r5, d3
+; CHECK-NEXT:    vmov.16 q2[3], r7
 ; CHECK-NEXT:    vmov r6, r7, d0
-; CHECK-NEXT:    vmov.i16 q2, #0x18
-; CHECK-NEXT:    vadd.i16 q6, q6, q2
-; CHECK-NEXT:    vadd.i16 q5, q5, q2
-; CHECK-NEXT:    vadd.i16 q4, q4, q2
-; CHECK-NEXT:    ldrh r3, [r3]
+; CHECK-NEXT:    vmov.i16 q1, #0x18
+; CHECK-NEXT:    vadd.i16 q3, q3, q1
+; CHECK-NEXT:    vadd.i16 q5, q5, q1
+; CHECK-NEXT:    vadd.i16 q4, q4, q1
+; CHECK-NEXT:    ldrh r2, [r2]
 ; CHECK-NEXT:    ldrh r6, [r6]
 ; CHECK-NEXT:    ldrh r7, [r7]
-; CHECK-NEXT:    vmov.16 q7[2], r3
-; CHECK-NEXT:    vmov.16 q3[4], r6
+; CHECK-NEXT:    vmov.16 q6[2], r2
+; CHECK-NEXT:    vmov.16 q2[4], r6
 ; CHECK-NEXT:    ldrh r5, [r5]
-; CHECK-NEXT:    vmov.16 q3[5], r7
+; CHECK-NEXT:    vmov.16 q2[5], r7
 ; CHECK-NEXT:    vmov r6, r7, d1
-; CHECK-NEXT:    vmov.16 q7[3], r5
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    vmov.16 q7[4], r11
-; CHECK-NEXT:    vmov.16 q7[5], r0
-; CHECK-NEXT:    vmov.16 q7[6], r10
-; CHECK-NEXT:    vmov.16 q7[7], r2
+; CHECK-NEXT:    vmov.16 q6[3], r5
+; CHECK-NEXT:    vmov.16 q6[4], r3
+; CHECK-NEXT:    add r3, sp, #88
+; CHECK-NEXT:    vmov.16 q6[5], r4
+; CHECK-NEXT:    vmov.16 q6[6], r11
+; CHECK-NEXT:    vmov.16 q6[7], r0
 ; CHECK-NEXT:    ldrh r6, [r6]
 ; CHECK-NEXT:    ldrh r7, [r7]
-; CHECK-NEXT:    vmov.16 q3[6], r6
-; CHECK-NEXT:    vmov.16 q3[7], r7
-; CHECK-NEXT:    mov r7, r1
-; CHECK-NEXT:    vadd.i16 q0, q3, q1
-; CHECK-NEXT:    vadd.i16 q0, q0, q7
-; CHECK-NEXT:    vstrb.8 q0, [r12], #16
+; CHECK-NEXT:    vmov.16 q2[6], r6
+; CHECK-NEXT:    mov r6, r1
+; CHECK-NEXT:    vmov.16 q2[7], r7
+; CHECK-NEXT:    vadd.i16 q0, q2, q7
+; CHECK-NEXT:    vadd.i16 q0, q0, q6
+; CHECK-NEXT:    vstrb.8 q0, [r9], #16
 ; CHECK-NEXT:    le lr, .LBB15_3
 ; CHECK-NEXT:  @ %bb.4: @ %middle.block
 ; CHECK-NEXT:    @ in Loop: Header=BB15_2 Depth=1
-; CHECK-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    ldr r2, [sp, #64] @ 4-byte Reload
+; CHECK-NEXT:    ldr r1, [sp, #40] @ 4-byte Reload
+; CHECK-NEXT:    ldr r2, [sp, #48] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r1, r2
 ; CHECK-NEXT:    bne.w .LBB15_2
 ; CHECK-NEXT:  .LBB15_5: @ %for.cond.cleanup
-; CHECK-NEXT:    add sp, #120
+; CHECK-NEXT:    add sp, #104
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -952,228 +949,231 @@ define arm_aapcs_vfpcc void @gather_inc_v16i8_complex(ptr noalias nocapture read
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    .pad #224
 ; CHECK-NEXT:    sub sp, #224
-; CHECK-NEXT:    adr r3, .LCPI16_0
 ; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    vldrw.u32 q0, [r3]
-; CHECK-NEXT:    adr r3, .LCPI16_1
-; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q0, [r3]
-; CHECK-NEXT:    adr r3, .LCPI16_8
+; CHECK-NEXT:    adr r1, .LCPI16_0
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    adr r1, .LCPI16_1
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q0, [r3]
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
 ; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
 ; CHECK-NEXT:  .LBB16_2: @ %vector.ph
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB16_3 Depth 2
-; CHECK-NEXT:    adr r1, .LCPI16_7
+; CHECK-NEXT:    adr r1, .LCPI16_8
 ; CHECK-NEXT:    bic r11, r2, #7
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    adr r1, .LCPI16_7
+; CHECK-NEXT:    mov r12, r10
+; CHECK-NEXT:    mov lr, r11
+; CHECK-NEXT:    vstrw.32 q0, [sp, #208] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
 ; CHECK-NEXT:    adr r1, .LCPI16_9
-; CHECK-NEXT:    vldrw.u32 q6, [r1]
+; CHECK-NEXT:    vldrw.u32 q5, [r1]
 ; CHECK-NEXT:    adr r1, .LCPI16_5
-; CHECK-NEXT:    vstrw.32 q0, [sp, #144] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q4, [r1]
 ; CHECK-NEXT:    adr r1, .LCPI16_6
-; CHECK-NEXT:    mov r9, r10
-; CHECK-NEXT:    vldrw.u32 q2, [r1]
-; CHECK-NEXT:    adr r1, .LCPI16_3
 ; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    adr r1, .LCPI16_3
+; CHECK-NEXT:    vldrw.u32 q6, [r1]
 ; CHECK-NEXT:    adr r1, .LCPI16_4
-; CHECK-NEXT:    vstrw.32 q0, [sp, #192] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #96] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
 ; CHECK-NEXT:    adr r1, .LCPI16_2
-; CHECK-NEXT:    mov r12, r11
-; CHECK-NEXT:    vldrw.u32 q7, [r1]
+; CHECK-NEXT:    vldrw.u32 q2, [r1]
 ; CHECK-NEXT:    adr r1, .LCPI16_10
-; CHECK-NEXT:    vstrw.32 q0, [sp, #208] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #144] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
 ; CHECK-NEXT:    adr r1, .LCPI16_11
-; CHECK-NEXT:    vstrw.32 q0, [sp, #176] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
-; CHECK-NEXT:    vldrw.u32 q4, [r1]
-; CHECK-NEXT:    vstrw.32 q0, [sp, #128] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #192] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q7, [r1]
+; CHECK-NEXT:    vstrw.32 q0, [sp, #128] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #112] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
-; CHECK-NEXT:    vstrw.32 q0, [sp, #96] @ 16-byte Spill
 ; CHECK-NEXT:  .LBB16_3: @ %vector.body
 ; CHECK-NEXT:    @ Parent Loop BB16_2 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    vstrw.32 q1, [sp, #160] @ 16-byte Spill
-; CHECK-NEXT:    vadd.i32 q3, q2, r0
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #160] @ 16-byte Reload
-; CHECK-NEXT:    vmov r1, r8, d6
+; CHECK-NEXT:    vstrw.32 q7, [sp, #160] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q1, [sp, #176] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #160] @ 16-byte Reload
+; CHECK-NEXT:    vadd.i32 q0, q6, r0
+; CHECK-NEXT:    vmov r7, r3, d1
 ; CHECK-NEXT:    vstrw.32 q2, [sp, #80] @ 16-byte Spill
-; CHECK-NEXT:    vadd.i32 q2, q7, r0
-; CHECK-NEXT:    vadd.i32 q1, q0, r0
-; CHECK-NEXT:    vadd.i32 q0, q4, r0
-; CHECK-NEXT:    vmov r5, r6, d0
-; CHECK-NEXT:    vldrw.u32 q5, [sp, #144] @ 16-byte Reload
-; CHECK-NEXT:    vmov r4, lr, d1
-; CHECK-NEXT:    vstrw.32 q7, [sp, #64] @ 16-byte Spill
-; CHECK-NEXT:    vmov r7, r3, d3
-; CHECK-NEXT:    vldrw.u32 q7, [sp, #128] @ 16-byte Reload
+; CHECK-NEXT:    vadd.i32 q7, q1, r0
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #176] @ 16-byte Reload
+; CHECK-NEXT:    vmov r6, r5, d14
+; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vmov r4, r8, d15
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
+; CHECK-NEXT:    vmov r1, r9, d2
+; CHECK-NEXT:    vstrw.32 q1, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vmov q1, q6
+; CHECK-NEXT:    vmov q6, q4
 ; CHECK-NEXT:    vstrw.32 q6, [sp, #48] @ 16-byte Spill
-; CHECK-NEXT:    subs.w r12, r12, #16
-; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vstrw.32 q5, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #96] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q3, [sp, #112] @ 16-byte Reload
+; CHECK-NEXT:    subs.w lr, lr, #16
+; CHECK-NEXT:    ldrb r3, [r3]
+; CHECK-NEXT:    ldrb r6, [r6]
 ; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q7[0], r6
 ; CHECK-NEXT:    ldrb r4, [r4]
-; CHECK-NEXT:    vmov.8 q0[0], r5
-; CHECK-NEXT:    ldrb r5, [r6]
+; CHECK-NEXT:    vmov.8 q7[1], r5
 ; CHECK-NEXT:    ldrb r6, [r7]
-; CHECK-NEXT:    vmov.8 q0[1], r5
-; CHECK-NEXT:    ldrb.w r7, [lr]
-; CHECK-NEXT:    vmov.8 q0[2], r4
-; CHECK-NEXT:    vmov r4, r5, d2
-; CHECK-NEXT:    ldrb r3, [r3]
-; CHECK-NEXT:    vmov.8 q0[3], r7
+; CHECK-NEXT:    vmov.8 q7[2], r4
+; CHECK-NEXT:    vmov r4, r5, d0
+; CHECK-NEXT:    ldrb.w r7, [r8]
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vmov.8 q7[3], r7
 ; CHECK-NEXT:    ldrb r4, [r4]
 ; CHECK-NEXT:    ldrb r5, [r5]
-; CHECK-NEXT:    vmov.8 q1[0], r4
+; CHECK-NEXT:    vmov.8 q0[0], r4
 ; CHECK-NEXT:    vmov r4, r7, d4
-; CHECK-NEXT:    vmov.8 q1[1], r5
-; CHECK-NEXT:    vmov.8 q1[2], r6
-; CHECK-NEXT:    vmov r6, lr, d5
-; CHECK-NEXT:    vmov.8 q1[3], r3
-; CHECK-NEXT:    vldrw.u32 q2, [sp, #176] @ 16-byte Reload
-; CHECK-NEXT:    vmov.8 q1[4], r1
-; CHECK-NEXT:    vadd.i32 q2, q2, r0
+; CHECK-NEXT:    vmov.8 q0[1], r5
+; CHECK-NEXT:    vmov.8 q0[2], r6
+; CHECK-NEXT:    vmov r6, r8, d5
+; CHECK-NEXT:    vmov q2, q6
+; CHECK-NEXT:    vmov q6, q1
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    vmov.8 q0[3], r3
+; CHECK-NEXT:    vmov.8 q0[4], r1
 ; CHECK-NEXT:    ldrb r5, [r7]
-; CHECK-NEXT:    ldrb.w r7, [r8]
+; CHECK-NEXT:    ldrb.w r7, [r9]
 ; CHECK-NEXT:    ldrb r3, [r6]
-; CHECK-NEXT:    vmov r1, r6, d7
-; CHECK-NEXT:    vmov.8 q1[5], r7
-; CHECK-NEXT:    vldrw.u32 q3, [sp, #112] @ 16-byte Reload
+; CHECK-NEXT:    vmov r1, r6, d3
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #192] @ 16-byte Reload
+; CHECK-NEXT:    vmov.8 q0[5], r7
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
 ; CHECK-NEXT:    ldrb r1, [r1]
 ; CHECK-NEXT:    ldrb r6, [r6]
-; CHECK-NEXT:    vmov.8 q1[6], r1
-; CHECK-NEXT:    vmov r1, r7, d4
-; CHECK-NEXT:    vmov.8 q1[7], r6
+; CHECK-NEXT:    vmov.8 q0[6], r1
+; CHECK-NEXT:    vmov r1, r7, d2
+; CHECK-NEXT:    vmov.8 q0[7], r6
 ; CHECK-NEXT:    ldrb r1, [r1]
 ; CHECK-NEXT:    ldrb r7, [r7]
-; CHECK-NEXT:    vmov.8 q0[4], r1
-; CHECK-NEXT:    vmov r1, r6, d5
-; CHECK-NEXT:    vmov.8 q0[5], r7
-; CHECK-NEXT:    vadd.i32 q2, q6, r0
-; CHECK-NEXT:    vldrw.u32 q6, [sp, #96] @ 16-byte Reload
+; CHECK-NEXT:    vmov.8 q7[4], r1
+; CHECK-NEXT:    vmov r1, r6, d3
+; CHECK-NEXT:    vmov.8 q7[5], r7
+; CHECK-NEXT:    vadd.i32 q1, q5, r0
+; CHECK-NEXT:    vldrw.u32 q5, [sp, #144] @ 16-byte Reload
 ; CHECK-NEXT:    ldrb r1, [r1]
 ; CHECK-NEXT:    ldrb r6, [r6]
-; CHECK-NEXT:    vmov.8 q0[6], r1
+; CHECK-NEXT:    vmov.8 q7[6], r1
 ; CHECK-NEXT:    ldrb r1, [r4]
-; CHECK-NEXT:    vmov.8 q0[7], r6
-; CHECK-NEXT:    vmov r4, r7, d4
-; CHECK-NEXT:    vmov.8 q0[8], r1
-; CHECK-NEXT:    vmov r1, r8, d5
-; CHECK-NEXT:    vldrw.u32 q2, [sp, #192] @ 16-byte Reload
-; CHECK-NEXT:    vmov.8 q0[9], r5
-; CHECK-NEXT:    vmov.8 q0[10], r3
-; CHECK-NEXT:    vadd.i32 q2, q2, r0
-; CHECK-NEXT:    vmov r5, r6, d4
+; CHECK-NEXT:    vmov.8 q7[7], r6
+; CHECK-NEXT:    vmov r4, r7, d2
+; CHECK-NEXT:    vmov.8 q7[8], r1
+; CHECK-NEXT:    vmov r1, r9, d3
+; CHECK-NEXT:    vadd.i32 q1, q2, r0
+; CHECK-NEXT:    vmov.8 q7[9], r5
+; CHECK-NEXT:    vmov r5, r6, d2
+; CHECK-NEXT:    vmov.8 q7[10], r3
+; CHECK-NEXT:    vldrw.u32 q2, [sp, #128] @ 16-byte Reload
 ; CHECK-NEXT:    ldrb r3, [r7]
-; CHECK-NEXT:    ldrb.w r7, [lr]
+; CHECK-NEXT:    ldrb.w r7, [r8]
 ; CHECK-NEXT:    ldrb r4, [r4]
-; CHECK-NEXT:    vmov.8 q0[11], r7
-; CHECK-NEXT:    ldrb r1, [r1]
 ; CHECK-NEXT:    ldrb r5, [r5]
+; CHECK-NEXT:    vmov.8 q7[11], r7
 ; CHECK-NEXT:    ldrb r6, [r6]
-; CHECK-NEXT:    vmov.8 q1[8], r5
-; CHECK-NEXT:    vmov r7, r5, d5
-; CHECK-NEXT:    vmov.8 q1[9], r6
-; CHECK-NEXT:    vldrw.u32 q2, [sp, #208] @ 16-byte Reload
-; CHECK-NEXT:    vadd.i32 q2, q2, r0
-; CHECK-NEXT:    ldrb r7, [r7]
+; CHECK-NEXT:    vmov.8 q0[8], r5
+; CHECK-NEXT:    vmov r5, r7, d3
+; CHECK-NEXT:    vmov.8 q0[9], r6
+; CHECK-NEXT:    ldrb r1, [r1]
+; CHECK-NEXT:    vadd.i32 q1, q5, r0
 ; CHECK-NEXT:    ldrb r5, [r5]
-; CHECK-NEXT:    vmov.8 q1[10], r7
-; CHECK-NEXT:    vmov.8 q1[11], r5
-; CHECK-NEXT:    vmov.8 q1[12], r4
-; CHECK-NEXT:    vmov.8 q1[13], r3
-; CHECK-NEXT:    vmov.8 q1[14], r1
-; CHECK-NEXT:    vmov r1, r3, d4
+; CHECK-NEXT:    ldrb r7, [r7]
+; CHECK-NEXT:    vmov.8 q0[10], r5
+; CHECK-NEXT:    vmov.8 q0[11], r7
+; CHECK-NEXT:    vmov.8 q0[12], r4
+; CHECK-NEXT:    vmov.8 q0[13], r3
+; CHECK-NEXT:    vmov.8 q0[14], r1
+; CHECK-NEXT:    vmov r1, r3, d2
 ; CHECK-NEXT:    ldrb r1, [r1]
-; CHECK-NEXT:    vmov.8 q0[12], r1
+; CHECK-NEXT:    vmov.8 q7[12], r1
 ; CHECK-NEXT:    ldrb r1, [r3]
-; CHECK-NEXT:    vmov.8 q0[13], r1
-; CHECK-NEXT:    vmov r1, r3, d5
-; CHECK-NEXT:    vadd.i32 q2, q5, r0
+; CHECK-NEXT:    vmov.8 q7[13], r1
+; CHECK-NEXT:    vmov r1, r3, d3
+; CHECK-NEXT:    vadd.i32 q1, q4, r0
 ; CHECK-NEXT:    ldrb r1, [r1]
-; CHECK-NEXT:    vmov.8 q0[14], r1
+; CHECK-NEXT:    vmov.8 q7[14], r1
 ; CHECK-NEXT:    ldrb r1, [r3]
+; CHECK-NEXT:    vmov.8 q7[15], r1
+; CHECK-NEXT:    ldrb.w r1, [r9]
 ; CHECK-NEXT:    vmov.8 q0[15], r1
-; CHECK-NEXT:    ldrb.w r1, [r8]
-; CHECK-NEXT:    vmov.8 q1[15], r1
-; CHECK-NEXT:    vmov r1, r3, d4
-; CHECK-NEXT:    vadd.i8 q0, q1, q0
+; CHECK-NEXT:    vmov r1, r3, d2
+; CHECK-NEXT:    vadd.i8 q0, q0, q7
 ; CHECK-NEXT:    ldrb r1, [r1]
 ; CHECK-NEXT:    ldrb r3, [r3]
-; CHECK-NEXT:    vmov.8 q1[0], r1
-; CHECK-NEXT:    vmov.8 q1[1], r3
-; CHECK-NEXT:    vmov r1, r3, d5
-; CHECK-NEXT:    vadd.i32 q2, q7, r0
+; CHECK-NEXT:    vmov.8 q7[0], r1
+; CHECK-NEXT:    vmov.8 q7[1], r3
+; CHECK-NEXT:    vmov r1, r3, d3
+; CHECK-NEXT:    vadd.i32 q1, q2, r0
 ; CHECK-NEXT:    ldrb r1, [r1]
-; CHECK-NEXT:    vmov.8 q1[2], r1
+; CHECK-NEXT:    vmov.8 q7[2], r1
 ; CHECK-NEXT:    ldrb r1, [r3]
-; CHECK-NEXT:    vmov.8 q1[3], r1
-; CHECK-NEXT:    vmov r1, r3, d4
+; CHECK-NEXT:    vmov.8 q7[3], r1
+; CHECK-NEXT:    vmov r1, r3, d2
 ; CHECK-NEXT:    ldrb r1, [r1]
-; CHECK-NEXT:    vmov.8 q1[4], r1
+; CHECK-NEXT:    vmov.8 q7[4], r1
 ; CHECK-NEXT:    ldrb r1, [r3]
-; CHECK-NEXT:    vmov.8 q1[5], r1
-; CHECK-NEXT:    vmov r1, r3, d5
-; CHECK-NEXT:    vadd.i32 q2, q3, r0
+; CHECK-NEXT:    vmov.8 q7[5], r1
+; CHECK-NEXT:    vmov r1, r3, d3
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #208] @ 16-byte Reload
+; CHECK-NEXT:    vadd.i32 q1, q1, r0
 ; CHECK-NEXT:    ldrb r1, [r1]
-; CHECK-NEXT:    vmov.8 q1[6], r1
+; CHECK-NEXT:    vmov.8 q7[6], r1
 ; CHECK-NEXT:    ldrb r1, [r3]
-; CHECK-NEXT:    vmov.8 q1[7], r1
-; CHECK-NEXT:    vmov r1, r3, d4
+; CHECK-NEXT:    vmov.8 q7[7], r1
+; CHECK-NEXT:    vmov r1, r3, d2
 ; CHECK-NEXT:    ldrb r1, [r1]
-; CHECK-NEXT:    vmov.8 q1[8], r1
+; CHECK-NEXT:    vmov.8 q7[8], r1
 ; CHECK-NEXT:    ldrb r1, [r3]
-; CHECK-NEXT:    vmov.8 q1[9], r1
-; CHECK-NEXT:    vmov r1, r3, d5
-; CHECK-NEXT:    vadd.i32 q2, q6, r0
+; CHECK-NEXT:    vmov.8 q7[9], r1
+; CHECK-NEXT:    vmov r1, r3, d3
+; CHECK-NEXT:    vadd.i32 q1, q3, r0
 ; CHECK-NEXT:    ldrb r1, [r1]
-; CHECK-NEXT:    vmov.8 q1[10], r1
+; CHECK-NEXT:    vmov.8 q7[10], r1
 ; CHECK-NEXT:    ldrb r1, [r3]
-; CHECK-NEXT:    vmov.8 q1[11], r1
-; CHECK-NEXT:    vmov r1, r3, d4
+; CHECK-NEXT:    vmov.8 q7[11], r1
+; CHECK-NEXT:    vmov r1, r3, d2
 ; CHECK-NEXT:    ldrb r1, [r1]
-; CHECK-NEXT:    vmov.8 q1[12], r1
+; CHECK-NEXT:    vmov.8 q7[12], r1
 ; CHECK-NEXT:    ldrb r1, [r3]
-; CHECK-NEXT:    vmov.8 q1[13], r1
-; CHECK-NEXT:    vmov r1, r3, d5
-; CHECK-NEXT:    vldrw.u32 q2, [sp, #80] @ 16-byte Reload
+; CHECK-NEXT:    vmov.8 q7[13], r1
+; CHECK-NEXT:    vmov r1, r3, d3
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #176] @ 16-byte Reload
 ; CHECK-NEXT:    ldrb r1, [r1]
-; CHECK-NEXT:    vmov.8 q1[14], r1
+; CHECK-NEXT:    vmov.8 q7[14], r1
 ; CHECK-NEXT:    ldrb r1, [r3]
-; CHECK-NEXT:    vmov.8 q1[15], r1
-; CHECK-NEXT:    vadd.i8 q0, q0, q1
-; CHECK-NEXT:    vldrw.u32 q1, [sp, #160] @ 16-byte Reload
-; CHECK-NEXT:    vstrb.8 q0, [r9], #16
+; CHECK-NEXT:    vmov.8 q7[15], r1
+; CHECK-NEXT:    vadd.i8 q0, q0, q7
+; CHECK-NEXT:    vldrw.u32 q7, [sp, #160] @ 16-byte Reload
+; CHECK-NEXT:    vstrb.8 q0, [r12], #16
 ; CHECK-NEXT:    vmov.i32 q0, #0x30
-; CHECK-NEXT:    vadd.i32 q3, q3, q0
+; CHECK-NEXT:    vadd.i32 q4, q4, q0
+; CHECK-NEXT:    vadd.i32 q2, q2, q0
+; CHECK-NEXT:    vstrw.32 q4, [sp, #96] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #192] @ 16-byte Reload
 ; CHECK-NEXT:    vadd.i32 q5, q5, q0
-; CHECK-NEXT:    vstrw.32 q3, [sp, #112] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q3, [sp, #192] @ 16-byte Reload
-; CHECK-NEXT:    vadd.i32 q7, q7, q0
-; CHECK-NEXT:    vadd.i32 q6, q6, q0
-; CHECK-NEXT:    vadd.i32 q3, q3, q0
+; CHECK-NEXT:    vstrw.32 q2, [sp, #128] @ 16-byte Spill
+; CHECK-NEXT:    vadd.i32 q4, q4, q0
+; CHECK-NEXT:    vldrw.u32 q2, [sp, #80] @ 16-byte Reload
+; CHECK-NEXT:    vstrw.32 q4, [sp, #192] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #208] @ 16-byte Reload
 ; CHECK-NEXT:    vstrw.32 q5, [sp, #144] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q5, [sp, #176] @ 16-byte Reload
-; CHECK-NEXT:    vstrw.32 q7, [sp, #128] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q7, [sp, #64] @ 16-byte Reload
-; CHECK-NEXT:    vstrw.32 q3, [sp, #192] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q3, [sp, #208] @ 16-byte Reload
-; CHECK-NEXT:    vstrw.32 q6, [sp, #96] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q6, [sp, #48] @ 16-byte Reload
-; CHECK-NEXT:    vadd.i32 q5, q5, q0
-; CHECK-NEXT:    vadd.i32 q3, q3, q0
+; CHECK-NEXT:    vldrw.u32 q5, [sp, #64] @ 16-byte Reload
 ; CHECK-NEXT:    vadd.i32 q4, q4, q0
-; CHECK-NEXT:    vadd.i32 q1, q1, q0
-; CHECK-NEXT:    vadd.i32 q2, q2, q0
+; CHECK-NEXT:    vadd.i32 q3, q3, q0
+; CHECK-NEXT:    vstrw.32 q4, [sp, #208] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q4, [sp, #48] @ 16-byte Reload
 ; CHECK-NEXT:    vadd.i32 q7, q7, q0
 ; CHECK-NEXT:    vadd.i32 q6, q6, q0
-; CHECK-NEXT:    vstrw.32 q5, [sp, #176] @ 16-byte Spill
-; CHECK-NEXT:    vstrw.32 q3, [sp, #208] @ 16-byte Spill
+; CHECK-NEXT:    vadd.i32 q1, q1, q0
+; CHECK-NEXT:    vadd.i32 q2, q2, q0
+; CHECK-NEXT:    vadd.i32 q4, q4, q0
+; CHECK-NEXT:    vadd.i32 q5, q5, q0
+; CHECK-NEXT:    vstrw.32 q3, [sp, #112] @ 16-byte Spill
 ; CHECK-NEXT:    bne.w .LBB16_3
 ; CHECK-NEXT:  @ %bb.4: @ %middle.block
 ; CHECK-NEXT:    @ in Loop: Header=BB16_2 Depth=1
diff --git a/llvm/test/CodeGen/X86/combine-pmuldq.ll b/llvm/test/CodeGen/X86/combine-pmuldq.ll
index fff9b5ac3b2748..bcefe52063e160 100644
--- a/llvm/test/CodeGen/X86/combine-pmuldq.ll
+++ b/llvm/test/CodeGen/X86/combine-pmuldq.ll
@@ -412,33 +412,33 @@ define <8 x i32> @PR49658_zext(ptr %ptr, i32 %mul) {
 ; AVX1-NEXT:    movl %esi, %eax
 ; AVX1-NEXT:    vmovq %rax, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm5
-; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm4[2,2],xmm5[2,2]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm4[0,2],xmm5[0,2]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm4[2,2],xmm5[2,2]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[0,2],xmm5[0,2]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[2,2],xmm2[2,2]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm1[0,2],xmm2[0,2]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm5 = xmm1[2,2],xmm2[2,2]
 ; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB7_1: # %loop
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
-; AVX1-NEXT:    vmovdqu 2097152(%rdi,%rax), %xmm5
-; AVX1-NEXT:    vmovdqu 2097168(%rdi,%rax), %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm5[1,1,3,3]
-; AVX1-NEXT:    vpmuludq %xmm1, %xmm7, %xmm7
-; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm7[2,3],xmm5[4,5],xmm7[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; AVX1-NEXT:    vpmuludq %xmm3, %xmm7, %xmm7
+; AVX1-NEXT:    vmovdqu 2097152(%rdi,%rax), %xmm6
+; AVX1-NEXT:    vmovdqu 2097168(%rdi,%rax), %xmm7
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm8 = xmm6[1,1,3,3]
+; AVX1-NEXT:    vpmuludq %xmm3, %xmm8, %xmm8
 ; AVX1-NEXT:    vpmuludq %xmm4, %xmm6, %xmm6
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm7[2,3],xmm6[4,5],xmm7[6,7]
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpaddd %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2,3],xmm6[4,5],xmm8[6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
+; AVX1-NEXT:    vpmuludq %xmm5, %xmm8, %xmm8
+; AVX1-NEXT:    vshufps {{.*#+}} xmm9 = xmm1[0,2],xmm2[0,2]
+; AVX1-NEXT:    vpmuludq %xmm7, %xmm9, %xmm7
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm8[2,3],xmm7[4,5],xmm8[6,7]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm8
+; AVX1-NEXT:    vpaddd %xmm7, %xmm8, %xmm7
+; AVX1-NEXT:    vpaddd %xmm0, %xmm6, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm0
 ; AVX1-NEXT:    subq $-128, %rax
 ; AVX1-NEXT:    jne .LBB7_1
 ; AVX1-NEXT:  # %bb.2: # %end
@@ -604,33 +604,33 @@ define <8 x i32> @PR49658_sext(ptr %ptr, i32 %mul) {
 ; AVX1-NEXT:    movslq %esi, %rax
 ; AVX1-NEXT:    vmovq %rax, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
 ; AVX1-NEXT:    vpxor %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    movq $-2097152, %rax # imm = 0xFFE00000
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm5
-; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm4[2,2],xmm5[2,2]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm4[0,2],xmm5[0,2]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm4[2,2],xmm5[2,2]
-; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[0,2],xmm5[0,2]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[2,2],xmm2[2,2]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm4 = xmm1[0,2],xmm2[0,2]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm5 = xmm1[2,2],xmm2[2,2]
 ; AVX1-NEXT:    .p2align 4
 ; AVX1-NEXT:  .LBB8_1: # %loop
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
-; AVX1-NEXT:    vmovdqu 2097152(%rdi,%rax), %xmm5
-; AVX1-NEXT:    vmovdqu 2097168(%rdi,%rax), %xmm6
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm5[1,1,3,3]
-; AVX1-NEXT:    vpmuldq %xmm1, %xmm7, %xmm7
-; AVX1-NEXT:    vpmuldq %xmm2, %xmm5, %xmm5
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm7[2,3],xmm5[4,5],xmm7[6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; AVX1-NEXT:    vpmuldq %xmm3, %xmm7, %xmm7
+; AVX1-NEXT:    vmovdqu 2097152(%rdi,%rax), %xmm6
+; AVX1-NEXT:    vmovdqu 2097168(%rdi,%rax), %xmm7
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm8 = xmm6[1,1,3,3]
+; AVX1-NEXT:    vpmuldq %xmm3, %xmm8, %xmm8
 ; AVX1-NEXT:    vpmuldq %xmm4, %xmm6, %xmm6
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm7[2,3],xmm6[4,5],xmm7[6,7]
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpaddd %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2,3],xmm6[4,5],xmm8[6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
+; AVX1-NEXT:    vpmuldq %xmm5, %xmm8, %xmm8
+; AVX1-NEXT:    vshufps {{.*#+}} xmm9 = xmm1[0,2],xmm2[0,2]
+; AVX1-NEXT:    vpmuldq %xmm9, %xmm7, %xmm7
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm8[2,3],xmm7[4,5],xmm8[6,7]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm8
+; AVX1-NEXT:    vpaddd %xmm7, %xmm8, %xmm7
+; AVX1-NEXT:    vpaddd %xmm0, %xmm6, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm0
 ; AVX1-NEXT:    subq $-128, %rax
 ; AVX1-NEXT:    jne .LBB8_1
 ; AVX1-NEXT:  # %bb.2: # %end
diff --git a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
index a4fda1558a08b9..719876b4e3b212 100644
--- a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
@@ -359,15 +359,21 @@ define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture r
 ; XOP-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
 ; XOP-NEXT:    xorl %ecx, %ecx
 ; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; XOP-NEXT:    vextractf128 $1, %ymm1, %xmm4
 ; XOP-NEXT:    .p2align 4
 ; XOP-NEXT:  .LBB0_4: # %vector.body
 ; XOP-NEXT:    # =>This Inner Loop Header: Depth=1
+; XOP-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero
+; XOP-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero
 ; XOP-NEXT:    vmovq {{.*#+}} xmm5 = mem[0],zero
 ; XOP-NEXT:    vmovq {{.*#+}} xmm6 = mem[0],zero
-; XOP-NEXT:    vmovq {{.*#+}} xmm7 = mem[0],zero
-; XOP-NEXT:    vmovq {{.*#+}} xmm8 = mem[0],zero
+; XOP-NEXT:    vpcomeqb %xmm2, %xmm3, %xmm3
+; XOP-NEXT:    vpmovsxbd %xmm3, %xmm7
+; XOP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]
+; XOP-NEXT:    vpmovsxbd %xmm3, %xmm3
+; XOP-NEXT:    vpcomeqb %xmm2, %xmm4, %xmm4
+; XOP-NEXT:    vpmovsxbd %xmm4, %xmm8
+; XOP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
+; XOP-NEXT:    vpmovsxbd %xmm4, %xmm4
 ; XOP-NEXT:    vpcomeqb %xmm2, %xmm5, %xmm5
 ; XOP-NEXT:    vpmovsxbd %xmm5, %xmm9
 ; XOP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,1,1]
@@ -376,38 +382,32 @@ define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture r
 ; XOP-NEXT:    vpmovsxbd %xmm6, %xmm10
 ; XOP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[1,1,1,1]
 ; XOP-NEXT:    vpmovsxbd %xmm6, %xmm6
-; XOP-NEXT:    vpcomeqb %xmm2, %xmm7, %xmm7
-; XOP-NEXT:    vpmovsxbd %xmm7, %xmm11
-; XOP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,1,1]
-; XOP-NEXT:    vpmovsxbd %xmm7, %xmm7
-; XOP-NEXT:    vpcomeqb %xmm2, %xmm8, %xmm8
-; XOP-NEXT:    vpmovsxbd %xmm8, %xmm12
-; XOP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[1,1,1,1]
-; XOP-NEXT:    vpmovsxbd %xmm8, %xmm8
-; XOP-NEXT:    vblendvps %xmm5, %xmm3, %xmm4, %xmm5
-; XOP-NEXT:    vpshld %xmm5, 16(%rdi,%rcx,4), %xmm5
+; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm11
+; XOP-NEXT:    vextractf128 $1, %ymm1, %xmm12
+; XOP-NEXT:    vblendvps %xmm3, %xmm11, %xmm12, %xmm3
+; XOP-NEXT:    vpshld %xmm3, 16(%rdi,%rcx,4), %xmm3
+; XOP-NEXT:    vblendvps %xmm7, %xmm0, %xmm1, %xmm7
+; XOP-NEXT:    vpshld %xmm7, (%rdi,%rcx,4), %xmm7
+; XOP-NEXT:    vblendvps %xmm4, %xmm11, %xmm12, %xmm4
+; XOP-NEXT:    vpshld %xmm4, 48(%rdi,%rcx,4), %xmm4
+; XOP-NEXT:    vblendvps %xmm8, %xmm0, %xmm1, %xmm8
+; XOP-NEXT:    vpshld %xmm8, 32(%rdi,%rcx,4), %xmm8
+; XOP-NEXT:    vblendvps %xmm5, %xmm11, %xmm12, %xmm5
+; XOP-NEXT:    vpshld %xmm5, 80(%rdi,%rcx,4), %xmm5
 ; XOP-NEXT:    vblendvps %xmm9, %xmm0, %xmm1, %xmm9
-; XOP-NEXT:    vpshld %xmm9, (%rdi,%rcx,4), %xmm9
-; XOP-NEXT:    vblendvps %xmm6, %xmm3, %xmm4, %xmm6
-; XOP-NEXT:    vpshld %xmm6, 48(%rdi,%rcx,4), %xmm6
+; XOP-NEXT:    vpshld %xmm9, 64(%rdi,%rcx,4), %xmm9
+; XOP-NEXT:    vblendvps %xmm6, %xmm11, %xmm12, %xmm6
+; XOP-NEXT:    vpshld %xmm6, 112(%rdi,%rcx,4), %xmm6
 ; XOP-NEXT:    vblendvps %xmm10, %xmm0, %xmm1, %xmm10
-; XOP-NEXT:    vpshld %xmm10, 32(%rdi,%rcx,4), %xmm10
-; XOP-NEXT:    vblendvps %xmm7, %xmm3, %xmm4, %xmm7
-; XOP-NEXT:    vpshld %xmm7, 80(%rdi,%rcx,4), %xmm7
-; XOP-NEXT:    vblendvps %xmm11, %xmm0, %xmm1, %xmm11
-; XOP-NEXT:    vpshld %xmm11, 64(%rdi,%rcx,4), %xmm11
-; XOP-NEXT:    vblendvps %xmm8, %xmm3, %xmm4, %xmm8
-; XOP-NEXT:    vpshld %xmm8, 112(%rdi,%rcx,4), %xmm8
-; XOP-NEXT:    vblendvps %xmm12, %xmm0, %xmm1, %xmm12
-; XOP-NEXT:    vpshld %xmm12, 96(%rdi,%rcx,4), %xmm12
-; XOP-NEXT:    vmovdqu %xmm9, (%rdi,%rcx,4)
-; XOP-NEXT:    vmovdqu %xmm5, 16(%rdi,%rcx,4)
-; XOP-NEXT:    vmovdqu %xmm10, 32(%rdi,%rcx,4)
-; XOP-NEXT:    vmovdqu %xmm6, 48(%rdi,%rcx,4)
-; XOP-NEXT:    vmovdqu %xmm11, 64(%rdi,%rcx,4)
-; XOP-NEXT:    vmovdqu %xmm7, 80(%rdi,%rcx,4)
-; XOP-NEXT:    vmovdqu %xmm12, 96(%rdi,%rcx,4)
-; XOP-NEXT:    vmovdqu %xmm8, 112(%rdi,%rcx,4)
+; XOP-NEXT:    vpshld %xmm10, 96(%rdi,%rcx,4), %xmm10
+; XOP-NEXT:    vmovdqu %xmm7, (%rdi,%rcx,4)
+; XOP-NEXT:    vmovdqu %xmm3, 16(%rdi,%rcx,4)
+; XOP-NEXT:    vmovdqu %xmm8, 32(%rdi,%rcx,4)
+; XOP-NEXT:    vmovdqu %xmm4, 48(%rdi,%rcx,4)
+; XOP-NEXT:    vmovdqu %xmm9, 64(%rdi,%rcx,4)
+; XOP-NEXT:    vmovdqu %xmm5, 80(%rdi,%rcx,4)
+; XOP-NEXT:    vmovdqu %xmm10, 96(%rdi,%rcx,4)
+; XOP-NEXT:    vmovdqu %xmm6, 112(%rdi,%rcx,4)
 ; XOP-NEXT:    addq $32, %rcx
 ; XOP-NEXT:    cmpq %rcx, %rdx
 ; XOP-NEXT:    jne .LBB0_4
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll
index 082b876b542e5d..f2bfe5dab3006e 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll
@@ -40,22 +40,22 @@ define i32 @simple(ptr %a, ptr %b, i32 %x) nounwind {
 ; X32-NEXT:    pushl %edi
 ; X32-NEXT:    pushl %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    shll $2, %edx
 ; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    leal (%edx,%edx), %esi
 ; X32-NEXT:    .p2align 4
 ; X32-NEXT:  .LBB0_1: # %loop
 ; X32-NEXT:    # =>This Inner Loop Header: Depth=1
-; X32-NEXT:    addl (%edi), %eax
-; X32-NEXT:    addl (%edi,%edx), %eax
-; X32-NEXT:    addl %edx, %edi
-; X32-NEXT:    addl (%edx,%edi), %eax
-; X32-NEXT:    addl %edx, %edi
-; X32-NEXT:    addl (%edx,%edi), %eax
-; X32-NEXT:    addl %esi, %edi
-; X32-NEXT:    cmpl %ecx, %edi
+; X32-NEXT:    addl (%esi), %eax
+; X32-NEXT:    addl (%esi,%edx), %eax
+; X32-NEXT:    addl %edx, %esi
+; X32-NEXT:    addl (%edx,%esi), %eax
+; X32-NEXT:    addl %edx, %esi
+; X32-NEXT:    addl (%edx,%esi), %eax
+; X32-NEXT:    addl %edx, %esi
+; X32-NEXT:    addl %edx, %esi
+; X32-NEXT:    cmpl %ecx, %esi
 ; X32-NEXT:    jne .LBB0_1
 ; X32-NEXT:  # %bb.2: # %exit
 ; X32-NEXT:    popl %esi



More information about the llvm-branch-commits mailing list