[llvm] [AMDGPU] Avoid slow 64-bit max/min/multiply instructions on GFX1250 (PR #207006)
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 02:33:02 PDT 2026
https://github.com/jayfoad updated https://github.com/llvm/llvm-project/pull/207006
>From 07c909e4dc65472eb1e8393d5a97e5cee57324ea Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Wed, 1 Jul 2026 14:58:07 +0100
Subject: [PATCH] [AMDGPU] Avoid slow 64-bit max/min/multiply instructions on
GFX1250
On GFX1250 64-bit integer VALU max/min instructions are slower than
expanding to compare and select, and 64-bit integer VALU multiply is
slower than expanding to a sequence of 32-bit multiplies and adds. Avoid
them by pretending that these cases are not legal during instruction
selection.
---
llvm/lib/Target/AMDGPU/AMDGPU.td | 9 +-
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 4 +-
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 6 +-
.../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 6 +-
llvm/lib/Target/AMDGPU/GCNSubtarget.h | 12 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 2 +-
.../AMDGPU/GlobalISel/inst-select-smax-64.mir | 2 +-
.../AMDGPU/GlobalISel/inst-select-smin-64.mir | 2 +-
.../AMDGPU/GlobalISel/inst-select-umax-64.mir | 2 +-
.../AMDGPU/GlobalISel/inst-select-umin-64.mir | 2 +-
.../AMDGPU/GlobalISel/legalize-smax.mir | 5 +-
.../AMDGPU/GlobalISel/legalize-smin.mir | 5 +-
.../AMDGPU/GlobalISel/legalize-umax.mir | 5 +-
.../AMDGPU/GlobalISel/legalize-umin.mir | 5 +-
.../AMDGPU/GlobalISel/minmaxabs-i64.ll | 92 +++--
llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll | 7 +-
.../CodeGen/AMDGPU/atomics-system-scope.ll | 368 ++++++++++--------
llvm/test/CodeGen/AMDGPU/clmul.ll | 136 ++++---
.../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 96 +++--
.../CodeGen/AMDGPU/integer-mad-patterns.ll | 258 +++++++++---
llvm/test/CodeGen/AMDGPU/mad_64_32.ll | 63 ++-
llvm/test/CodeGen/AMDGPU/max.ll | 28 +-
llvm/test/CodeGen/AMDGPU/min.ll | 28 +-
llvm/test/CodeGen/AMDGPU/mul.ll | 103 ++---
.../AMDGPU/reassoc-mul-add-1-to-mad.ll | 31 +-
26 files changed, 793 insertions(+), 488 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 9da123c517b81..1a68de155bf88 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1217,6 +1217,11 @@ defm FlatSignedOffset : AMDGPUSubtargetFeature<"flat-signed-offset",
"Immediate offset of FLAT instructions are always signed"
>;
+defm SlowMaxMinMulI64Insts : AMDGPUSubtargetFeature<"slow-max-min-mul-i64-insts",
+ "64-bit integer max/min/mul VALU instructions are slower than expanding to an equivalent code sequence",
+ GenPredicate=false
+>;
+
//===------------------------------------------------------------===//
// Subtarget Features (options and debugging)
//===------------------------------------------------------------===//
@@ -2253,7 +2258,9 @@ def FeatureISAVersion12_50 : FeatureSet<
FeatureSWMMACGfx1200Insts,
FeatureSWMMACGfx1250Insts,
FeatureTransCoexecutionHazard,
- FeatureWMMACoexecutionHazards])>;
+ FeatureWMMACoexecutionHazards,
+ FeatureSlowMaxMinMulI64Insts,
+])>;
def FeatureISAVersion12_51 : FeatureSet<
!listconcat(FeatureISAVersion12_50_Common.Features,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 40dfa3aca26b6..8e77dec772097 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1468,7 +1468,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.widenScalarToNextPow2(0)
.scalarize(0)
.lower();
- if (ST.hasMinMaxI64Insts()) {
+ if (ST.useMinMaxI64Insts()) {
getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
.legalFor({S32, S16, S64, V2S16})
.clampMaxNumElements(0, S16, 2)
@@ -4695,7 +4695,7 @@ bool AMDGPULegalizerInfo::legalizeMul(LegalizerHelper &Helper,
assert(Ty.isScalar());
unsigned Size = Ty.getSizeInBits();
- if (ST.hasVMulU64Inst() && Size == 64)
+ if (ST.useVMulU64Inst() && Size == 64)
return true;
unsigned NumParts = Size / 32;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 7f7225ad6d311..ff94bd9450696 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -613,7 +613,7 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(V2S16, {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
- bool HasVecMulU64 = ST->hasVMulU64Inst();
+ bool UseVecMulU64 = ST->useVMulU64Inst();
addRulesForGOpcs({G_MUL}, Standard)
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})
@@ -622,8 +622,8 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(V2S16, {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
.Uni(S16, {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}})
- .Div(S64, {{VgprB64}, {VgprB64, VgprB64}}, HasVecMulU64)
- .Div(S64, {{VgprB64}, {VgprB64, VgprB64}, SplitTo32Mul}, !HasVecMulU64);
+ .Div(S64, {{VgprB64}, {VgprB64, VgprB64}}, UseVecMulU64)
+ .Div(S64, {{VgprB64}, {VgprB64, VgprB64}, SplitTo32Mul}, !UseVecMulU64);
bool hasMulHi = ST->hasScalarMulHiInsts();
addRulesForGOpcs({G_UMULH, G_SMULH}, Standard)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 200234c23c886..95280ebd2a907 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -2529,7 +2529,7 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
// Special case for s_mul_u64. There is not a vector equivalent of
// s_mul_u64. Hence, we have to break down s_mul_u64 into 32-bit vector
// multiplications.
- if (!Subtarget.hasVMulU64Inst() && Opc == AMDGPU::G_MUL &&
+ if (!Subtarget.useVMulU64Inst() && Opc == AMDGPU::G_MUL &&
DstTy.getSizeInBits() == 64) {
applyMappingSMULU64(B, OpdMapper);
return;
@@ -4027,7 +4027,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
OpdsMapping[0] = getValueMappingSGPR64Only(AMDGPU::SGPRRegBankID, Size);
OpdsMapping[1] = OpdsMapping[2] = OpdsMapping[0];
} else {
- if (MI.getOpcode() == AMDGPU::G_MUL && Subtarget.hasVMulU64Inst())
+ if (MI.getOpcode() == AMDGPU::G_MUL && Subtarget.useVMulU64Inst())
OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
else
OpdsMapping[0] =
@@ -4079,7 +4079,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
if (isSALUMapping(MI)) {
// There are no scalar 64-bit min and max, use vector instruction instead.
if (MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 64 &&
- Subtarget.hasMinMaxI64Insts())
+ Subtarget.useMinMaxI64Insts())
return getDefaultMappingVOP(MI);
return getDefaultMappingSOP(MI);
}
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 7e4d9bd20cb2f..01a838e7f0de5 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -1042,6 +1042,18 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
bool requiresWaitOnWorkgroupReleaseFence() const {
return getGeneration() >= GFX10 || isTgSplitEnabled();
}
+
+ // \returns true if ISel should select the native i64 min/max instructions
+ // (V_MIN/MAX_{I|U}64).
+ bool useMinMaxI64Insts() const {
+ return hasMinMaxI64Insts() && !hasSlowMaxMinMulI64Insts();
+ }
+
+ // \returns true if ISel should select the native i64 mul instruction
+ // V_MUL_U64.
+ bool useVMulU64Inst() const {
+ return hasVMulU64Inst() && !hasSlowMaxMinMulI64Insts();
+ }
};
class GCNUserSGPRUsageInfo {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index b2f91e97fd426..bc5aa7efee1a5 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -950,7 +950,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction({ISD::SMULO, ISD::UMULO}, MVT::i64, Custom);
- if (Subtarget->hasVMulU64Inst())
+ if (Subtarget->useVMulU64Inst())
setOperationAction(ISD::MUL, MVT::i64, Legal);
else if (Subtarget->hasScalarSMulU64())
setOperationAction(ISD::MUL, MVT::i64, Custom);
@@ -985,7 +985,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
Custom);
}
- if (Subtarget->hasMinMaxI64Insts())
+ if (Subtarget->useMinMaxI64Insts())
setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, MVT::i64,
Legal);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 48a85fdc01ea6..923d54cb13401 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7995,7 +7995,7 @@ void SIInstrInfo::moveToVALUImpl(
}
case AMDGPU::S_MUL_U64:
- if (ST.hasVMulU64Inst()) {
+ if (ST.useVMulU64Inst()) {
NewOpcode = AMDGPU::V_MUL_U64_e64;
break;
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smax-64.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smax-64.mir
index d9e3a8fc5e065..f68f50e6bc703 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smax-64.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smax-64.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=instruction-select %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1251 -run-pass=instruction-select %s -o - | FileCheck -check-prefix=GCN %s
---
name: smax_s64_sv
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smin-64.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smin-64.mir
index bf9d07008edc8..40af38df56f48 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smin-64.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smin-64.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=instruction-select %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1251 -run-pass=instruction-select %s -o - | FileCheck -check-prefix=GCN %s
---
name: smin_s64_sv
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umax-64.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umax-64.mir
index ed062db6a3060..ec42c3513a531 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umax-64.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umax-64.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=instruction-select %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1251 -run-pass=instruction-select %s -o - | FileCheck -check-prefix=GCN %s
---
name: umax_s64_sv
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umin-64.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umin-64.mir
index 5da10087ff74f..59291c663f0ab 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umin-64.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umin-64.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=instruction-select %s -o - | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1251 -run-pass=instruction-select %s -o - | FileCheck -check-prefix=GCN %s
---
name: umin_s64_sv
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir
index b5ec2ac393be7..48807bb502cf0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir
@@ -87,8 +87,9 @@ body: |
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
- ; GFX1250-NEXT: [[SMAX:%[0-9]+]]:_(s64) = G_SMAX [[COPY]], [[COPY1]]
- ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[SMAX]](s64)
+ ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY]](s64), [[COPY1]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[COPY]], [[COPY1]]
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(s64) = G_SMAX %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir
index 3eaecf6b19883..0f9fb929ff9aa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir
@@ -87,8 +87,9 @@ body: |
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
- ; GFX1250-NEXT: [[SMIN:%[0-9]+]]:_(s64) = G_SMIN [[COPY]], [[COPY1]]
- ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[SMIN]](s64)
+ ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[COPY]](s64), [[COPY1]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[COPY]], [[COPY1]]
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(s64) = G_SMIN %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir
index f89425f5e91cb..19f86b4443375 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir
@@ -87,8 +87,9 @@ body: |
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
- ; GFX1250-NEXT: [[UMAX:%[0-9]+]]:_(s64) = G_UMAX [[COPY]], [[COPY1]]
- ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[UMAX]](s64)
+ ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ugt), [[COPY]](s64), [[COPY1]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[COPY]], [[COPY1]]
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(s64) = G_UMAX %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir
index e3dd7d4137806..e003dfb58377a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir
@@ -87,8 +87,9 @@ body: |
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
- ; GFX1250-NEXT: [[UMIN:%[0-9]+]]:_(s64) = G_UMIN [[COPY]], [[COPY1]]
- ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[UMIN]](s64)
+ ; GFX1250-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY]](s64), [[COPY1]]
+ ; GFX1250-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[COPY]], [[COPY1]]
+ ; GFX1250-NEXT: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s64) = COPY $vgpr2_vgpr3
%2:_(s64) = G_UMIN %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll
index 6a7725d19086a..a0cc2ae1d0ba8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll
@@ -17,7 +17,8 @@ define i64 @test_umin_i64(i64 %a, i64 %b) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_min_u64 v[0:1], v[0:1], v[2:3]
+; CHECK-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]
+; CHECK-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call i64 @llvm.umin.i64(i64 %a, i64 %b)
ret i64 %r
@@ -28,7 +29,8 @@ define i64 @test_umax_i64(i64 %a, i64 %b) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_max_u64 v[0:1], v[0:1], v[2:3]
+; CHECK-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3]
+; CHECK-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call i64 @llvm.umax.i64(i64 %a, i64 %b)
ret i64 %r
@@ -39,7 +41,8 @@ define i64 @test_smin_i64(i64 %a, i64 %b) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_min_i64 v[0:1], v[0:1], v[2:3]
+; CHECK-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3]
+; CHECK-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call i64 @llvm.smin.i64(i64 %a, i64 %b)
ret i64 %r
@@ -50,7 +53,8 @@ define i64 @test_smax_i64(i64 %a, i64 %b) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_max_i64 v[0:1], v[0:1], v[2:3]
+; CHECK-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3]
+; CHECK-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call i64 @llvm.smax.i64(i64 %a, i64 %b)
ret i64 %r
@@ -61,10 +65,15 @@ define <4 x i64> @test_umin_v4i64(<4 x i64> %a, <4 x i64> %b) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_min_u64 v[0:1], v[0:1], v[8:9]
-; CHECK-NEXT: v_min_u64 v[2:3], v[2:3], v[10:11]
-; CHECK-NEXT: v_min_u64 v[4:5], v[4:5], v[12:13]
-; CHECK-NEXT: v_min_u64 v[6:7], v[6:7], v[14:15]
+; CHECK-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[8:9]
+; CHECK-NEXT: v_cmp_lt_u64_e64 s0, v[2:3], v[10:11]
+; CHECK-NEXT: v_cmp_lt_u64_e64 s1, v[4:5], v[12:13]
+; CHECK-NEXT: v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
+; CHECK-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[6:7], v[14:15]
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; CHECK-NEXT: v_dual_cndmask_b32 v2, v10, v2, s0 :: v_dual_cndmask_b32 v3, v11, v3, s0
+; CHECK-NEXT: v_dual_cndmask_b32 v4, v12, v4, s1 :: v_dual_cndmask_b32 v5, v13, v5, s1
+; CHECK-NEXT: v_dual_cndmask_b32 v6, v14, v6 :: v_dual_cndmask_b32 v7, v15, v7
; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %a, <4 x i64> %b)
ret <4 x i64> %r
@@ -75,10 +84,15 @@ define <4 x i64> @test_umax_v4i64(<4 x i64> %a, <4 x i64> %b) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_max_u64 v[0:1], v[0:1], v[8:9]
-; CHECK-NEXT: v_max_u64 v[2:3], v[2:3], v[10:11]
-; CHECK-NEXT: v_max_u64 v[4:5], v[4:5], v[12:13]
-; CHECK-NEXT: v_max_u64 v[6:7], v[6:7], v[14:15]
+; CHECK-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[8:9]
+; CHECK-NEXT: v_cmp_gt_u64_e64 s0, v[2:3], v[10:11]
+; CHECK-NEXT: v_cmp_gt_u64_e64 s1, v[4:5], v[12:13]
+; CHECK-NEXT: v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
+; CHECK-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[6:7], v[14:15]
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; CHECK-NEXT: v_dual_cndmask_b32 v2, v10, v2, s0 :: v_dual_cndmask_b32 v3, v11, v3, s0
+; CHECK-NEXT: v_dual_cndmask_b32 v4, v12, v4, s1 :: v_dual_cndmask_b32 v5, v13, v5, s1
+; CHECK-NEXT: v_dual_cndmask_b32 v6, v14, v6 :: v_dual_cndmask_b32 v7, v15, v7
; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call <4 x i64> @llvm.umax.v4i64(<4 x i64> %a, <4 x i64> %b)
ret <4 x i64> %r
@@ -89,10 +103,15 @@ define <4 x i64> @test_smin_v4i64(<4 x i64> %a, <4 x i64> %b) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_min_i64 v[0:1], v[0:1], v[8:9]
-; CHECK-NEXT: v_min_i64 v[2:3], v[2:3], v[10:11]
-; CHECK-NEXT: v_min_i64 v[4:5], v[4:5], v[12:13]
-; CHECK-NEXT: v_min_i64 v[6:7], v[6:7], v[14:15]
+; CHECK-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[8:9]
+; CHECK-NEXT: v_cmp_lt_i64_e64 s0, v[2:3], v[10:11]
+; CHECK-NEXT: v_cmp_lt_i64_e64 s1, v[4:5], v[12:13]
+; CHECK-NEXT: v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
+; CHECK-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[6:7], v[14:15]
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; CHECK-NEXT: v_dual_cndmask_b32 v2, v10, v2, s0 :: v_dual_cndmask_b32 v3, v11, v3, s0
+; CHECK-NEXT: v_dual_cndmask_b32 v4, v12, v4, s1 :: v_dual_cndmask_b32 v5, v13, v5, s1
+; CHECK-NEXT: v_dual_cndmask_b32 v6, v14, v6 :: v_dual_cndmask_b32 v7, v15, v7
; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %a, <4 x i64> %b)
ret <4 x i64> %r
@@ -103,10 +122,15 @@ define <4 x i64> @test_smax_v4i64(<4 x i64> %a, <4 x i64> %b) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_wait_loadcnt_dscnt 0x0
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_max_i64 v[0:1], v[0:1], v[8:9]
-; CHECK-NEXT: v_max_i64 v[2:3], v[2:3], v[10:11]
-; CHECK-NEXT: v_max_i64 v[4:5], v[4:5], v[12:13]
-; CHECK-NEXT: v_max_i64 v[6:7], v[6:7], v[14:15]
+; CHECK-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[8:9]
+; CHECK-NEXT: v_cmp_gt_i64_e64 s0, v[2:3], v[10:11]
+; CHECK-NEXT: v_cmp_gt_i64_e64 s1, v[4:5], v[12:13]
+; CHECK-NEXT: v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
+; CHECK-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[6:7], v[14:15]
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; CHECK-NEXT: v_dual_cndmask_b32 v2, v10, v2, s0 :: v_dual_cndmask_b32 v3, v11, v3, s0
+; CHECK-NEXT: v_dual_cndmask_b32 v4, v12, v4, s1 :: v_dual_cndmask_b32 v5, v13, v5, s1
+; CHECK-NEXT: v_dual_cndmask_b32 v6, v14, v6 :: v_dual_cndmask_b32 v7, v15, v7
; CHECK-NEXT: s_set_pc_i64 s[30:31]
%r = call <4 x i64> @llvm.smax.v4i64(<4 x i64> %a, <4 x i64> %b)
ret <4 x i64> %r
@@ -133,10 +157,9 @@ define amdgpu_ps i64 @test_umin_i64_s(i64 inreg %a, i64 inreg %b) {
; CHECK-LABEL: test_umin_i64_s:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; CHECK-NEXT: v_min_u64 v[0:1], s[0:1], s[2:3]
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
-; CHECK-NEXT: v_readfirstlane_b32 s1, v1
+; CHECK-NEXT: v_cmp_lt_u64_e64 s4, s[0:1], s[2:3]
+; CHECK-NEXT: s_cmp_lg_u32 s4, 0
+; CHECK-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; CHECK-NEXT: ; return to shader part epilog
%r = call i64 @llvm.umin.i64(i64 %a, i64 %b)
ret i64 %r
@@ -146,10 +169,9 @@ define amdgpu_ps i64 @test_umax_i64_s(i64 inreg %a, i64 inreg %b) {
; CHECK-LABEL: test_umax_i64_s:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; CHECK-NEXT: v_max_u64 v[0:1], s[0:1], s[2:3]
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
-; CHECK-NEXT: v_readfirstlane_b32 s1, v1
+; CHECK-NEXT: v_cmp_gt_u64_e64 s4, s[0:1], s[2:3]
+; CHECK-NEXT: s_cmp_lg_u32 s4, 0
+; CHECK-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; CHECK-NEXT: ; return to shader part epilog
%r = call i64 @llvm.umax.i64(i64 %a, i64 %b)
ret i64 %r
@@ -159,10 +181,9 @@ define amdgpu_ps i64 @test_smin_i64_s(i64 inreg %a, i64 inreg %b) {
; CHECK-LABEL: test_smin_i64_s:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; CHECK-NEXT: v_min_i64 v[0:1], s[0:1], s[2:3]
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
-; CHECK-NEXT: v_readfirstlane_b32 s1, v1
+; CHECK-NEXT: v_cmp_lt_i64_e64 s4, s[0:1], s[2:3]
+; CHECK-NEXT: s_cmp_lg_u32 s4, 0
+; CHECK-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; CHECK-NEXT: ; return to shader part epilog
%r = call i64 @llvm.smin.i64(i64 %a, i64 %b)
ret i64 %r
@@ -172,10 +193,9 @@ define amdgpu_ps i64 @test_smax_i64_s(i64 inreg %a, i64 inreg %b) {
; CHECK-LABEL: test_smax_i64_s:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; CHECK-NEXT: v_max_i64 v[0:1], s[0:1], s[2:3]
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
-; CHECK-NEXT: v_readfirstlane_b32 s1, v1
+; CHECK-NEXT: v_cmp_gt_i64_e64 s4, s[0:1], s[2:3]
+; CHECK-NEXT: s_cmp_lg_u32 s4, 0
+; CHECK-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; CHECK-NEXT: ; return to shader part epilog
%r = call i64 @llvm.smax.i64(i64 %a, i64 %b)
ret i64 %r
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index 976fef0e3807e..5844eef90d0a8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -852,7 +852,12 @@ define i64 @v_mul_i64(i64 %num, i64 %den) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mul_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT: v_mul_hi_u32 v4, v0, v2
+; GFX1250-NEXT: v_mul_lo_u32 v3, v0, v3
+; GFX1250-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX1250-NEXT: v_mul_lo_u32 v0, v0, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_add3_u32 v1, v3, v1, v4
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-LABEL: v_mul_i64:
diff --git a/llvm/test/CodeGen/AMDGPU/atomics-system-scope.ll b/llvm/test/CodeGen/AMDGPU/atomics-system-scope.ll
index e3c375fae7386..786792d54b175 100644
--- a/llvm/test/CodeGen/AMDGPU/atomics-system-scope.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomics-system-scope.ll
@@ -1149,36 +1149,42 @@ define i64 @flat_one_as_atomic_min_i64(ptr %ptr, i64 %val) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB52_2
-; GFX1250-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX1250-NEXT: flat_atomic_min_i64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT: s_cbranch_execnz .LBB52_3
+; GFX1250-NEXT: ; %bb.1: ; %Flow
+; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB52_4
+; GFX1250-NEXT: .LBB52_2: ; %atomicrmw.phi
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-NEXT: .LBB52_3: ; %atomicrmw.global
+; GFX1250-NEXT: flat_atomic_min_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-NEXT: .LBB52_2: ; %Flow
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB52_4
-; GFX1250-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT: s_cbranch_execz .LBB52_2
+; GFX1250-NEXT: .LBB52_4: ; %atomicrmw.private
+; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_min_i64 v[0:1], v[4:5], v[2:3]
-; GFX1250-NEXT: scratch_store_b64 v6, v[0:1], off
-; GFX1250-NEXT: .LBB52_4: ; %atomicrmw.phi
+; GFX1250-NEXT: v_cmp_le_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
+; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%result = atomicrmw min ptr %ptr, i64 %val syncscope("one-as") monotonic
ret i64 %result
@@ -1189,36 +1195,42 @@ define i64 @flat_system_atomic_min_i64(ptr %ptr, i64 %val) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB53_2
-; GFX1250-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX1250-NEXT: flat_atomic_min_i64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT: s_cbranch_execnz .LBB53_3
+; GFX1250-NEXT: ; %bb.1: ; %Flow
+; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB53_4
+; GFX1250-NEXT: .LBB53_2: ; %atomicrmw.phi
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-NEXT: .LBB53_3: ; %atomicrmw.global
+; GFX1250-NEXT: flat_atomic_min_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-NEXT: .LBB53_2: ; %Flow
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB53_4
-; GFX1250-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT: s_cbranch_execz .LBB53_2
+; GFX1250-NEXT: .LBB53_4: ; %atomicrmw.private
+; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_min_i64 v[0:1], v[4:5], v[2:3]
-; GFX1250-NEXT: scratch_store_b64 v6, v[0:1], off
-; GFX1250-NEXT: .LBB53_4: ; %atomicrmw.phi
+; GFX1250-NEXT: v_cmp_le_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
+; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%result = atomicrmw min ptr %ptr, i64 %val monotonic
ret i64 %result
@@ -1229,36 +1241,42 @@ define i64 @flat_one_as_atomic_max_i64(ptr %ptr, i64 %val) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB54_2
-; GFX1250-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX1250-NEXT: flat_atomic_max_i64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT: s_cbranch_execnz .LBB54_3
+; GFX1250-NEXT: ; %bb.1: ; %Flow
+; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB54_4
+; GFX1250-NEXT: .LBB54_2: ; %atomicrmw.phi
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-NEXT: .LBB54_3: ; %atomicrmw.global
+; GFX1250-NEXT: flat_atomic_max_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-NEXT: .LBB54_2: ; %Flow
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB54_4
-; GFX1250-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT: s_cbranch_execz .LBB54_2
+; GFX1250-NEXT: .LBB54_4: ; %atomicrmw.private
+; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_max_i64 v[0:1], v[4:5], v[2:3]
-; GFX1250-NEXT: scratch_store_b64 v6, v[0:1], off
-; GFX1250-NEXT: .LBB54_4: ; %atomicrmw.phi
+; GFX1250-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
+; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%result = atomicrmw max ptr %ptr, i64 %val syncscope("one-as") monotonic
ret i64 %result
@@ -1269,36 +1287,42 @@ define i64 @flat_system_atomic_max_i64(ptr %ptr, i64 %val) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB55_2
-; GFX1250-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX1250-NEXT: flat_atomic_max_i64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT: s_cbranch_execnz .LBB55_3
+; GFX1250-NEXT: ; %bb.1: ; %Flow
+; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB55_4
+; GFX1250-NEXT: .LBB55_2: ; %atomicrmw.phi
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-NEXT: .LBB55_3: ; %atomicrmw.global
+; GFX1250-NEXT: flat_atomic_max_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-NEXT: .LBB55_2: ; %Flow
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB55_4
-; GFX1250-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT: s_cbranch_execz .LBB55_2
+; GFX1250-NEXT: .LBB55_4: ; %atomicrmw.private
+; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_max_i64 v[0:1], v[4:5], v[2:3]
-; GFX1250-NEXT: scratch_store_b64 v6, v[0:1], off
-; GFX1250-NEXT: .LBB55_4: ; %atomicrmw.phi
+; GFX1250-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
+; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%result = atomicrmw max ptr %ptr, i64 %val monotonic
ret i64 %result
@@ -1309,36 +1333,42 @@ define i64 @flat_one_as_atomic_umin_i64(ptr %ptr, i64 %val) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB56_2
-; GFX1250-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX1250-NEXT: flat_atomic_min_u64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT: s_cbranch_execnz .LBB56_3
+; GFX1250-NEXT: ; %bb.1: ; %Flow
+; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB56_4
+; GFX1250-NEXT: .LBB56_2: ; %atomicrmw.phi
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-NEXT: .LBB56_3: ; %atomicrmw.global
+; GFX1250-NEXT: flat_atomic_min_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-NEXT: .LBB56_2: ; %Flow
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB56_4
-; GFX1250-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT: s_cbranch_execz .LBB56_2
+; GFX1250-NEXT: .LBB56_4: ; %atomicrmw.private
+; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_min_u64 v[0:1], v[4:5], v[2:3]
-; GFX1250-NEXT: scratch_store_b64 v6, v[0:1], off
-; GFX1250-NEXT: .LBB56_4: ; %atomicrmw.phi
+; GFX1250-NEXT: v_cmp_le_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
+; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%result = atomicrmw umin ptr %ptr, i64 %val syncscope("one-as") monotonic
ret i64 %result
@@ -1349,36 +1379,42 @@ define i64 @flat_system_atomic_umin_i64(ptr %ptr, i64 %val) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB57_2
-; GFX1250-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX1250-NEXT: flat_atomic_min_u64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT: s_cbranch_execnz .LBB57_3
+; GFX1250-NEXT: ; %bb.1: ; %Flow
+; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB57_4
+; GFX1250-NEXT: .LBB57_2: ; %atomicrmw.phi
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-NEXT: .LBB57_3: ; %atomicrmw.global
+; GFX1250-NEXT: flat_atomic_min_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-NEXT: .LBB57_2: ; %Flow
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB57_4
-; GFX1250-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT: s_cbranch_execz .LBB57_2
+; GFX1250-NEXT: .LBB57_4: ; %atomicrmw.private
+; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_min_u64 v[0:1], v[4:5], v[2:3]
-; GFX1250-NEXT: scratch_store_b64 v6, v[0:1], off
-; GFX1250-NEXT: .LBB57_4: ; %atomicrmw.phi
+; GFX1250-NEXT: v_cmp_le_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
+; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%result = atomicrmw umin ptr %ptr, i64 %val monotonic
ret i64 %result
@@ -1389,36 +1425,42 @@ define i64 @flat_one_as_atomic_umax_i64(ptr %ptr, i64 %val) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB58_2
-; GFX1250-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX1250-NEXT: flat_atomic_max_u64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT: s_cbranch_execnz .LBB58_3
+; GFX1250-NEXT: ; %bb.1: ; %Flow
+; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB58_4
+; GFX1250-NEXT: .LBB58_2: ; %atomicrmw.phi
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-NEXT: .LBB58_3: ; %atomicrmw.global
+; GFX1250-NEXT: flat_atomic_max_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-NEXT: .LBB58_2: ; %Flow
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB58_4
-; GFX1250-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT: s_cbranch_execz .LBB58_2
+; GFX1250-NEXT: .LBB58_4: ; %atomicrmw.private
+; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_max_u64 v[0:1], v[4:5], v[2:3]
-; GFX1250-NEXT: scratch_store_b64 v6, v[0:1], off
-; GFX1250-NEXT: .LBB58_4: ; %atomicrmw.phi
+; GFX1250-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
+; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%result = atomicrmw umax ptr %ptr, i64 %val syncscope("one-as") monotonic
ret i64 %result
@@ -1429,36 +1471,42 @@ define i64 @flat_system_atomic_umax_i64(ptr %ptr, i64 %val) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
-; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5
+; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0
+; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB59_2
-; GFX1250-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX1250-NEXT: flat_atomic_max_u64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT: s_cbranch_execnz .LBB59_3
+; GFX1250-NEXT: ; %bb.1: ; %Flow
+; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB59_4
+; GFX1250-NEXT: .LBB59_2: ; %atomicrmw.phi
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-NEXT: .LBB59_3: ; %atomicrmw.global
+; GFX1250-NEXT: flat_atomic_max_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-NEXT: .LBB59_2: ; %Flow
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX1250-NEXT: s_cbranch_execz .LBB59_4
-; GFX1250-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT: s_cbranch_execz .LBB59_2
+; GFX1250-NEXT: .LBB59_4: ; %atomicrmw.private
+; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0
+; GFX1250-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
-; GFX1250-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_max_u64 v[0:1], v[4:5], v[2:3]
-; GFX1250-NEXT: scratch_store_b64 v6, v[0:1], off
-; GFX1250-NEXT: .LBB59_4: ; %atomicrmw.phi
+; GFX1250-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
+; GFX1250-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%result = atomicrmw umax ptr %ptr, i64 %val monotonic
ret i64 %result
diff --git a/llvm/test/CodeGen/AMDGPU/clmul.ll b/llvm/test/CodeGen/AMDGPU/clmul.ll
index fea2ff0681a88..130824d373c29 100644
--- a/llvm/test/CodeGen/AMDGPU/clmul.ll
+++ b/llvm/test/CodeGen/AMDGPU/clmul.ll
@@ -1776,52 +1776,51 @@ define amdgpu_kernel void @test_clmulr_i32(ptr addrspace(1) %out, ptr addrspace(
; GFX1250-NEXT: buffer_load_b64 v[0:1], off, s[8:11], null
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 s9, s3
-; GFX1250-NEXT: s_mov_b32 s13, s3
-; GFX1250-NEXT: s_mov_b32 s15, s3
; GFX1250-NEXT: s_mov_b32 s11, s3
+; GFX1250-NEXT: s_mov_b32 s13, s3
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1250-NEXT: v_readfirstlane_b32 s10, v0
-; GFX1250-NEXT: s_and_b64 s[16:17], s[2:3], 0x11111111
-; GFX1250-NEXT: s_and_b32 s8, s10, 0x22222222
-; GFX1250-NEXT: s_and_b64 s[18:19], s[2:3], 0x22222222
-; GFX1250-NEXT: s_and_b32 s12, s10, 0x11111111
-; GFX1250-NEXT: s_and_b32 s14, s2, 0x88888888
-; GFX1250-NEXT: s_and_b64 s[20:21], s[10:11], 0x44444444
-; GFX1250-NEXT: s_and_b32 s2, s2, 0x44444444
-; GFX1250-NEXT: s_and_b64 s[4:5], s[10:11], s[4:5]
-; GFX1250-NEXT: s_mul_u64 s[10:11], s[8:9], s[16:17]
-; GFX1250-NEXT: s_mul_u64 s[22:23], s[12:13], s[18:19]
-; GFX1250-NEXT: s_mul_u64 s[28:29], s[8:9], s[14:15]
-; GFX1250-NEXT: s_mul_u64 s[30:31], s[12:13], s[16:17]
-; GFX1250-NEXT: s_mul_u64 s[24:25], s[20:21], s[14:15]
-; GFX1250-NEXT: s_mul_u64 s[26:27], s[4:5], s[2:3]
-; GFX1250-NEXT: s_mul_u64 s[34:35], s[20:21], s[2:3]
-; GFX1250-NEXT: s_mul_u64 s[38:39], s[8:9], s[18:19]
-; GFX1250-NEXT: s_mul_u64 s[40:41], s[12:13], s[2:3]
-; GFX1250-NEXT: s_mul_u64 s[44:45], s[4:5], s[14:15]
-; GFX1250-NEXT: s_mul_u64 s[2:3], s[8:9], s[2:3]
-; GFX1250-NEXT: s_mul_u64 s[8:9], s[12:13], s[14:15]
-; GFX1250-NEXT: s_xor_b64 s[10:11], s[22:23], s[10:11]
+; GFX1250-NEXT: v_readfirstlane_b32 s20, v0
+; GFX1250-NEXT: s_and_b64 s[14:15], s[2:3], 0x11111111
+; GFX1250-NEXT: s_and_b32 s8, s20, 0x22222222
+; GFX1250-NEXT: s_and_b64 s[16:17], s[2:3], 0x22222222
+; GFX1250-NEXT: s_and_b32 s10, s20, 0x11111111
+; GFX1250-NEXT: s_and_b64 s[4:5], s[2:3], s[4:5]
+; GFX1250-NEXT: s_and_b32 s12, s20, 0x44444444
+; GFX1250-NEXT: s_and_b64 s[18:19], s[2:3], 0x44444444
+; GFX1250-NEXT: s_and_b32 s2, s20, 0x88888888
+; GFX1250-NEXT: s_mul_u64 s[20:21], s[8:9], s[14:15]
+; GFX1250-NEXT: s_mul_u64 s[22:23], s[10:11], s[16:17]
+; GFX1250-NEXT: s_mul_u64 s[28:29], s[8:9], s[4:5]
+; GFX1250-NEXT: s_mul_u64 s[30:31], s[10:11], s[14:15]
+; GFX1250-NEXT: s_mul_u64 s[24:25], s[12:13], s[4:5]
+; GFX1250-NEXT: s_mul_u64 s[26:27], s[2:3], s[18:19]
+; GFX1250-NEXT: s_mul_u64 s[34:35], s[12:13], s[18:19]
+; GFX1250-NEXT: s_mul_u64 s[36:37], s[2:3], s[16:17]
+; GFX1250-NEXT: s_mul_u64 s[38:39], s[8:9], s[16:17]
+; GFX1250-NEXT: s_mul_u64 s[40:41], s[10:11], s[18:19]
+; GFX1250-NEXT: s_mul_u64 s[42:43], s[12:13], s[14:15]
+; GFX1250-NEXT: s_mul_u64 s[44:45], s[2:3], s[4:5]
+; GFX1250-NEXT: s_mul_u64 s[8:9], s[8:9], s[18:19]
+; GFX1250-NEXT: s_mul_u64 s[4:5], s[10:11], s[4:5]
+; GFX1250-NEXT: s_mul_u64 s[10:11], s[12:13], s[16:17]
+; GFX1250-NEXT: s_mul_u64 s[2:3], s[2:3], s[14:15]
+; GFX1250-NEXT: s_xor_b64 s[12:13], s[22:23], s[20:21]
; GFX1250-NEXT: s_xor_b64 s[14:15], s[30:31], s[28:29]
-; GFX1250-NEXT: s_mul_u64 s[36:37], s[4:5], s[18:19]
-; GFX1250-NEXT: s_mul_u64 s[42:43], s[20:21], s[16:17]
-; GFX1250-NEXT: s_mul_u64 s[4:5], s[4:5], s[16:17]
; GFX1250-NEXT: s_xor_b64 s[16:17], s[40:41], s[38:39]
-; GFX1250-NEXT: s_xor_b64 s[2:3], s[8:9], s[2:3]
-; GFX1250-NEXT: s_xor_b64 s[8:9], s[10:11], s[24:25]
-; GFX1250-NEXT: s_xor_b64 s[10:11], s[14:15], s[34:35]
-; GFX1250-NEXT: s_mul_u64 s[12:13], s[20:21], s[18:19]
+; GFX1250-NEXT: s_xor_b64 s[4:5], s[4:5], s[8:9]
+; GFX1250-NEXT: s_xor_b64 s[8:9], s[12:13], s[24:25]
+; GFX1250-NEXT: s_xor_b64 s[12:13], s[14:15], s[34:35]
; GFX1250-NEXT: s_xor_b64 s[14:15], s[16:17], s[42:43]
+; GFX1250-NEXT: s_xor_b64 s[4:5], s[4:5], s[10:11]
; GFX1250-NEXT: s_xor_b64 s[8:9], s[8:9], s[26:27]
-; GFX1250-NEXT: s_xor_b64 s[10:11], s[10:11], s[36:37]
-; GFX1250-NEXT: s_xor_b64 s[2:3], s[2:3], s[12:13]
+; GFX1250-NEXT: s_xor_b64 s[10:11], s[12:13], s[36:37]
; GFX1250-NEXT: s_xor_b64 s[12:13], s[14:15], s[44:45]
; GFX1250-NEXT: s_and_b64 s[8:9], s[8:9], 0x2222222200000000
; GFX1250-NEXT: s_and_b64 s[10:11], s[10:11], 0x1111111100000000
; GFX1250-NEXT: s_and_b64 s[12:13], s[12:13], 0x4444444400000000
; GFX1250-NEXT: s_or_b64 s[8:9], s[10:11], s[8:9]
-; GFX1250-NEXT: s_xor_b64 s[2:3], s[2:3], s[4:5]
+; GFX1250-NEXT: s_xor_b64 s[2:3], s[4:5], s[2:3]
; GFX1250-NEXT: s_or_b64 s[4:5], s[8:9], s[12:13]
; GFX1250-NEXT: s_and_b64 s[2:3], s[2:3], 0x888888880000000
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -3248,54 +3247,53 @@ define amdgpu_kernel void @test_clmulh_i32(ptr addrspace(1) %out, ptr addrspace(
; GFX1250-NEXT: buffer_load_b64 v[0:1], off, s[8:11], null
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_mov_b32 s9, s3
-; GFX1250-NEXT: s_mov_b32 s13, s3
-; GFX1250-NEXT: s_mov_b32 s15, s3
; GFX1250-NEXT: s_mov_b32 s11, s3
+; GFX1250-NEXT: s_mov_b32 s13, s3
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1250-NEXT: v_readfirstlane_b32 s10, v0
-; GFX1250-NEXT: s_and_b64 s[16:17], s[2:3], 0x11111111
-; GFX1250-NEXT: s_and_b32 s8, s10, 0x22222222
-; GFX1250-NEXT: s_and_b64 s[18:19], s[2:3], 0x22222222
-; GFX1250-NEXT: s_and_b32 s12, s10, 0x11111111
-; GFX1250-NEXT: s_and_b32 s14, s2, 0x88888888
-; GFX1250-NEXT: s_and_b64 s[20:21], s[10:11], 0x44444444
-; GFX1250-NEXT: s_and_b32 s2, s2, 0x44444444
-; GFX1250-NEXT: s_and_b64 s[4:5], s[10:11], s[4:5]
-; GFX1250-NEXT: s_mul_u64 s[10:11], s[8:9], s[16:17]
-; GFX1250-NEXT: s_mul_u64 s[22:23], s[12:13], s[18:19]
-; GFX1250-NEXT: s_mul_u64 s[28:29], s[8:9], s[14:15]
-; GFX1250-NEXT: s_mul_u64 s[30:31], s[12:13], s[16:17]
-; GFX1250-NEXT: s_mul_u64 s[24:25], s[20:21], s[14:15]
-; GFX1250-NEXT: s_mul_u64 s[26:27], s[4:5], s[2:3]
-; GFX1250-NEXT: s_mul_u64 s[34:35], s[20:21], s[2:3]
-; GFX1250-NEXT: s_mul_u64 s[38:39], s[8:9], s[18:19]
-; GFX1250-NEXT: s_mul_u64 s[40:41], s[12:13], s[2:3]
-; GFX1250-NEXT: s_mul_u64 s[44:45], s[4:5], s[14:15]
-; GFX1250-NEXT: s_mul_u64 s[2:3], s[8:9], s[2:3]
-; GFX1250-NEXT: s_mul_u64 s[8:9], s[12:13], s[14:15]
-; GFX1250-NEXT: s_xor_b64 s[10:11], s[22:23], s[10:11]
+; GFX1250-NEXT: v_readfirstlane_b32 s20, v0
+; GFX1250-NEXT: s_and_b64 s[14:15], s[2:3], 0x11111111
+; GFX1250-NEXT: s_and_b32 s8, s20, 0x22222222
+; GFX1250-NEXT: s_and_b64 s[16:17], s[2:3], 0x22222222
+; GFX1250-NEXT: s_and_b32 s10, s20, 0x11111111
+; GFX1250-NEXT: s_and_b64 s[4:5], s[2:3], s[4:5]
+; GFX1250-NEXT: s_and_b32 s12, s20, 0x44444444
+; GFX1250-NEXT: s_and_b64 s[18:19], s[2:3], 0x44444444
+; GFX1250-NEXT: s_and_b32 s2, s20, 0x88888888
+; GFX1250-NEXT: s_mul_u64 s[20:21], s[8:9], s[14:15]
+; GFX1250-NEXT: s_mul_u64 s[22:23], s[10:11], s[16:17]
+; GFX1250-NEXT: s_mul_u64 s[28:29], s[8:9], s[4:5]
+; GFX1250-NEXT: s_mul_u64 s[30:31], s[10:11], s[14:15]
+; GFX1250-NEXT: s_mul_u64 s[24:25], s[12:13], s[4:5]
+; GFX1250-NEXT: s_mul_u64 s[26:27], s[2:3], s[18:19]
+; GFX1250-NEXT: s_mul_u64 s[34:35], s[12:13], s[18:19]
+; GFX1250-NEXT: s_mul_u64 s[36:37], s[2:3], s[16:17]
+; GFX1250-NEXT: s_mul_u64 s[38:39], s[8:9], s[16:17]
+; GFX1250-NEXT: s_mul_u64 s[40:41], s[10:11], s[18:19]
+; GFX1250-NEXT: s_mul_u64 s[42:43], s[12:13], s[14:15]
+; GFX1250-NEXT: s_mul_u64 s[44:45], s[2:3], s[4:5]
+; GFX1250-NEXT: s_mul_u64 s[8:9], s[8:9], s[18:19]
+; GFX1250-NEXT: s_mul_u64 s[4:5], s[10:11], s[4:5]
+; GFX1250-NEXT: s_mul_u64 s[10:11], s[12:13], s[16:17]
+; GFX1250-NEXT: s_mul_u64 s[2:3], s[2:3], s[14:15]
+; GFX1250-NEXT: s_xor_b64 s[12:13], s[22:23], s[20:21]
; GFX1250-NEXT: s_xor_b64 s[14:15], s[30:31], s[28:29]
-; GFX1250-NEXT: s_mul_u64 s[36:37], s[4:5], s[18:19]
-; GFX1250-NEXT: s_mul_u64 s[42:43], s[20:21], s[16:17]
-; GFX1250-NEXT: s_mul_u64 s[4:5], s[4:5], s[16:17]
; GFX1250-NEXT: s_xor_b64 s[16:17], s[40:41], s[38:39]
-; GFX1250-NEXT: s_xor_b64 s[2:3], s[8:9], s[2:3]
-; GFX1250-NEXT: s_xor_b64 s[8:9], s[10:11], s[24:25]
-; GFX1250-NEXT: s_xor_b64 s[10:11], s[14:15], s[34:35]
-; GFX1250-NEXT: s_mul_u64 s[12:13], s[20:21], s[18:19]
+; GFX1250-NEXT: s_xor_b64 s[4:5], s[4:5], s[8:9]
+; GFX1250-NEXT: s_xor_b64 s[8:9], s[12:13], s[24:25]
+; GFX1250-NEXT: s_xor_b64 s[12:13], s[14:15], s[34:35]
; GFX1250-NEXT: s_xor_b64 s[14:15], s[16:17], s[42:43]
+; GFX1250-NEXT: s_xor_b64 s[4:5], s[4:5], s[10:11]
; GFX1250-NEXT: s_xor_b64 s[8:9], s[8:9], s[26:27]
-; GFX1250-NEXT: s_xor_b64 s[10:11], s[10:11], s[36:37]
-; GFX1250-NEXT: s_xor_b64 s[2:3], s[2:3], s[12:13]
+; GFX1250-NEXT: s_xor_b64 s[10:11], s[12:13], s[36:37]
; GFX1250-NEXT: s_xor_b64 s[12:13], s[14:15], s[44:45]
; GFX1250-NEXT: s_and_b64 s[8:9], s[8:9], 0x2222222200000000
; GFX1250-NEXT: s_and_b64 s[10:11], s[10:11], 0x1111111100000000
; GFX1250-NEXT: s_and_b64 s[12:13], s[12:13], 0x4444444400000000
; GFX1250-NEXT: s_or_b64 s[8:9], s[10:11], s[8:9]
-; GFX1250-NEXT: s_xor_b64 s[2:3], s[2:3], s[4:5]
+; GFX1250-NEXT: s_xor_b64 s[2:3], s[4:5], s[2:3]
; GFX1250-NEXT: s_or_b64 s[4:5], s[8:9], s[12:13]
-; GFX1250-NEXT: s_and_b64 s[2:3], s[2:3], 0x888888800000000
+; GFX1250-NEXT: s_and_b64 s[2:3], s[2:3], 0x8888888800000000
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_or_b64 s[2:3], s[4:5], s[2:3]
; GFX1250-NEXT: s_mov_b32 s4, s0
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index 678d4ffc7f094..644a1296623c8 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -6013,7 +6013,8 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_max_i64 v[2:3], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -6058,7 +6059,8 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_max_i64 v[2:3], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -6197,7 +6199,8 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_max_i64 v[2:3], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -6245,7 +6248,8 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_max_i64 v[2:3], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -6384,7 +6388,8 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_max_i64 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
; GFX1250-SDAG-NEXT: s_endpgm
;
@@ -6421,7 +6426,8 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_max_i64 v[0:1], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX1250-GISEL-NEXT: s_endpgm
;
@@ -6536,7 +6542,8 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_max_i64 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
; GFX1250-SDAG-NEXT: s_endpgm
;
@@ -6576,7 +6583,8 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_max_i64 v[0:1], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX1250-GISEL-NEXT: s_endpgm
;
@@ -6843,7 +6851,8 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_min_i64 v[2:3], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_le_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -6888,7 +6897,8 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_min_i64 v[2:3], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -7027,7 +7037,8 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_min_i64 v[2:3], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_le_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -7075,7 +7086,8 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_min_i64 v[2:3], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -7214,7 +7226,8 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_min_i64 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_le_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
; GFX1250-SDAG-NEXT: s_endpgm
;
@@ -7251,7 +7264,8 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_min_i64 v[0:1], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX1250-GISEL-NEXT: s_endpgm
;
@@ -7366,7 +7380,8 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_min_i64 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_le_i64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
; GFX1250-SDAG-NEXT: s_endpgm
;
@@ -7406,7 +7421,8 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_min_i64 v[0:1], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX1250-GISEL-NEXT: s_endpgm
;
@@ -7673,7 +7689,8 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_max_u64 v[2:3], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -7718,7 +7735,8 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_max_u64 v[2:3], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -7857,7 +7875,8 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_max_u64 v[2:3], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -7905,7 +7924,8 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_max_u64 v[2:3], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -8044,7 +8064,8 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_max_u64 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
; GFX1250-SDAG-NEXT: s_endpgm
;
@@ -8081,7 +8102,8 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_max_u64 v[0:1], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX1250-GISEL-NEXT: s_endpgm
;
@@ -8196,7 +8218,8 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_max_u64 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
; GFX1250-SDAG-NEXT: s_endpgm
;
@@ -8236,7 +8259,8 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_max_u64 v[0:1], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX1250-GISEL-NEXT: s_endpgm
;
@@ -8503,7 +8527,8 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_min_u64 v[2:3], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_le_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -8548,7 +8573,8 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_min_u64 v[2:3], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -8687,7 +8713,8 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_min_u64 v[2:3], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_le_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -8735,7 +8762,8 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_min_u64 v[2:3], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -8874,7 +8902,8 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_min_u64 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_le_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
; GFX1250-SDAG-NEXT: s_endpgm
;
@@ -8911,7 +8940,8 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_min_u64 v[0:1], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX1250-GISEL-NEXT: s_endpgm
;
@@ -9026,7 +9056,8 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_min_u64 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_cmp_le_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0
; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off
; GFX1250-SDAG-NEXT: s_endpgm
;
@@ -9066,7 +9097,8 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: v_min_u64 v[0:1], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX1250-GISEL-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index 9325595c882f2..0491d98e54a5e 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -6946,19 +6946,25 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) {
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 1, v[0:1]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_mul_u64_e32 v[4:5], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[4:5], v0, v2, 0
+; GFX1250-SDAG-NEXT: v_mad_u32 v5, v0, v3, v5
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_mad_u32 v5, v1, v2, v5
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_mul_u64_e32 v[2:3], v[0:1], v[2:3]
-; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[6:7], v2, v4, v[2:3]
+; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[6:7], v0, v2, 0
+; GFX1250-SDAG-NEXT: v_mad_u32 v0, v0, v3, v7
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_mad_u32 v7, v1, v2, v0
+; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[2:3], v6, v4, v[6:7]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_mad_u32 v0, v3, v4, v7
-; GFX1250-SDAG-NEXT: v_mad_u32 v7, v2, v5, v0
+; GFX1250-SDAG-NEXT: v_mad_u32 v0, v7, v4, v3
+; GFX1250-SDAG-NEXT: v_mad_u32 v3, v6, v5, v0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[0:1], v6, v2, v[6:7]
-; GFX1250-SDAG-NEXT: v_mad_u32 v1, v7, v2, v1
+; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[0:1], v2, v6, v[2:3]
+; GFX1250-SDAG-NEXT: v_mad_u32 v1, v3, v6, v1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_mad_u32 v1, v6, v3, v1
+; GFX1250-SDAG-NEXT: v_mad_u32 v1, v2, v7, v1
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-GISEL-FAKE16-LABEL: clpeak_imad_pat_i64:
@@ -6966,16 +6972,37 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) {
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], 1, v[0:1]
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_mul_u64_e32 v[4:5], v[0:1], v[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_hi_u32 v5, v0, v2
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v6, v0, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v7, v1, v2
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v4, v0, v2
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add3_u32 v5, v6, v7, v5
; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_mul_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_hi_u32 v6, v0, v2
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v7, v0, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[4:5]
-; GFX1250-GISEL-FAKE16-NEXT: v_mul_u64_e32 v[2:3], v[0:1], v[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-FAKE16-NEXT: v_add3_u32 v1, v7, v1, v6
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_hi_u32 v4, v0, v2
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v3, v0, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v5, v1, v2
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v2, v0, v2
; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], 1, v[0:1]
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_mul_u64_e32 v[0:1], v[2:3], v[0:1]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-FAKE16-NEXT: v_add3_u32 v3, v3, v5, v4
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_hi_u32 v4, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v1, v2, v1
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v3, v3, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v0, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v3, v4
; GFX1250-GISEL-FAKE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-GISEL-REAL16-LABEL: clpeak_imad_pat_i64:
@@ -6983,16 +7010,37 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) {
; GFX1250-GISEL-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u64_e32 v[0:1], 1, v[0:1]
-; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-REAL16-NEXT: v_mul_u64_e32 v[4:5], v[0:1], v[2:3]
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32 v5, v0, v2
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v6, v0, v3
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v7, v1, v2
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v4, v0, v2
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT: v_add3_u32 v5, v6, v7, v5
; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
-; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-REAL16-NEXT: v_mul_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32 v6, v0, v2
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v7, v0, v3
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[4:5]
-; GFX1250-GISEL-REAL16-NEXT: v_mul_u64_e32 v[2:3], v[0:1], v[2:3]
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-REAL16-NEXT: v_add3_u32 v1, v7, v1, v6
+; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32 v4, v0, v2
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v3, v0, v3
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v5, v1, v2
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v2, v0, v2
; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u64_e32 v[0:1], 1, v[0:1]
-; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-REAL16-NEXT: v_mul_u64_e32 v[0:1], v[2:3], v[0:1]
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-REAL16-NEXT: v_add3_u32 v3, v3, v5, v4
+; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32 v4, v2, v0
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v1, v2, v1
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v3, v3, v0
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v0, v2, v0
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-GISEL-REAL16-NEXT: v_add3_u32 v1, v1, v3, v4
; GFX1250-GISEL-REAL16-NEXT: s_set_pc_i64 s[30:31]
; GFX1250-GISEL-LABEL: clpeak_imad_pat_i64:
; GFX1250-GISEL: ; %bb.0: ; %entry
@@ -7897,32 +7945,44 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) {
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 1, v[0:1]
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[2:3]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-SDAG-NEXT: v_mul_u64_e32 v[8:9], v[0:1], v[4:5]
-; GFX1250-SDAG-NEXT: v_mul_u64_e32 v[10:11], v[2:3], v[6:7]
+; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[8:9], v0, v4, 0
+; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[10:11], v2, v6, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_mad_u32 v9, v0, v5, v9
+; GFX1250-SDAG-NEXT: v_mad_u32 v11, v2, v7, v11
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_mad_u32 v9, v1, v4, v9
+; GFX1250-SDAG-NEXT: v_mad_u32 v11, v3, v6, v11
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[8:9], v[0:1]
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], v[10:11], v[2:3]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-SDAG-NEXT: v_mul_u64_e32 v[4:5], v[0:1], v[4:5]
-; GFX1250-SDAG-NEXT: v_mul_u64_e32 v[6:7], v[2:3], v[6:7]
+; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[12:13], v0, v4, 0
+; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[14:15], v2, v6, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_mad_u32 v0, v0, v5, v13
+; GFX1250-SDAG-NEXT: v_mad_u32 v2, v2, v7, v15
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[12:13], v4, v8, v[4:5]
-; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[14:15], v6, v10, v[6:7]
+; GFX1250-SDAG-NEXT: v_mad_u32 v13, v1, v4, v0
+; GFX1250-SDAG-NEXT: v_mad_u32 v15, v3, v6, v2
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-SDAG-NEXT: v_mad_u32 v0, v5, v8, v13
-; GFX1250-SDAG-NEXT: v_mad_u32 v1, v7, v10, v15
+; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[4:5], v12, v8, v[12:13]
+; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[6:7], v14, v10, v[14:15]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-SDAG-NEXT: v_mad_u32 v13, v4, v9, v0
-; GFX1250-SDAG-NEXT: v_mad_u32 v15, v6, v11, v1
+; GFX1250-SDAG-NEXT: v_mad_u32 v0, v13, v8, v5
+; GFX1250-SDAG-NEXT: v_mad_u32 v1, v15, v10, v7
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[0:1], v12, v4, v[12:13]
-; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[2:3], v14, v6, v[14:15]
+; GFX1250-SDAG-NEXT: v_mad_u32 v5, v12, v9, v0
+; GFX1250-SDAG-NEXT: v_mad_u32 v7, v14, v11, v1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-SDAG-NEXT: v_mad_u32 v1, v13, v4, v1
-; GFX1250-SDAG-NEXT: v_mad_u32 v3, v15, v6, v3
+; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[0:1], v4, v12, v[4:5]
+; GFX1250-SDAG-NEXT: v_mad_nc_u64_u32 v[2:3], v6, v14, v[6:7]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-SDAG-NEXT: v_mad_u32 v1, v12, v5, v1
-; GFX1250-SDAG-NEXT: v_mad_u32 v3, v14, v7, v3
+; GFX1250-SDAG-NEXT: v_mad_u32 v1, v5, v12, v1
+; GFX1250-SDAG-NEXT: v_mad_u32 v3, v7, v14, v3
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_mad_u32 v1, v4, v13, v1
+; GFX1250-SDAG-NEXT: v_mad_u32 v3, v6, v15, v3
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-GISEL-FAKE16-LABEL: clpeak_imad_pat_v2i64:
@@ -7931,25 +7991,60 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) {
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], 1, v[0:1]
; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[2:3]
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-FAKE16-NEXT: v_mul_u64_e32 v[8:9], v[0:1], v[4:5]
-; GFX1250-GISEL-FAKE16-NEXT: v_mul_u64_e32 v[10:11], v[2:3], v[6:7]
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_hi_u32 v9, v0, v4
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v11, v0, v5
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v12, v1, v4
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_hi_u32 v13, v2, v6
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v14, v2, v7
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v15, v3, v6
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v8, v0, v4
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v10, v2, v6
+; GFX1250-GISEL-FAKE16-NEXT: v_add3_u32 v9, v11, v12, v9
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-FAKE16-NEXT: v_add3_u32 v11, v14, v15, v13
; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], v[8:9], v[0:1]
-; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u64_e32 v[2:3], v[10:11], v[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-FAKE16-NEXT: v_mul_u64_e32 v[0:1], v[0:1], v[4:5]
-; GFX1250-GISEL-FAKE16-NEXT: v_mul_u64_e32 v[2:3], v[2:3], v[6:7]
+; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u64_e32 v[2:3], v[10:11], v[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_hi_u32 v12, v0, v4
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v13, v0, v5
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v1, v1, v4
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_hi_u32 v14, v2, v6
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v15, v2, v7
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v3, v3, v6
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v0, v0, v4
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], 1, v[8:9]
; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u64_e32 v[6:7], 1, v[10:11]
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-FAKE16-NEXT: v_mul_u64_e32 v[4:5], v[0:1], v[4:5]
-; GFX1250-GISEL-FAKE16-NEXT: v_mul_u64_e32 v[6:7], v[2:3], v[6:7]
+; GFX1250-GISEL-FAKE16-NEXT: v_add3_u32 v1, v13, v1, v12
+; GFX1250-GISEL-FAKE16-NEXT: v_add3_u32 v3, v15, v3, v14
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v8, v0, v4
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_hi_u32 v9, v0, v4
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v5, v0, v5
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_hi_u32 v10, v2, v6
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v7, v2, v7
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v4, v1, v4
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v11, v3, v6
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v6, v2, v6
; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], 1, v[0:1]
; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[2:3]
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-FAKE16-NEXT: v_mul_u64_e32 v[0:1], v[4:5], v[0:1]
-; GFX1250-GISEL-FAKE16-NEXT: v_mul_u64_e32 v[2:3], v[6:7], v[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: v_add3_u32 v4, v5, v4, v9
+; GFX1250-GISEL-FAKE16-NEXT: v_add3_u32 v5, v7, v11, v10
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_hi_u32 v12, v8, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v1, v8, v1
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_hi_u32 v7, v6, v2
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v3, v6, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v4, v4, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v5, v5, v2
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v0, v8, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v2, v6, v2
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v4, v12
+; GFX1250-GISEL-FAKE16-NEXT: v_add3_u32 v3, v3, v5, v7
; GFX1250-GISEL-FAKE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-GISEL-REAL16-LABEL: clpeak_imad_pat_v2i64:
@@ -7958,25 +8053,60 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) {
; GFX1250-GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u64_e32 v[0:1], 1, v[0:1]
; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[2:3]
-; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-REAL16-NEXT: v_mul_u64_e32 v[8:9], v[0:1], v[4:5]
-; GFX1250-GISEL-REAL16-NEXT: v_mul_u64_e32 v[10:11], v[2:3], v[6:7]
-; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32 v9, v0, v4
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v11, v0, v5
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v12, v1, v4
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32 v13, v2, v6
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v14, v2, v7
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v15, v3, v6
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v8, v0, v4
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v10, v2, v6
+; GFX1250-GISEL-REAL16-NEXT: v_add3_u32 v9, v11, v12, v9
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-REAL16-NEXT: v_add3_u32 v11, v14, v15, v13
; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u64_e32 v[0:1], v[8:9], v[0:1]
-; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u64_e32 v[2:3], v[10:11], v[2:3]
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-REAL16-NEXT: v_mul_u64_e32 v[0:1], v[0:1], v[4:5]
-; GFX1250-GISEL-REAL16-NEXT: v_mul_u64_e32 v[2:3], v[2:3], v[6:7]
+; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u64_e32 v[2:3], v[10:11], v[2:3]
+; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32 v12, v0, v4
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v13, v0, v5
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v1, v1, v4
+; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32 v14, v2, v6
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v15, v2, v7
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v3, v3, v6
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v0, v0, v4
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u64_e32 v[4:5], 1, v[8:9]
; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u64_e32 v[6:7], 1, v[10:11]
-; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-REAL16-NEXT: v_mul_u64_e32 v[4:5], v[0:1], v[4:5]
-; GFX1250-GISEL-REAL16-NEXT: v_mul_u64_e32 v[6:7], v[2:3], v[6:7]
+; GFX1250-GISEL-REAL16-NEXT: v_add3_u32 v1, v13, v1, v12
+; GFX1250-GISEL-REAL16-NEXT: v_add3_u32 v3, v15, v3, v14
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v8, v0, v4
+; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32 v9, v0, v4
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v5, v0, v5
+; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32 v10, v2, v6
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v7, v2, v7
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v4, v1, v4
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v11, v3, v6
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v6, v2, v6
; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u64_e32 v[0:1], 1, v[0:1]
; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[2:3]
-; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-REAL16-NEXT: v_mul_u64_e32 v[0:1], v[4:5], v[0:1]
-; GFX1250-GISEL-REAL16-NEXT: v_mul_u64_e32 v[2:3], v[6:7], v[2:3]
+; GFX1250-GISEL-REAL16-NEXT: v_add3_u32 v4, v5, v4, v9
+; GFX1250-GISEL-REAL16-NEXT: v_add3_u32 v5, v7, v11, v10
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32 v12, v8, v0
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v1, v8, v1
+; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32 v7, v6, v2
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v3, v6, v3
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v4, v4, v0
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v5, v5, v2
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v0, v8, v0
+; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v2, v6, v2
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-GISEL-REAL16-NEXT: v_add3_u32 v1, v1, v4, v12
+; GFX1250-GISEL-REAL16-NEXT: v_add3_u32 v3, v3, v5, v7
; GFX1250-GISEL-REAL16-NEXT: s_set_pc_i64 s[30:31]
; GFX1250-GISEL-LABEL: clpeak_imad_pat_v2i64:
; GFX1250-GISEL: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
index 4e24b02a9b15e..b882d1a07dc50 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
@@ -426,32 +426,29 @@ define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v9, 0
-; GFX1250-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_ashrrev_i32 v12, 31, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_dual_ashrrev_i32 v7, 31, v6 :: v_dual_mov_b32 v1, v9
-; GFX1250-NEXT: v_mov_b32_e32 v21, v9
-; GFX1250-NEXT: v_mul_u64_e32 v[10:11], v[0:1], v[8:9]
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[6:7], v0, v1, 0
+; GFX1250-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_ashrrev_i32 v16, 31, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_mad_nc_i64_i32 v[14:15], v1, v16, 0
+; GFX1250-NEXT: v_dual_mov_b32 v8, v7 :: v_dual_ashrrev_i32 v7, 31, v1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_dual_mov_b32 v13, v12 :: v_dual_mov_b32 v8, v11
-; GFX1250-NEXT: v_mul_u64_e32 v[14:15], v[6:7], v[12:13]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_mad_nc_u64_u32 v[16:17], v12, v6, v[8:9]
-; GFX1250-NEXT: v_mov_b32_e32 v8, v16
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_mad_nc_u64_u32 v[18:19], v0, v7, v[8:9]
-; GFX1250-NEXT: v_dual_mov_b32 v8, v17 :: v_dual_mov_b32 v20, v19
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[8:9], v[8:9], v[20:21]
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[10:11], v16, v1, v[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v8, v11 :: v_dual_mov_b32 v11, v9
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[10:11], v0, v7, v[10:11]
+; GFX1250-NEXT: v_mov_b32_e32 v13, v9
; GFX1250-NEXT: v_mad_nc_i64_i32 v[0:1], v7, v0, v[14:15]
-; GFX1250-NEXT: v_mad_nc_u64_u32 v[6:7], v12, v7, v[8:9]
+; GFX1250-NEXT: v_mov_b32_e32 v12, v11
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_nc_u64_e32 v[8:9], v[8:9], v[12:13]
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[8:9], v16, v7, v[8:9]
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[6:7], v[6:7], v[0:1]
-; GFX1250-NEXT: v_add_co_u32 v0, vcc_lo, v10, v2
-; GFX1250-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v18, v3, vcc_lo
-; GFX1250-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v6, v4, vcc_lo
+; GFX1250-NEXT: v_add_nc_u64_e32 v[8:9], v[8:9], v[0:1]
+; GFX1250-NEXT: v_add_co_u32 v0, vcc_lo, v6, v2
+; GFX1250-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v10, v3, vcc_lo
+; GFX1250-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v8, v4, vcc_lo
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1250-NEXT: v_add_co_ci_u32_e64 v3, null, v7, v5, vcc_lo
+; GFX1250-NEXT: v_add_co_ci_u32_e64 v3, null, v9, v5, vcc_lo
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%sext0 = sext i32 %arg0 to i128
%sext1 = sext i32 %arg1 to i128
@@ -1343,10 +1340,7 @@ define i64 @mad_i64_i32_thrice(i32 %arg0, i32 %arg1, i64 %arg2, i64 %arg3, i64 %
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_ashrrev_i32 v1, 31, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX1250-NEXT: v_mul_u64_e32 v[0:1], v[0:1], v[8:9]
+; GFX1250-NEXT: v_mad_nc_i64_i32 v[0:1], v0, v1, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], v[0:1], v[2:3]
; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], v[0:1], v[4:5]
@@ -1445,10 +1439,7 @@ define i64 @mad_i64_i32_secondary_use(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_ashrrev_i32 v1, 31, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX1250-NEXT: v_mul_u64_e32 v[0:1], v[0:1], v[4:5]
+; GFX1250-NEXT: v_mad_nc_i64_i32 v[0:1], v0, v1, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], v[0:1], v[2:3]
; GFX1250-NEXT: v_xor_b32_e32 v0, v2, v0
@@ -1842,13 +1833,11 @@ define i64 @lshr_mad_i64_4(i32 %arg0, i64 %arg1) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_u64_e32 v[2:3], v[2:3], v[0:1]
-; GFX1250-NEXT: v_mov_b32_e32 v0, v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], 0xfffffc88, v3, v[0:1]
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[4:5], v1, v0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mad_u32 v0, v2, v0, v5
+; GFX1250-NEXT: v_mov_b32_e32 v5, 0
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], 0xfffffc88, v0, v[4:5]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ext = zext i32 %arg0 to i64
%mul1 = mul i64 %arg1, %ext
diff --git a/llvm/test/CodeGen/AMDGPU/max.ll b/llvm/test/CodeGen/AMDGPU/max.ll
index 84ef83d47a8db..5354bb0471c6f 100644
--- a/llvm/test/CodeGen/AMDGPU/max.ll
+++ b/llvm/test/CodeGen/AMDGPU/max.ll
@@ -1137,7 +1137,12 @@ define amdgpu_kernel void @test_umax_ugt_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_max_u64 v[0:1], s[2:3], s[6:7]
+; GFX1250-NEXT: v_cmp_gt_u64_e64 s4, s[2:3], s[6:7]
+; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
+; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_mov_b32_e32 v1, s3
; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
;
@@ -1191,7 +1196,12 @@ define amdgpu_kernel void @test_umax_uge_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_max_u64 v[0:1], s[2:3], s[6:7]
+; GFX1250-NEXT: v_cmp_ge_u64_e64 s4, s[2:3], s[6:7]
+; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
+; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_mov_b32_e32 v1, s3
; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
;
@@ -1245,7 +1255,12 @@ define amdgpu_kernel void @test_imax_sgt_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_max_i64 v[0:1], s[2:3], s[6:7]
+; GFX1250-NEXT: v_cmp_gt_i64_e64 s4, s[2:3], s[6:7]
+; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
+; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_mov_b32_e32 v1, s3
; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
;
@@ -1299,7 +1314,12 @@ define amdgpu_kernel void @test_imax_sge_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_max_i64 v[0:1], s[2:3], s[6:7]
+; GFX1250-NEXT: v_cmp_ge_i64_e64 s4, s[2:3], s[6:7]
+; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
+; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_mov_b32_e32 v1, s3
; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index 7e241bf2dbdb6..4f067bade12fb 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -4390,7 +4390,12 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_min_u64 v[0:1], s[2:3], s[6:7]
+; GFX1250-NEXT: v_cmp_lt_u64_e64 s4, s[2:3], s[6:7]
+; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
+; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_mov_b32_e32 v1, s3
; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
%tmp = icmp ult i64 %a, %b
@@ -4515,7 +4520,12 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_min_u64 v[0:1], s[2:3], s[6:7]
+; GFX1250-NEXT: v_cmp_le_u64_e64 s4, s[2:3], s[6:7]
+; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
+; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_mov_b32_e32 v1, s3
; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
%tmp = icmp ule i64 %a, %b
@@ -4640,7 +4650,12 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_min_i64 v[0:1], s[2:3], s[6:7]
+; GFX1250-NEXT: v_cmp_lt_i64_e64 s4, s[2:3], s[6:7]
+; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
+; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_mov_b32_e32 v1, s3
; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
%tmp = icmp slt i64 %a, %b
@@ -4765,7 +4780,12 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_min_i64 v[0:1], s[2:3], s[6:7]
+; GFX1250-NEXT: v_cmp_le_i64_e64 s4, s[2:3], s[6:7]
+; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
+; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_mov_b32_e32 v1, s3
; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
%tmp = icmp sle i64 %a, %b
diff --git a/llvm/test/CodeGen/AMDGPU/mul.ll b/llvm/test/CodeGen/AMDGPU/mul.ll
index cb18594654611..8e45e51c333f7 100644
--- a/llvm/test/CodeGen/AMDGPU/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul.ll
@@ -1139,9 +1139,8 @@ define amdgpu_kernel void @v_mul64_sext_c(ptr addrspace(1) %out, ptr addrspace(1
; GFX1250-NEXT: buffer_load_b32 v0, off, s[8:11], null
; GFX1250-NEXT: s_mov_b32 s5, s1
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_u64_e32 v[0:1], 0x50, v[0:1]
+; GFX1250-NEXT: v_mul_hi_i32 v1, 0x50, v0
+; GFX1250-NEXT: v_mul_lo_u32 v0, 0x50, v0
; GFX1250-NEXT: buffer_store_b64 v[0:1], off, s[4:7], null
; GFX1250-NEXT: s_endpgm
;
@@ -1315,7 +1314,6 @@ define amdgpu_kernel void @v_mul64_zext_c(ptr addrspace(1) %out, ptr addrspace(1
; GFX1250-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-NEXT: s_mov_b32 s10, s6
; GFX1250-NEXT: s_mov_b32 s11, s7
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_mov_b32 s8, s2
; GFX1250-NEXT: s_mov_b32 s9, s3
@@ -1323,7 +1321,8 @@ define amdgpu_kernel void @v_mul64_zext_c(ptr addrspace(1) %out, ptr addrspace(1
; GFX1250-NEXT: buffer_load_b32 v0, off, s[8:11], null
; GFX1250-NEXT: s_mov_b32 s5, s1
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mul_u64_e32 v[0:1], 0x50, v[0:1]
+; GFX1250-NEXT: v_mul_hi_u32 v1, 0x50, v0
+; GFX1250-NEXT: v_mul_lo_u32 v0, 0x50, v0
; GFX1250-NEXT: buffer_store_b64 v[0:1], off, s[4:7], null
; GFX1250-NEXT: s_endpgm
;
@@ -1501,9 +1500,8 @@ define amdgpu_kernel void @v_mul64_sext_inline_imm(ptr addrspace(1) %out, ptr ad
; GFX1250-NEXT: buffer_load_b32 v0, off, s[8:11], null
; GFX1250-NEXT: s_mov_b32 s5, s1
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_u64_e32 v[0:1], 9, v[0:1]
+; GFX1250-NEXT: v_mul_hi_i32 v1, 9, v0
+; GFX1250-NEXT: v_mul_lo_u32 v0, 9, v0
; GFX1250-NEXT: buffer_store_b64 v[0:1], off, s[4:7], null
; GFX1250-NEXT: s_endpgm
;
@@ -2557,19 +2555,25 @@ define amdgpu_kernel void @v_mul_i64(ptr addrspace(1) %out, ptr addrspace(1) %ap
; GFX1250-NEXT: s_load_b64 s[8:9], s[4:5], 0x34 nv
; GFX1250-NEXT: s_mov_b32 s6, -1
; GFX1250-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1250-NEXT: s_mov_b32 s14, s6
-; GFX1250-NEXT: s_mov_b32 s15, s7
; GFX1250-NEXT: s_mov_b32 s10, s6
; GFX1250-NEXT: s_mov_b32 s11, s7
+; GFX1250-NEXT: s_mov_b32 s14, s6
+; GFX1250-NEXT: s_mov_b32 s15, s7
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_mov_b32 s12, s2
; GFX1250-NEXT: s_mov_b32 s13, s3
-; GFX1250-NEXT: buffer_load_b64 v[0:1], off, s[12:15], null
-; GFX1250-NEXT: buffer_load_b64 v[2:3], off, s[8:11], null
+; GFX1250-NEXT: buffer_load_b64 v[0:1], off, s[8:11], null
+; GFX1250-NEXT: buffer_load_b64 v[2:3], off, s[12:15], null
; GFX1250-NEXT: s_mov_b32 s4, s0
; GFX1250-NEXT: s_mov_b32 s5, s1
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_mul_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT: v_mul_lo_u32 v3, v0, v3
+; GFX1250-NEXT: v_mul_lo_u32 v1, v1, v2
+; GFX1250-NEXT: v_mul_hi_u32 v4, v0, v2
+; GFX1250-NEXT: v_mul_lo_u32 v0, v0, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_nc_u32_e32 v1, v3, v1
+; GFX1250-NEXT: v_add_nc_u32_e32 v1, v1, v4
; GFX1250-NEXT: buffer_store_b64 v[0:1], off, s[4:7], null
; GFX1250-NEXT: s_endpgm
;
@@ -3653,28 +3657,27 @@ define amdgpu_kernel void @s_mul_i128(ptr addrspace(1) %out, [8 x i32], i128 %a,
; GFX1250-NEXT: s_mov_b32 s2, s8
; GFX1250-NEXT: s_and_b64 s[4:5], s[12:13], s[4:5]
; GFX1250-NEXT: s_mov_b32 s6, s13
-; GFX1250-NEXT: s_mul_u64 s[10:11], s[10:11], s[12:13]
-; GFX1250-NEXT: s_mul_u64 s[12:13], s[4:5], s[2:3]
+; GFX1250-NEXT: s_mul_u64 s[22:23], s[4:5], s[2:3]
+; GFX1250-NEXT: s_mul_u64 s[24:25], s[6:7], s[2:3]
+; GFX1250-NEXT: s_mov_b32 s2, s23
; GFX1250-NEXT: s_mov_b32 s16, s9
-; GFX1250-NEXT: s_mul_u64 s[8:9], s[8:9], s[14:15]
-; GFX1250-NEXT: s_mul_u64 s[14:15], s[6:7], s[2:3]
-; GFX1250-NEXT: s_mov_b32 s2, s13
+; GFX1250-NEXT: s_mul_u64 s[10:11], s[10:11], s[12:13]
+; GFX1250-NEXT: s_add_nc_u64 s[12:13], s[24:25], s[2:3]
; GFX1250-NEXT: s_mul_u64 s[4:5], s[4:5], s[16:17]
-; GFX1250-NEXT: s_add_nc_u64 s[14:15], s[14:15], s[2:3]
-; GFX1250-NEXT: s_mul_u64 s[6:7], s[6:7], s[16:17]
-; GFX1250-NEXT: s_mov_b32 s2, s15
-; GFX1250-NEXT: s_mov_b32 s15, s3
+; GFX1250-NEXT: s_mov_b32 s2, s13
; GFX1250-NEXT: s_mov_b32 s13, s3
-; GFX1250-NEXT: s_add_nc_u64 s[4:5], s[4:5], s[14:15]
-; GFX1250-NEXT: s_add_nc_u64 s[8:9], s[10:11], s[8:9]
+; GFX1250-NEXT: s_mul_u64 s[8:9], s[8:9], s[14:15]
+; GFX1250-NEXT: s_add_nc_u64 s[4:5], s[4:5], s[12:13]
+; GFX1250-NEXT: s_mul_u64 s[6:7], s[6:7], s[16:17]
; GFX1250-NEXT: s_mov_b32 s18, s5
-; GFX1250-NEXT: s_mov_b32 s21, s4
+; GFX1250-NEXT: s_mov_b32 s23, s3
; GFX1250-NEXT: s_add_nc_u64 s[2:3], s[2:3], s[18:19]
-; GFX1250-NEXT: s_or_b64 s[4:5], s[12:13], s[20:21]
+; GFX1250-NEXT: s_add_nc_u64 s[8:9], s[10:11], s[8:9]
+; GFX1250-NEXT: s_mov_b32 s21, s4
; GFX1250-NEXT: s_add_nc_u64 s[2:3], s[6:7], s[2:3]
-; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX1250-NEXT: s_or_b64 s[4:5], s[22:23], s[20:21]
; GFX1250-NEXT: s_add_nc_u64 s[2:3], s[2:3], s[8:9]
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
; GFX1250-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX1250-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-NEXT: s_mov_b32 s2, -1
@@ -4008,33 +4011,31 @@ define amdgpu_kernel void @v_mul_i128(ptr addrspace(1) %out, ptr addrspace(1) %a
; GFX1250-NEXT: v_and_b32_e32 v16, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_load_b128 v[0:3], v16, s[2:3] scale_offset
-; GFX1250-NEXT: global_load_b128 v[4:7], v16, s[0:1] scale_offset
-; GFX1250-NEXT: s_wait_loadcnt 0x1
-; GFX1250-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_mov_b32 v10, v0
+; GFX1250-NEXT: global_load_b128 v[0:3], v16, s[0:1] scale_offset
+; GFX1250-NEXT: global_load_b128 v[4:7], v16, s[2:3] scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_dual_mov_b32 v9, v11 :: v_dual_mov_b32 v8, v4
-; GFX1250-NEXT: v_mul_u64_e32 v[6:7], v[0:1], v[6:7]
-; GFX1250-NEXT: v_mul_u64_e32 v[8:9], v[8:9], v[10:11]
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[8:9], v0, v4, 0
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[10:11], v4, v2, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_mad_nc_u64_u32 v[6:7], v2, v4, v[6:7]
-; GFX1250-NEXT: v_mov_b32_e32 v10, v9
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_mad_nc_u64_u32 v[12:13], v5, v0, v[10:11]
-; GFX1250-NEXT: v_mad_u32 v0, v3, v4, v7
+; GFX1250-NEXT: v_dual_mov_b32 v13, 0 :: v_dual_mov_b32 v12, v9
+; GFX1250-NEXT: v_mad_u32 v3, v4, v3, v11
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_dual_mov_b32 v10, v13 :: v_dual_mov_b32 v13, v11
-; GFX1250-NEXT: v_mad_u32 v7, v2, v5, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_mad_nc_u64_u32 v[12:13], v4, v1, v[12:13]
-; GFX1250-NEXT: v_dual_mov_b32 v15, v11 :: v_dual_mov_b32 v9, v12
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[14:15], v1, v4, v[12:13]
+; GFX1250-NEXT: v_mad_u32 v11, v5, v2, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_dual_mov_b32 v12, v15 :: v_dual_mov_b32 v15, v13
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[10:11], v6, v0, v[10:11]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[2:3], v0, v5, v[14:15]
+; GFX1250-NEXT: v_mad_u32 v0, v7, v0, v11
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_mov_b32_e32 v14, v13
-; GFX1250-NEXT: v_add_nc_u64_e32 v[10:11], v[10:11], v[14:15]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], v5, v1, v[10:11]
-; GFX1250-NEXT: v_add_nc_u64_e32 v[10:11], v[0:1], v[6:7]
+; GFX1250-NEXT: v_dual_mov_b32 v14, v3 :: v_dual_mov_b32 v9, v2
+; GFX1250-NEXT: v_add_nc_u64_e32 v[12:13], v[12:13], v[14:15]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_mad_u32 v11, v6, v1, v0
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], v1, v5, v[12:13]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_add_nc_u64_e32 v[10:11], v[0:1], v[10:11]
; GFX1250-NEXT: global_store_b128 v16, v[8:11], s[2:3] scale_offset
; GFX1250-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
index 74bd338c7c022..69c07e6601ca5 100644
--- a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
+++ b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
@@ -2033,8 +2033,12 @@ define i64 @v_mul_sub_1_i64(i64 %x, i64 %y) {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], -1, v[2:3]
+; GFX1250-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], v4, v2, 0
+; GFX1250-NEXT: v_mad_u32 v1, v4, v3, v1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT: v_mad_u32 v1, v5, v2, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-LABEL: v_mul_sub_1_i64:
@@ -2125,8 +2129,12 @@ define i64 @v_mul_sub_1_i64_commute(i64 %x, i64 %y) {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], -1, v[2:3]
+; GFX1250-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], v2, v4, 0
+; GFX1250-NEXT: v_mad_u32 v1, v2, v5, v1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_u64_e32 v[0:1], v[2:3], v[0:1]
+; GFX1250-NEXT: v_mad_u32 v1, v3, v4, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-LABEL: v_mul_sub_1_i64_commute:
@@ -2214,9 +2222,12 @@ define i64 @v_mul_sub_x_i64(i64 %x, i64 %y) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mul_u64_e32 v[2:3], v[0:1], v[2:3]
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[4:5], v0, v2, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mad_u32 v3, v0, v3, v5
+; GFX1250-NEXT: v_mad_u32 v5, v1, v2, v3
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_sub_nc_u64_e32 v[0:1], v[2:3], v[0:1]
+; GFX1250-NEXT: v_sub_nc_u64_e32 v[0:1], v[4:5], v[0:1]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-LABEL: v_mul_sub_x_i64:
@@ -2307,8 +2318,12 @@ define i64 @v_mul_add_2_i64(i64 %x, i64 %y) {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], 2, v[2:3]
+; GFX1250-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], v4, v2, 0
+; GFX1250-NEXT: v_mad_u32 v1, v4, v3, v1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT: v_mad_u32 v1, v5, v2, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-LABEL: v_mul_add_2_i64:
@@ -2399,8 +2414,12 @@ define i64 @v_mul_sub_2_i64(i64 %x, i64 %y) {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], -2, v[2:3]
+; GFX1250-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], v4, v2, 0
+; GFX1250-NEXT: v_mad_u32 v1, v4, v3, v1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT: v_mad_u32 v1, v5, v2, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-LABEL: v_mul_sub_2_i64:
More information about the llvm-commits
mailing list