[llvm] Correctly round FP -> BF16 when SDAG expands such nodes (PR #82399)
David Majnemer via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 21 08:16:05 PST 2024
https://github.com/majnemer updated https://github.com/llvm/llvm-project/pull/82399
>From 0573396bfe302400c616d9f30a3ca461df15a9ee Mon Sep 17 00:00:00 2001
From: David Majnemer <david.majnemer at gmail.com>
Date: Thu, 15 Feb 2024 18:15:45 +0000
Subject: [PATCH] Correctly round FP -> BF16 when SDAG expands such nodes
We did something pretty naive:
- round FP64 -> BF16 by first rounding to FP32
- skip FP32 -> BF16 rounding entirely
- taking the top 16 bits of a FP32 which will turn some NaNs into infinities
Let's do this in a more principled way by rounding types with more
precision than FP32 to FP32 using round-inexact-to-odd which will
negate double rounding issues.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 13 +
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 10 +-
.../CodeGen/SelectionDAG/TargetLowering.cpp | 122 +
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 69 +
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 3 +
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 10 +-
llvm/test/CodeGen/AMDGPU/bf16.ll | 19201 +++++++++++++---
.../test/CodeGen/AMDGPU/fmed3-cast-combine.ll | 14 +
.../CodeGen/AMDGPU/fneg-modifier-casting.ll | 4 +
llvm/test/CodeGen/AMDGPU/function-args.ll | 95 +-
llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll | 224 +-
.../isel-amdgpu-cs-chain-preserve-cc.ll | 1462 +-
.../CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll | 53 +
llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll | 54 +-
.../CodeGen/AMDGPU/vector_shuffle.packed.ll | 293 +-
llvm/test/CodeGen/NVPTX/bf16-instructions.ll | 2 +-
16 files changed, 17189 insertions(+), 4440 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 612433b54f6e44..f2e00aab8d5da2 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -5124,6 +5124,19 @@ class TargetLowering : public TargetLoweringBase {
/// \returns The expansion result
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const;
+ /// Truncate Op to ResultVT. If the result is exact, leave it alone. If it is
+ /// not exact, force the result to be odd.
+ /// \param ResultVT The type of result.
+ /// \param Op The value to round.
+ /// \returns The expansion result
+ SDValue expandRoundInexactToOdd(EVT ResultVT, SDValue Op, const SDLoc &DL,
+ SelectionDAG &DAG) const;
+
+ /// Expand round(fp) to fp conversion
+ /// \param N Node to expand
+ /// \returns The expansion result
+ SDValue expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const;
+
/// Expand check for floating point class.
/// \param ResultVT The type of intrinsic call result.
/// \param Op The tested value.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 252b6e9997a710..68de9ed8b91624 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3217,10 +3217,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
}
break;
case ISD::FP_ROUND: {
- EVT VT = Node->getValueType(0);
- if (VT.getScalarType() == MVT::bf16) {
- Results.push_back(
- DAG.getNode(ISD::FP_TO_BF16, SDLoc(Node), VT, Node->getOperand(0)));
+ if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
+ Results.push_back(Tmp1);
break;
}
@@ -3293,6 +3291,10 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
if (Op.getValueType() != MVT::f32)
Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
+ // Certain SNaNs will turn into infinities if we do a simple shift right.
+ if (!DAG.isKnownNeverSNaN(Op)) {
+ Op = DAG.getNode(ISD::FCANONICALIZE, dl, MVT::f32, Op, Node->getFlags());
+ }
Op = DAG.getNode(
ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op),
DAG.getConstant(16, dl,
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index f689cd120ecbdc..0a730ec056e7e4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -10855,6 +10855,128 @@ SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
return DAG.getSelect(dl, DstVT, IsNan, ZeroInt, Select);
}
+SDValue TargetLowering::expandRoundInexactToOdd(EVT ResultVT, SDValue Op,
+ const SDLoc &dl,
+ SelectionDAG &DAG) const {
+ EVT OperandVT = Op.getValueType();
+ if (OperandVT.getScalarType() == ResultVT.getScalarType())
+ return Op;
+ EVT ResultIntVT = ResultVT.changeTypeToInteger();
+ // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
+ // can induce double-rounding which may alter the results. We can
+ // correct for this using a trick explained in: Boldo, Sylvie, and
+ // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
+ // World Congress. 2005.
+ unsigned BitSize = OperandVT.getScalarSizeInBits();
+ EVT WideIntVT = OperandVT.changeTypeToInteger();
+ SDValue OpAsInt = DAG.getBitcast(WideIntVT, Op);
+ SDValue SignBit =
+ DAG.getNode(ISD::AND, dl, WideIntVT, OpAsInt,
+ DAG.getConstant(APInt::getSignMask(BitSize), dl, WideIntVT));
+ SDValue AbsWide;
+ if (isOperationLegalOrCustom(ISD::FABS, OperandVT)) {
+ AbsWide = DAG.getNode(ISD::FABS, dl, OperandVT, Op);
+ } else {
+ SDValue ClearedSign = DAG.getNode(
+ ISD::AND, dl, WideIntVT, OpAsInt,
+ DAG.getConstant(APInt::getSignedMaxValue(BitSize), dl, WideIntVT));
+ AbsWide = DAG.getBitcast(OperandVT, ClearedSign);
+ }
+ SDValue AbsNarrow = DAG.getFPExternOrRound(AbsWide, dl, ResultVT);
+ SDValue AbsNarrowAsWide = DAG.getFPExtendOrRound(AbsNarrow, dl, OperandVT);
+
+ // We can keep the narrow value as-is if narrowing was exact (no
+ // rounding error), the wide value was NaN (the narrow value is also
+ // NaN and should be preserved) or if we rounded to the odd value.
+ SDValue NarrowBits = DAG.getNode(ISD::BITCAST, dl, ResultIntVT, AbsNarrow);
+ SDValue One = DAG.getConstant(1, dl, ResultIntVT);
+ SDValue NegativeOne = DAG.getAllOnesConstant(dl, ResultIntVT);
+ SDValue And = DAG.getNode(ISD::AND, dl, ResultIntVT, NarrowBits, One);
+ EVT ResultIntVTCCVT = getSetCCResultType(
+ DAG.getDataLayout(), *DAG.getContext(), And.getValueType());
+ SDValue Zero = DAG.getConstant(0, dl, ResultIntVT);
+ SDValue AlreadyOdd = DAG.getSetCC(dl, ResultIntVTCCVT, And, Zero, ISD::SETNE);
+
+ EVT WideSetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
+ AbsWide.getValueType());
+ SDValue KeepNarrow =
+ DAG.getSetCC(dl, WideSetCCVT, AbsWide, AbsNarrowAsWide, ISD::SETUEQ);
+ KeepNarrow = DAG.getNode(ISD::OR, dl, WideSetCCVT, KeepNarrow, AlreadyOdd);
+ // We morally performed a round-down if `abs_narrow` is smaller than
+ // `abs_wide`.
+ SDValue NarrowIsRd =
+ DAG.getSetCC(dl, WideSetCCVT, AbsWide, AbsNarrowAsWide, ISD::SETOGT);
+ // If the narrow value is odd or exact, pick it.
+ // Otherwise, narrow is even and corresponds to either the rounded-up
+ // or rounded-down value. If narrow is the rounded-down value, we want
+ // the rounded-up value as it will be odd.
+ SDValue Adjust = DAG.getSelect(dl, ResultIntVT, NarrowIsRd, One, NegativeOne);
+ Adjust = DAG.getSelect(dl, ResultIntVT, KeepNarrow, Zero, Adjust);
+ int ShiftAmount = BitSize - ResultVT.getScalarSizeInBits();
+ SDValue ShiftCnst = DAG.getShiftAmountConstant(ShiftAmount, WideIntVT, dl);
+ SignBit = DAG.getNode(ISD::SRL, dl, WideIntVT, SignBit, ShiftCnst);
+ SignBit = DAG.getNode(ISD::TRUNCATE, dl, ResultIntVT, SignBit);
+ Op = DAG.getNode(ISD::OR, dl, ResultIntVT, Adjust, SignBit);
+ return DAG.getNode(ISD::BITCAST, dl, ResultVT, Op);
+}
+
+SDValue TargetLowering::expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const {
+ assert(Node->getOpcode() == ISD::FP_ROUND && "Unexpected opcode!");
+ SDValue Op = Node->getOperand(0);
+ EVT VT = Node->getValueType(0);
+ SDLoc dl(Node);
+ if (VT.getScalarType() == MVT::bf16) {
+ if (Node->getConstantOperandVal(1) == 1) {
+ return DAG.getNode(ISD::FP_TO_BF16, dl, VT, Node->getOperand(0));
+ }
+ EVT OperandVT = Op.getValueType();
+ SDValue IsNaN = DAG.getSetCC(
+ dl,
+ getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), OperandVT),
+ Op, Op, ISD::SETUO);
+
+ // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
+ // can induce double-rounding which may alter the results. We can
+ // correct for this using a trick explained in: Boldo, Sylvie, and
+ // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
+ // World Congress. 2005.
+ EVT F32 = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;
+ EVT I32 = F32.changeTypeToInteger();
+ Op = expandRoundInexactToOdd(F32, Op, dl, DAG);
+ Op = DAG.getNode(ISD::BITCAST, dl, I32, Op);
+
+ // Extract the sign bit.
+ SDValue SignBit = DAG.getNode(
+ ISD::AND, dl, I32, Op,
+ DAG.getConstant(APInt::getSignMask(32), dl, I32));
+ // Set the quiet bit.
+ SDValue NaN =
+ DAG.getNode(ISD::OR, dl, I32, SignBit, DAG.getConstant(0x400000, dl, I32));
+
+ // Factor in the contribution of the low 16 bits.
+ SDValue One = DAG.getConstant(1, dl, I32);
+ SDValue Lsb = DAG.getNode(ISD::SRL, dl, I32, Op,
+ DAG.getShiftAmountConstant(16, I32, dl));
+ Lsb = DAG.getNode(ISD::AND, dl, I32, Lsb, One);
+ SDValue RoundingBias = DAG.getNode(
+ ISD::ADD, dl, I32, DAG.getConstant(0x7fff, dl, I32), Lsb);
+ SDValue Add = DAG.getNode(ISD::ADD, dl, I32, Op, RoundingBias);
+
+ // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
+ // 0x80000000.
+ Op = DAG.getSelect(dl, I32, IsNaN, NaN, Add);
+
+ // Now that we have rounded, shift the bits into position.
+ Op = DAG.getNode(ISD::SRL, dl, I32, Op,
+ DAG.getShiftAmountConstant(16, I32, dl));
+ Op = DAG.getNode(ISD::BITCAST, dl, I32, Op);
+ EVT I16 = I32.isVector() ? I32.changeVectorElementType(MVT::i16) : MVT::i16;
+ Op = DAG.getNode(ISD::TRUNCATE, dl, I16, Op);
+ return DAG.getNode(ISD::BITCAST, dl, VT, Op);
+ }
+ return SDValue();
+}
+
SDValue TargetLowering::expandVectorSplice(SDNode *Node,
SelectionDAG &DAG) const {
assert(Node->getOpcode() == ISD::VECTOR_SPLICE && "Unexpected opcode!");
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 7f58b312e7a201..a6ad5ecc458a3a 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -776,6 +776,11 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
AddPromotedToType(Op, MVT::bf16, MVT::f32);
}
+ if (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78) {
+ setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);
+ setOperationAction(ISD::FP_ROUND, MVT::bf16, Custom);
+ }
+
// sm_80 only has conversions between f32 and bf16. Custom lower all other
// bf16 conversions.
if (STI.hasBF16Math() &&
@@ -2465,6 +2470,66 @@ SDValue NVPTXTargetLowering::LowerFP_TO_INT(SDValue Op,
return Op;
}
+SDValue NVPTXTargetLowering::LowerFP_ROUND(SDValue Op,
+ SelectionDAG &DAG) const {
+ EVT NarrowVT = Op.getValueType();
+ SDValue Wide = Op.getOperand(0);
+ EVT WideVT = Wide.getValueType();
+ if (NarrowVT.getScalarType() == MVT::bf16) {
+ const TargetLowering *TLI = STI.getTargetLowering();
+ if (STI.getSmVersion() < 80 || STI.getPTXVersion() < 70) {
+ return TLI->expandFP_ROUND(Op.getNode(), DAG);
+ }
+ if (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78) {
+ // This combination was the first to support f32 -> bf16.
+ if (STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70) {
+ if (WideVT.getScalarType() == MVT::f32) {
+ return Op;
+ }
+ if (WideVT.getScalarType() == MVT::f64) {
+ SDLoc Loc(Op);
+ // Round-inexact-to-odd f64 to f32, then do the final rounding using
+ // the hardware f32 -> bf16 instruction.
+ SDValue rod = TLI->expandRoundInexactToOdd(
+ WideVT.isVector() ? WideVT.changeVectorElementType(MVT::f32)
+ : MVT::f32,
+ Wide, Loc, DAG);
+ return DAG.getFPExtendOrRound(rod, Loc, NarrowVT);
+ }
+ }
+ return TLI->expandFP_ROUND(Op.getNode(), DAG);
+ }
+ }
+
+ // Everything else is considered legal.
+ return Op;
+}
+
+SDValue NVPTXTargetLowering::LowerFP_EXTEND(SDValue Op,
+ SelectionDAG &DAG) const {
+ if (Op.getOperand(0).getValueType() == MVT::bf16) {
+ if (Op.getValueType() == MVT::f32 &&
+ (STI.getSmVersion() < 80 || STI.getPTXVersion() < 71)) {
+ SDLoc Loc(Op);
+ return DAG.getNode(ISD::BF16_TO_FP, Loc, Op.getValueType(),
+ Op.getOperand(0));
+ }
+ if (Op.getValueType() == MVT::f64 &&
+ (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78)) {
+ SDLoc Loc(Op);
+ if (STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 71) {
+ Op = DAG.getNode(ISD::FP_EXTEND, Loc, MVT::f32, Op.getOperand(0));
+ return DAG.getNode(ISD::FP_EXTEND, Loc, MVT::f64, Op);
+ }
+ return DAG.getNode(ISD::BF16_TO_FP, Loc, Op.getValueType(),
+ Op.getOperand(0));
+ }
+ }
+
+ // Everything else is considered legal.
+ return Op;
+}
+
static SDValue LowerVectorArith(SDValue Op, SelectionDAG &DAG) {
SDLoc DL(Op);
if (Op.getValueType() != MVT::v2i16)
@@ -2527,6 +2592,10 @@ NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::FP_TO_SINT:
case ISD::FP_TO_UINT:
return LowerFP_TO_INT(Op, DAG);
+ case ISD::FP_ROUND:
+ return LowerFP_ROUND(Op, DAG);
+ case ISD::FP_EXTEND:
+ return LowerFP_EXTEND(Op, DAG);
case ISD::VAARG:
return LowerVAARG(Op, DAG);
case ISD::VASTART:
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 5d3fd992812ef9..cf1d4580766918 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -618,6 +618,9 @@ class NVPTXTargetLowering : public TargetLowering {
SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
+
SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerLOADi1(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 631136ad621464..40d82ebecbed35 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -662,7 +662,7 @@ let hasSideEffects = false in {
// bf16->f32 was introduced early.
[hasPTX<71>, hasSM<80>],
// bf16->everything else needs sm90/ptx78
- [hasPTX<78>, hasSM<90>])>;
+ [hasPTX<78>, hasSM<90>])>;
def _f32 :
NVPTXInst<(outs RC:$dst),
(ins Float32Regs:$src, CvtMode:$mode),
@@ -3647,7 +3647,7 @@ def : Pat<(f16 (fpround Float32Regs:$a)),
// fpround f32 -> bf16
def : Pat<(bf16 (fpround Float32Regs:$a)),
- (CVT_bf16_f32 Float32Regs:$a, CvtRN)>;
+ (CVT_bf16_f32 Float32Regs:$a, CvtRN)>, Requires<[hasPTX<70>, hasSM<80>]>;
// fpround f64 -> f16
def : Pat<(f16 (fpround Float64Regs:$a)),
@@ -3655,7 +3655,7 @@ def : Pat<(f16 (fpround Float64Regs:$a)),
// fpround f64 -> bf16
def : Pat<(bf16 (fpround Float64Regs:$a)),
- (CVT_bf16_f64 Float64Regs:$a, CvtRN)>;
+ (CVT_bf16_f64 Float64Regs:$a, CvtRN)>, Requires<[hasPTX<78>, hasSM<90>]>;
// fpround f64 -> f32
def : Pat<(f32 (fpround Float64Regs:$a)),
(CVT_f32_f64 Float64Regs:$a, CvtRN_FTZ)>, Requires<[doF32FTZ]>;
@@ -3671,7 +3671,7 @@ def : Pat<(f32 (fpextend (f16 Int16Regs:$a))),
def : Pat<(f32 (fpextend (bf16 Int16Regs:$a))),
(CVT_f32_bf16 Int16Regs:$a, CvtNONE_FTZ)>, Requires<[doF32FTZ]>;
def : Pat<(f32 (fpextend (bf16 Int16Regs:$a))),
- (CVT_f32_bf16 Int16Regs:$a, CvtNONE)>;
+ (CVT_f32_bf16 Int16Regs:$a, CvtNONE)>, Requires<[hasPTX<71>, hasSM<80>]>;
// fpextend f16 -> f64
def : Pat<(f64 (fpextend (f16 Int16Regs:$a))),
@@ -3679,7 +3679,7 @@ def : Pat<(f64 (fpextend (f16 Int16Regs:$a))),
// fpextend bf16 -> f64
def : Pat<(f64 (fpextend (bf16 Int16Regs:$a))),
- (CVT_f64_bf16 Int16Regs:$a, CvtNONE)>;
+ (CVT_f64_bf16 Int16Regs:$a, CvtNONE)>, Requires<[hasPTX<78>, hasSM<90>]>;
// fpextend f32 -> f64
def : Pat<(f64 (fpextend Float32Regs:$a)),
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 387c4a16a008ae..e841a8867fc522 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -924,8 +924,10 @@ define void @v_store_global_v2bf16(<2 x bfloat> %val, ptr addrspace(1) %ptr) {
; GCN-LABEL: v_store_global_v2bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s6, 0
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s4, s6
@@ -937,7 +939,9 @@ define void @v_store_global_v2bf16(<2 x bfloat> %val, ptr addrspace(1) %ptr) {
; GFX7-LABEL: v_store_global_v2bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX7-NEXT: s_mov_b32 s7, 0xf000
@@ -980,13 +984,16 @@ define void @v_store_global_v3bf16(<3 x bfloat> %val, ptr addrspace(1) %ptr) {
; GCN-LABEL: v_store_global_v3bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, 0
-; GCN-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: s_mov_b32 s4, s6
; GCN-NEXT: s_mov_b32 s5, s6
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GCN-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GCN-NEXT: buffer_store_short v2, v[3:4], s[4:7], 0 addr64 offset:4
; GCN-NEXT: buffer_store_dword v0, v[3:4], s[4:7], 0 addr64
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
@@ -995,13 +1002,16 @@ define void @v_store_global_v3bf16(<3 x bfloat> %val, ptr addrspace(1) %ptr) {
; GFX7-LABEL: v_store_global_v3bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v2
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: buffer_store_short v1, v[3:4], s[4:7], 0 addr64 offset:4
; GFX7-NEXT: buffer_store_dword v0, v[3:4], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -1047,9 +1057,13 @@ define void @v_store_global_v4bf16(<4 x bfloat> %val, ptr addrspace(1) %ptr) {
; GCN-LABEL: v_store_global_v4bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_alignbit_b32 v1, v3, v2, 16
; GCN-NEXT: v_alignbit_b32 v0, v6, v0, 16
; GCN-NEXT: s_mov_b32 s7, 0xf000
@@ -1062,8 +1076,12 @@ define void @v_store_global_v4bf16(<4 x bfloat> %val, ptr addrspace(1) %ptr) {
; GFX7-LABEL: v_store_global_v4bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: v_alignbit_b32 v2, v3, v2, 16
; GFX7-NEXT: v_alignbit_b32 v1, v1, v0, 16
@@ -1109,28 +1127,44 @@ define void @v_store_global_v8bf16(<8 x bfloat> %val, ptr addrspace(1) %ptr) {
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, 0
-; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GCN-NEXT: v_lshrrev_b32_e32 v10, 16, v5
-; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, s6
; GCN-NEXT: s_mov_b32 s5, s6
-; GCN-NEXT: v_alignbit_b32 v5, v7, v6, 16
-; GCN-NEXT: v_alignbit_b32 v4, v10, v4, 16
-; GCN-NEXT: v_alignbit_b32 v3, v3, v2, 16
-; GCN-NEXT: v_alignbit_b32 v2, v1, v0, 16
-; GCN-NEXT: buffer_store_dwordx4 v[2:5], v[8:9], s[4:7], 0 addr64
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v7
+; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GCN-NEXT: v_lshrrev_b32_e32 v11, 16, v1
+; GCN-NEXT: v_alignbit_b32 v3, v2, v6, 16
+; GCN-NEXT: v_alignbit_b32 v2, v5, v4, 16
+; GCN-NEXT: v_alignbit_b32 v1, v7, v10, 16
+; GCN-NEXT: v_alignbit_b32 v0, v11, v0, 16
+; GCN-NEXT: buffer_store_dwordx4 v[0:3], v[8:9], s[4:7], 0 addr64
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_store_global_v8bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
@@ -1175,53 +1209,85 @@ define void @v_store_global_v16bf16(<16 x bfloat> %val, ptr addrspace(1) %ptr) {
; GCN-LABEL: v_store_global_v16bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GCN-NEXT: v_lshrrev_b32_e32 v18, 16, v5
-; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, 0
-; GCN-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; GCN-NEXT: v_lshrrev_b32_e32 v19, 16, v13
-; GCN-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GCN-NEXT: v_alignbit_b32 v5, v7, v6, 16
-; GCN-NEXT: v_alignbit_b32 v4, v18, v4, 16
-; GCN-NEXT: v_alignbit_b32 v3, v3, v2, 16
-; GCN-NEXT: v_alignbit_b32 v2, v1, v0, 16
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v15
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT: v_lshrrev_b32_e32 v15, 16, v3
+; GCN-NEXT: v_lshrrev_b32_e32 v19, 16, v1
; GCN-NEXT: s_mov_b32 s4, s6
; GCN-NEXT: s_mov_b32 s5, s6
-; GCN-NEXT: v_alignbit_b32 v13, v15, v14, 16
-; GCN-NEXT: v_alignbit_b32 v12, v19, v12, 16
-; GCN-NEXT: v_alignbit_b32 v11, v11, v10, 16
-; GCN-NEXT: v_alignbit_b32 v10, v9, v8, 16
-; GCN-NEXT: buffer_store_dwordx4 v[10:13], v[16:17], s[4:7], 0 addr64 offset:16
-; GCN-NEXT: buffer_store_dwordx4 v[2:5], v[16:17], s[4:7], 0 addr64
+; GCN-NEXT: v_lshrrev_b32_e32 v20, 16, v2
+; GCN-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GCN-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT: v_alignbit_b32 v3, v7, v6, 16
+; GCN-NEXT: v_alignbit_b32 v2, v5, v4, 16
+; GCN-NEXT: v_alignbit_b32 v1, v15, v18, 16
+; GCN-NEXT: v_alignbit_b32 v0, v19, v0, 16
+; GCN-NEXT: v_alignbit_b32 v7, v20, v14, 16
+; GCN-NEXT: v_alignbit_b32 v6, v13, v12, 16
+; GCN-NEXT: v_alignbit_b32 v5, v11, v10, 16
+; GCN-NEXT: v_alignbit_b32 v4, v9, v8, 16
+; GCN-NEXT: buffer_store_dwordx4 v[4:7], v[16:17], s[4:7], 0 addr64 offset:16
+; GCN-NEXT: buffer_store_dwordx4 v[0:3], v[16:17], s[4:7], 0 addr64
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_store_global_v16bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
; GFX7-NEXT: v_alignbit_b32 v4, v3, v2, 16
; GFX7-NEXT: v_alignbit_b32 v3, v1, v0, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v15
-; GFX7-NEXT: v_alignbit_b32 v14, v0, v14, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v13
-; GFX7-NEXT: v_alignbit_b32 v13, v0, v12, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v11
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v15
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v14
+; GFX7-NEXT: v_alignbit_b32 v14, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v13
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v12
+; GFX7-NEXT: v_alignbit_b32 v13, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v11
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v10
+; GFX7-NEXT: v_alignbit_b32 v12, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: v_alignbit_b32 v12, v0, v10, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v9
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v8
; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: v_alignbit_b32 v11, v0, v8, 16
+; GFX7-NEXT: v_alignbit_b32 v11, v0, v1, 16
; GFX7-NEXT: v_alignbit_b32 v6, v7, v6, 16
; GFX7-NEXT: buffer_store_dwordx4 v[11:14], v[16:17], s[4:7], 0 addr64 offset:16
; GFX7-NEXT: buffer_store_dwordx4 v[3:6], v[16:17], s[4:7], 0 addr64
@@ -1269,49 +1335,82 @@ define void @v_store_global_v32bf16(<32 x bfloat> %val, ptr addrspace(1) %ptr) {
; GCN-LABEL: v_store_global_v32bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; GCN-NEXT: v_lshrrev_b32_e32 v31, 16, v5
; GCN-NEXT: v_alignbit_b32 v5, v7, v6, 16
; GCN-NEXT: v_alignbit_b32 v4, v31, v4, 16
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GCN-NEXT: v_alignbit_b32 v3, v3, v2, 16
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: v_alignbit_b32 v2, v1, v0, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v15
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v13
-; GCN-NEXT: v_alignbit_b32 v13, v0, v14, 16
-; GCN-NEXT: v_alignbit_b32 v12, v1, v12, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; GCN-NEXT: v_alignbit_b32 v11, v0, v10, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v9
-; GCN-NEXT: v_alignbit_b32 v10, v0, v8, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v23
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v21
-; GCN-NEXT: v_alignbit_b32 v9, v0, v22, 16
-; GCN-NEXT: v_alignbit_b32 v8, v1, v20, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v19
-; GCN-NEXT: v_alignbit_b32 v7, v0, v18, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v29
-; GCN-NEXT: v_lshrrev_b32_e32 v14, 16, v27
-; GCN-NEXT: v_alignbit_b32 v6, v0, v16, 16
-; GCN-NEXT: v_alignbit_b32 v16, v1, v28, 16
-; GCN-NEXT: v_alignbit_b32 v15, v14, v26, 16
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v15
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v12
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GCN-NEXT: v_alignbit_b32 v13, v0, v1, 16
+; GCN-NEXT: v_alignbit_b32 v12, v6, v7, 16
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v10
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_alignbit_b32 v11, v0, v1, 16
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v8
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_alignbit_b32 v10, v0, v1, 16
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v23
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v22
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v21
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v20
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GCN-NEXT: v_alignbit_b32 v9, v0, v1, 16
+; GCN-NEXT: v_alignbit_b32 v8, v6, v7, 16
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v19
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v18
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_alignbit_b32 v7, v0, v1, 16
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v17
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v16
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v29
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v28
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v27
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v26
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_lshrrev_b32_e32 v16, 16, v6
+; GCN-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GCN-NEXT: v_alignbit_b32 v6, v0, v1, 16
+; GCN-NEXT: v_alignbit_b32 v16, v16, v14, 16
+; GCN-NEXT: v_alignbit_b32 v15, v15, v17, 16
; GCN-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v25
-; GCN-NEXT: v_alignbit_b32 v14, v0, v24, 16
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v25
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v24
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_alignbit_b32 v14, v0, v14, 16
; GCN-NEXT: buffer_load_dword v17, off, s[0:3], s32
; GCN-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4
; GCN-NEXT: s_mov_b32 s6, 0
-; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v30
; GCN-NEXT: s_mov_b32 s4, s6
; GCN-NEXT: s_mov_b32 s5, s6
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_store_dwordx4 v[6:9], v[0:1], s[4:7], 0 addr64 offset:32
; GCN-NEXT: buffer_store_dwordx4 v[10:13], v[0:1], s[4:7], 0 addr64 offset:16
-; GCN-NEXT: v_alignbit_b32 v17, v17, v30, 16
+; GCN-NEXT: s_waitcnt expcnt(1)
+; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v17
+; GCN-NEXT: v_alignbit_b32 v17, v6, v18, 16
; GCN-NEXT: buffer_store_dwordx4 v[14:17], v[0:1], s[4:7], 0 addr64 offset:48
; GCN-NEXT: buffer_store_dwordx4 v[2:5], v[0:1], s[4:7], 0 addr64
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
@@ -1320,49 +1419,82 @@ define void @v_store_global_v32bf16(<32 x bfloat> %val, ptr addrspace(1) %ptr) {
; GFX7-LABEL: v_store_global_v32bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_alignbit_b32 v3, v3, v2, 16
; GFX7-NEXT: v_alignbit_b32 v2, v1, v0, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v15
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v13
-; GFX7-NEXT: v_alignbit_b32 v13, v0, v14, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; GFX7-NEXT: v_alignbit_b32 v11, v0, v10, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v9
-; GFX7-NEXT: v_alignbit_b32 v10, v0, v8, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v23
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v14
+; GFX7-NEXT: buffer_load_dword v14, off, s[0:3], s32
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v15
; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_alignbit_b32 v9, v0, v22, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v19
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v5
; GFX7-NEXT: v_alignbit_b32 v5, v7, v6, 16
-; GFX7-NEXT: v_alignbit_b32 v12, v1, v12, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v21
-; GFX7-NEXT: v_alignbit_b32 v7, v0, v18, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX7-NEXT: buffer_load_dword v17, off, s[0:3], s32
-; GFX7-NEXT: v_alignbit_b32 v8, v1, v20, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v29
-; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v27
-; GFX7-NEXT: v_alignbit_b32 v6, v0, v16, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v25
-; GFX7-NEXT: v_alignbit_b32 v16, v1, v28, 16
-; GFX7-NEXT: v_alignbit_b32 v15, v14, v26, 16
-; GFX7-NEXT: v_alignbit_b32 v14, v0, v24, 16
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v13
+; GFX7-NEXT: v_alignbit_b32 v13, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v11
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v10
+; GFX7-NEXT: v_alignbit_b32 v11, v0, v1, 16
; GFX7-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8
; GFX7-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v12
+; GFX7-NEXT: v_alignbit_b32 v12, v6, v7, 16
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v9
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v8
+; GFX7-NEXT: v_alignbit_b32 v10, v6, v7, 16
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v23
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v22
+; GFX7-NEXT: v_alignbit_b32 v9, v6, v7, 16
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v21
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v20
+; GFX7-NEXT: v_alignbit_b32 v8, v6, v7, 16
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v19
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v18
+; GFX7-NEXT: v_alignbit_b32 v7, v6, v7, 16
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v17
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v16
+; GFX7-NEXT: v_alignbit_b32 v6, v6, v15, 16
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v30
; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v24
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: v_alignbit_b32 v4, v31, v4, 16
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX7-NEXT: v_alignbit_b32 v17, v14, v15, 16
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v29
+; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v28
+; GFX7-NEXT: v_alignbit_b32 v16, v14, v15, 16
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v27
+; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v26
+; GFX7-NEXT: v_alignbit_b32 v15, v14, v15, 16
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v25
+; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX7-NEXT: v_alignbit_b32 v14, v14, v18, 16
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_store_dwordx4 v[14:17], v[0:1], s[4:7], 0 addr64 offset:48
; GFX7-NEXT: buffer_store_dwordx4 v[6:9], v[0:1], s[4:7], 0 addr64 offset:32
; GFX7-NEXT: buffer_store_dwordx4 v[10:13], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX7-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX7-NEXT: v_alignbit_b32 v17, v17, v30, 16
-; GFX7-NEXT: buffer_store_dwordx4 v[14:17], v[0:1], s[4:7], 0 addr64 offset:48
; GFX7-NEXT: buffer_store_dwordx4 v[2:5], v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -1420,301 +1552,428 @@ define void @v_store_global_v64bf16(<64 x bfloat> %val, ptr addrspace(1) %ptr) {
; GCN-LABEL: v_store_global_v64bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GCN-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; GCN-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; GCN-NEXT: v_alignbit_b32 v22, v23, v22, 16
-; GCN-NEXT: v_alignbit_b32 v21, v21, v20, 16
+; GCN-NEXT: v_lshrrev_b32_e32 v31, 16, v21
+; GCN-NEXT: v_alignbit_b32 v21, v23, v22, 16
+; GCN-NEXT: v_alignbit_b32 v20, v31, v20, 16
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GCN-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; GCN-NEXT: v_alignbit_b32 v20, v19, v18, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v19, 16, v17
-; GCN-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:136
-; GCN-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:132
-; GCN-NEXT: v_alignbit_b32 v19, v19, v16, 16
+; GCN-NEXT: v_alignbit_b32 v19, v19, v18, 16
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v17
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v16
+; GCN-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:136
+; GCN-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:132
+; GCN-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; GCN-NEXT: v_alignbit_b32 v18, v18, v22, 16
; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s4, s6
; GCN-NEXT: s_mov_b32 s5, s6
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_store_dwordx4 v[19:22], v[17:18], s[4:7], 0 addr64 offset:32
+; GCN-NEXT: buffer_store_dwordx4 v[18:21], v[16:17], s[4:7], 0 addr64 offset:32
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GCN-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; GCN-NEXT: v_lshrrev_b32_e32 v16, 16, v13
+; GCN-NEXT: s_waitcnt expcnt(0)
+; GCN-NEXT: v_lshrrev_b32_e32 v18, 16, v13
; GCN-NEXT: v_alignbit_b32 v13, v15, v14, 16
-; GCN-NEXT: v_alignbit_b32 v12, v16, v12, 16
+; GCN-NEXT: v_alignbit_b32 v12, v18, v12, 16
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GCN-NEXT: v_lshrrev_b32_e32 v11, 16, v11
; GCN-NEXT: v_alignbit_b32 v11, v11, v10, 16
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; GCN-NEXT: v_alignbit_b32 v10, v9, v8, 16
-; GCN-NEXT: buffer_store_dwordx4 v[10:13], v[17:18], s[4:7], 0 addr64 offset:16
+; GCN-NEXT: buffer_store_dwordx4 v[10:13], v[16:17], s[4:7], 0 addr64 offset:16
; GCN-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:128
-; GCN-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:120
+; GCN-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:124
; GCN-NEXT: s_waitcnt expcnt(0)
-; GCN-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124
-; GCN-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116
+; GCN-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:120
+; GCN-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:116
; GCN-NEXT: s_waitcnt vmcnt(3)
-; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v11
+; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GCN-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GCN-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:112
+; GCN-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:108
+; GCN-NEXT: v_alignbit_b32 v11, v8, v9, 16
+; GCN-NEXT: v_alignbit_b32 v10, v10, v12, 16
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_alignbit_b32 v11, v8, v10, 16
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v13
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v10, v9, v12, 16
-; GCN-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:112
-; GCN-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:108
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v14
; GCN-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:104
; GCN-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:100
-; GCN-NEXT: s_waitcnt vmcnt(3)
; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GCN-NEXT: s_waitcnt vmcnt(2)
; GCN-NEXT: v_alignbit_b32 v9, v8, v9, 16
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v12
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v12
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v8, v8, v13, 16
-; GCN-NEXT: buffer_store_dwordx4 v[8:11], v[17:18], s[4:7], 0 addr64 offset:112
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v13
+; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GCN-NEXT: v_alignbit_b32 v8, v8, v12, 16
+; GCN-NEXT: buffer_store_dwordx4 v[8:11], v[16:17], s[4:7], 0 addr64 offset:112
; GCN-NEXT: s_waitcnt expcnt(0)
; GCN-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:96
-; GCN-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:88
-; GCN-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:92
-; GCN-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:84
+; GCN-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:92
+; GCN-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:88
+; GCN-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:84
; GCN-NEXT: s_waitcnt vmcnt(3)
-; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v11
+; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GCN-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GCN-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:80
+; GCN-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:76
+; GCN-NEXT: v_alignbit_b32 v11, v8, v9, 16
+; GCN-NEXT: v_alignbit_b32 v10, v10, v12, 16
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_alignbit_b32 v11, v8, v10, 16
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v13
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v10, v9, v12, 16
-; GCN-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:80
-; GCN-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:76
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v14
; GCN-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:72
; GCN-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:68
-; GCN-NEXT: s_waitcnt vmcnt(3)
; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GCN-NEXT: s_waitcnt vmcnt(2)
; GCN-NEXT: v_alignbit_b32 v9, v8, v9, 16
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v12
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v12
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v8, v8, v13, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GCN-NEXT: v_lshrrev_b32_e32 v12, 16, v5
-; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GCN-NEXT: v_lshrrev_b32_e32 v13, 16, v29
-; GCN-NEXT: v_lshrrev_b32_e32 v14, 16, v27
-; GCN-NEXT: v_lshrrev_b32_e32 v15, 16, v25
-; GCN-NEXT: buffer_store_dwordx4 v[8:11], v[17:18], s[4:7], 0 addr64 offset:96
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v13
+; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GCN-NEXT: v_alignbit_b32 v8, v8, v12, 16
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v30
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v29
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v28
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v27
+; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v26
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v25
+; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v24
+; GCN-NEXT: buffer_store_dwordx4 v[8:11], v[16:17], s[4:7], 0 addr64 offset:96
; GCN-NEXT: s_waitcnt expcnt(0)
-; GCN-NEXT: buffer_load_dword v9, off, s[0:3], s32
-; GCN-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:32
-; GCN-NEXT: v_alignbit_b32 v5, v7, v6, 16
-; GCN-NEXT: v_alignbit_b32 v4, v12, v4, 16
-; GCN-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:28
-; GCN-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:24
-; GCN-NEXT: v_alignbit_b32 v3, v3, v2, 16
-; GCN-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:20
-; GCN-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:16
-; GCN-NEXT: v_alignbit_b32 v2, v1, v0, 16
-; GCN-NEXT: v_alignbit_b32 v8, v13, v28, 16
-; GCN-NEXT: v_alignbit_b32 v7, v14, v26, 16
-; GCN-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12
-; GCN-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8
-; GCN-NEXT: v_alignbit_b32 v6, v15, v24, 16
-; GCN-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:4
-; GCN-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64
+; GCN-NEXT: buffer_load_dword v8, off, s[0:3], s32
+; GCN-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:32
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; GCN-NEXT: v_alignbit_b32 v3, v1, v6, 16
+; GCN-NEXT: v_alignbit_b32 v2, v2, v4, 16
+; GCN-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:28
+; GCN-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:24
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v12
+; GCN-NEXT: v_alignbit_b32 v1, v1, v13, 16
+; GCN-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:20
+; GCN-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:16
+; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v14
+; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v18
+; GCN-NEXT: v_lshrrev_b32_e32 v13, 16, v20
+; GCN-NEXT: v_alignbit_b32 v0, v4, v0, 16
+; GCN-NEXT: v_alignbit_b32 v6, v5, v19, 16
+; GCN-NEXT: v_alignbit_b32 v5, v13, v21, 16
+; GCN-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:12
+; GCN-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:8
+; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v22
+; GCN-NEXT: v_alignbit_b32 v4, v4, v23, 16
+; GCN-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:4
+; GCN-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:64
; GCN-NEXT: s_waitcnt vmcnt(9)
-; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: s_waitcnt vmcnt(8)
-; GCN-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: s_waitcnt vmcnt(7)
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v7
; GCN-NEXT: s_waitcnt vmcnt(6)
-; GCN-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; GCN-NEXT: v_alignbit_b32 v9, v9, v30, 16
-; GCN-NEXT: v_alignbit_b32 v13, v10, v11, 16
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v10
; GCN-NEXT: s_waitcnt vmcnt(5)
-; GCN-NEXT: v_alignbit_b32 v12, v12, v16, 16
-; GCN-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:60
-; GCN-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:56
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v11
+; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT: v_lshrrev_b32_e32 v21, 16, v7
+; GCN-NEXT: v_alignbit_b32 v7, v8, v15, 16
+; GCN-NEXT: v_alignbit_b32 v11, v9, v20, 16
+; GCN-NEXT: v_alignbit_b32 v10, v21, v10, 16
+; GCN-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:60
+; GCN-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:56
; GCN-NEXT: s_waitcnt vmcnt(6)
-; GCN-NEXT: v_lshrrev_b32_e32 v11, 16, v19
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v12
; GCN-NEXT: s_waitcnt vmcnt(5)
-; GCN-NEXT: v_alignbit_b32 v11, v11, v0, 16
-; GCN-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:52
-; GCN-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:48
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v13
+; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GCN-NEXT: v_alignbit_b32 v9, v8, v9, 16
+; GCN-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52
+; GCN-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:48
; GCN-NEXT: s_waitcnt vmcnt(6)
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v14
+; GCN-NEXT: s_waitcnt vmcnt(5)
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v18
+; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GCN-NEXT: v_alignbit_b32 v8, v8, v14, 16
+; GCN-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:44
+; GCN-NEXT: s_waitcnt vmcnt(5)
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v19
; GCN-NEXT: s_waitcnt vmcnt(4)
-; GCN-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_lshrrev_b32_e32 v20, 16, v10
-; GCN-NEXT: v_alignbit_b32 v10, v1, v14, 16
-; GCN-NEXT: v_alignbit_b32 v22, v15, v16, 16
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_alignbit_b32 v21, v20, v0, 16
-; GCN-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:44
-; GCN-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:40
-; GCN-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:36
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: s_waitcnt vmcnt(3)
-; GCN-NEXT: v_lshrrev_b32_e32 v15, 16, v19
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v20
; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_alignbit_b32 v20, v15, v0, 16
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GCN-NEXT: v_lshrrev_b32_e32 v19, 16, v19
+; GCN-NEXT: v_alignbit_b32 v15, v14, v15, 16
+; GCN-NEXT: v_alignbit_b32 v14, v19, v12, 16
+; GCN-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:40
+; GCN-NEXT: s_waitcnt vmcnt(2)
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GCN-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:36
+; GCN-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GCN-NEXT: v_alignbit_b32 v13, v13, v18, 16
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v19, v0, v14, 16
-; GCN-NEXT: buffer_store_dwordx4 v[19:22], v[17:18], s[4:7], 0 addr64 offset:80
-; GCN-NEXT: buffer_store_dwordx4 v[10:13], v[17:18], s[4:7], 0 addr64 offset:64
-; GCN-NEXT: buffer_store_dwordx4 v[6:9], v[17:18], s[4:7], 0 addr64 offset:48
-; GCN-NEXT: buffer_store_dwordx4 v[2:5], v[17:18], s[4:7], 0 addr64
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v19
+; GCN-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GCN-NEXT: v_alignbit_b32 v12, v12, v18, 16
+; GCN-NEXT: buffer_store_dwordx4 v[12:15], v[16:17], s[4:7], 0 addr64 offset:80
+; GCN-NEXT: buffer_store_dwordx4 v[8:11], v[16:17], s[4:7], 0 addr64 offset:64
+; GCN-NEXT: buffer_store_dwordx4 v[4:7], v[16:17], s[4:7], 0 addr64 offset:48
+; GCN-NEXT: buffer_store_dwordx4 v[0:3], v[16:17], s[4:7], 0 addr64
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_store_global_v64bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:128
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124
+; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:120
+; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:116
+; GFX7-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:112
+; GFX7-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:108
+; GFX7-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:104
+; GFX7-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:100
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v5
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_alignbit_b32 v5, v7, v6, 16
-; GFX7-NEXT: v_alignbit_b32 v4, v31, v4, 16
; GFX7-NEXT: v_alignbit_b32 v3, v3, v2, 16
; GFX7-NEXT: v_alignbit_b32 v2, v1, v0, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v15
-; GFX7-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:128
-; GFX7-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:124
-; GFX7-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:120
-; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:116
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112
-; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:108
-; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:104
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v13
-; GFX7-NEXT: v_alignbit_b32 v13, v0, v14, 16
-; GFX7-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:100
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; GFX7-NEXT: v_alignbit_b32 v12, v1, v12, 16
-; GFX7-NEXT: v_alignbit_b32 v11, v0, v10, 16
-; GFX7-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:136
-; GFX7-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
-; GFX7-NEXT: buffer_load_dword v35, off, s[0:3], s32
-; GFX7-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX7-NEXT: v_lshrrev_b32_e32 v21, 16, v21
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; GFX7-NEXT: v_alignbit_b32 v10, v9, v8, 16
-; GFX7-NEXT: v_alignbit_b32 v8, v21, v20, 16
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: v_alignbit_b32 v9, v23, v22, 16
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: s_waitcnt vmcnt(10)
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: s_waitcnt vmcnt(9)
-; GFX7-NEXT: v_alignbit_b32 v23, v6, v7, 16
-; GFX7-NEXT: s_waitcnt vmcnt(8)
-; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v15
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v14
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: s_waitcnt vmcnt(7)
-; GFX7-NEXT: v_alignbit_b32 v22, v15, v31, 16
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GFX7-NEXT: s_waitcnt vmcnt(6)
-; GFX7-NEXT: v_lshrrev_b32_e32 v20, 16, v32
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v31
; GFX7-NEXT: s_waitcnt vmcnt(5)
-; GFX7-NEXT: v_alignbit_b32 v21, v20, v33, 16
-; GFX7-NEXT: s_waitcnt vmcnt(4)
-; GFX7-NEXT: v_lshrrev_b32_e32 v32, 16, v34
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v19
-; GFX7-NEXT: v_alignbit_b32 v7, v6, v18, 16
+; GFX7-NEXT: v_mul_f32_e32 v33, 1.0, v33
+; GFX7-NEXT: v_alignbit_b32 v36, v31, v32, 16
; GFX7-NEXT: s_waitcnt vmcnt(3)
-; GFX7-NEXT: v_alignbit_b32 v20, v32, v14, 16
-; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:32
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28
-; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:24
-; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:20
-; GFX7-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:16
-; GFX7-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:12
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX7-NEXT: s_waitcnt vmcnt(7)
-; GFX7-NEXT: buffer_store_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:112
-; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:96
-; GFX7-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:92
-; GFX7-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:88
-; GFX7-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:84
-; GFX7-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:80
-; GFX7-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:76
-; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v29
-; GFX7-NEXT: v_alignbit_b32 v6, v6, v16, 16
-; GFX7-NEXT: v_alignbit_b32 v16, v15, v28, 16
-; GFX7-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:72
-; GFX7-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:68
-; GFX7-NEXT: s_waitcnt vmcnt(14)
-; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v35
-; GFX7-NEXT: v_alignbit_b32 v17, v14, v30, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v27
-; GFX7-NEXT: v_alignbit_b32 v15, v14, v26, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v25
-; GFX7-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:8
-; GFX7-NEXT: v_alignbit_b32 v14, v14, v24, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v19, 16, v31
-; GFX7-NEXT: s_waitcnt vmcnt(14)
-; GFX7-NEXT: v_alignbit_b32 v21, v19, v32, 16
-; GFX7-NEXT: s_waitcnt vmcnt(13)
-; GFX7-NEXT: v_lshrrev_b32_e32 v19, 16, v33
-; GFX7-NEXT: s_waitcnt vmcnt(12)
-; GFX7-NEXT: v_alignbit_b32 v20, v19, v34, 16
-; GFX7-NEXT: s_waitcnt vmcnt(11)
-; GFX7-NEXT: v_lshrrev_b32_e32 v19, 16, v36
-; GFX7-NEXT: s_waitcnt vmcnt(10)
-; GFX7-NEXT: v_alignbit_b32 v19, v19, v37, 16
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v37
+; GFX7-NEXT: v_mul_f32_e32 v34, 1.0, v34
+; GFX7-NEXT: v_lshrrev_b32_e32 v33, 16, v33
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v38
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; GFX7-NEXT: v_alignbit_b32 v35, v33, v34, 16
+; GFX7-NEXT: v_alignbit_b32 v34, v31, v32, 16
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v39
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v48
+; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; GFX7-NEXT: v_alignbit_b32 v33, v31, v32, 16
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:136
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:132
+; GFX7-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:96
+; GFX7-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:92
+; GFX7-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:88
+; GFX7-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:84
+; GFX7-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:80
+; GFX7-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:76
+; GFX7-NEXT: s_waitcnt vmcnt(6)
+; GFX7-NEXT: buffer_store_dwordx4 v[33:36], v[31:32], s[4:7], 0 addr64 offset:112
+; GFX7-NEXT: s_nop 0
+; GFX7-NEXT: v_lshrrev_b32_e32 v33, 16, v5
+; GFX7-NEXT: v_alignbit_b32 v5, v7, v6, 16
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v13
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v12
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v12, v6, v7, 16
+; GFX7-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72
+; GFX7-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:68
+; GFX7-NEXT: buffer_load_dword v15, off, s[0:3], s32
+; GFX7-NEXT: v_alignbit_b32 v13, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v11
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v10
+; GFX7-NEXT: v_alignbit_b32 v11, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v9
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v8
+; GFX7-NEXT: v_alignbit_b32 v10, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v23
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v22
+; GFX7-NEXT: v_alignbit_b32 v9, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v21
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v20
+; GFX7-NEXT: v_alignbit_b32 v8, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v19
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v18
+; GFX7-NEXT: v_alignbit_b32 v7, v0, v1, 16
+; GFX7-NEXT: s_waitcnt vmcnt(9)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v37
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v28
+; GFX7-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:64
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: s_waitcnt vmcnt(9)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v38
+; GFX7-NEXT: v_alignbit_b32 v4, v33, v4, 16
; GFX7-NEXT: s_waitcnt vmcnt(8)
-; GFX7-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; GFX7-NEXT: s_waitcnt vmcnt(7)
-; GFX7-NEXT: v_alignbit_b32 v25, v22, v23, 16
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v39
+; GFX7-NEXT: v_alignbit_b32 v36, v0, v1, 16
; GFX7-NEXT: s_waitcnt vmcnt(6)
-; GFX7-NEXT: v_lshrrev_b32_e32 v24, 16, v38
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v49
+; GFX7-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v48
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt vmcnt(5)
-; GFX7-NEXT: v_alignbit_b32 v24, v24, v18, 16
-; GFX7-NEXT: s_waitcnt vmcnt(4)
-; GFX7-NEXT: v_lshrrev_b32_e32 v18, 16, v39
-; GFX7-NEXT: s_waitcnt vmcnt(3)
-; GFX7-NEXT: v_alignbit_b32 v23, v18, v48, 16
-; GFX7-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:4
-; GFX7-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:64
-; GFX7-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:60
-; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:56
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52
-; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:48
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v50
+; GFX7-NEXT: v_alignbit_b32 v35, v18, v19, 16
+; GFX7-NEXT: v_alignbit_b32 v34, v0, v1, 16
+; GFX7-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:32
+; GFX7-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:28
+; GFX7-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:24
+; GFX7-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:20
+; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:16
+; GFX7-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:12
; GFX7-NEXT: s_waitcnt vmcnt(8)
-; GFX7-NEXT: v_lshrrev_b32_e32 v22, 16, v28
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v33, v6, v14, 16
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v17
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v16
+; GFX7-NEXT: v_alignbit_b32 v6, v6, v14, 16
; GFX7-NEXT: s_waitcnt vmcnt(7)
-; GFX7-NEXT: v_alignbit_b32 v22, v22, v29, 16
-; GFX7-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:44
-; GFX7-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:40
-; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:36
-; GFX7-NEXT: buffer_store_dwordx4 v[22:25], v[0:1], s[4:7], 0 addr64 offset:96
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v15
+; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v30
+; GFX7-NEXT: buffer_store_dwordx4 v[33:36], v[31:32], s[4:7], 0 addr64 offset:96
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v29
+; GFX7-NEXT: v_alignbit_b32 v17, v14, v15, 16
+; GFX7-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:52
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v27
+; GFX7-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:48
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v26
+; GFX7-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:44
+; GFX7-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:8
+; GFX7-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:4
+; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60
+; GFX7-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:56
+; GFX7-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:40
+; GFX7-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36
+; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX7-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX7-NEXT: v_alignbit_b32 v15, v14, v15, 16
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v25
+; GFX7-NEXT: v_alignbit_b32 v16, v16, v20, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v24
+; GFX7-NEXT: v_alignbit_b32 v14, v14, v20, 16
+; GFX7-NEXT: s_waitcnt vmcnt(14)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_alignbit_b32 v21, v0, v1, 16
+; GFX7-NEXT: s_waitcnt vmcnt(13)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v18
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: s_waitcnt vmcnt(12)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v19
+; GFX7-NEXT: v_alignbit_b32 v20, v0, v1, 16
+; GFX7-NEXT: s_waitcnt vmcnt(11)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v22
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt vmcnt(10)
-; GFX7-NEXT: v_lshrrev_b32_e32 v22, 16, v26
-; GFX7-NEXT: s_waitcnt vmcnt(9)
-; GFX7-NEXT: v_alignbit_b32 v18, v22, v18, 16
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v23
+; GFX7-NEXT: v_alignbit_b32 v19, v0, v1, 16
; GFX7-NEXT: s_waitcnt vmcnt(8)
-; GFX7-NEXT: v_lshrrev_b32_e32 v22, 16, v27
-; GFX7-NEXT: s_waitcnt vmcnt(7)
-; GFX7-NEXT: v_alignbit_b32 v25, v22, v30, 16
-; GFX7-NEXT: s_waitcnt vmcnt(6)
-; GFX7-NEXT: v_lshrrev_b32_e32 v23, 16, v31
+; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v35
; GFX7-NEXT: s_waitcnt vmcnt(5)
-; GFX7-NEXT: v_alignbit_b32 v24, v23, v32, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v29
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt vmcnt(4)
-; GFX7-NEXT: v_lshrrev_b32_e32 v22, 16, v33
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v30
+; GFX7-NEXT: v_alignbit_b32 v18, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v28
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_waitcnt vmcnt(3)
-; GFX7-NEXT: v_alignbit_b32 v23, v22, v28, 16
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v33
; GFX7-NEXT: s_waitcnt vmcnt(2)
-; GFX7-NEXT: v_lshrrev_b32_e32 v22, 16, v29
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v34
+; GFX7-NEXT: v_alignbit_b32 v25, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v27
+; GFX7-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v26
+; GFX7-NEXT: v_alignbit_b32 v24, v22, v23, 16
+; GFX7-NEXT: v_alignbit_b32 v23, v0, v1, 16
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_alignbit_b32 v22, v22, v34, 16
-; GFX7-NEXT: buffer_store_dwordx4 v[22:25], v[0:1], s[4:7], 0 addr64 offset:80
-; GFX7-NEXT: buffer_store_dwordx4 v[18:21], v[0:1], s[4:7], 0 addr64 offset:64
-; GFX7-NEXT: buffer_store_dwordx4 v[14:17], v[0:1], s[4:7], 0 addr64 offset:48
-; GFX7-NEXT: buffer_store_dwordx4 v[6:9], v[0:1], s[4:7], 0 addr64 offset:32
-; GFX7-NEXT: buffer_store_dwordx4 v[10:13], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX7-NEXT: buffer_store_dwordx4 v[2:5], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v36
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v37
+; GFX7-NEXT: v_alignbit_b32 v22, v0, v1, 16
+; GFX7-NEXT: buffer_store_dwordx4 v[22:25], v[31:32], s[4:7], 0 addr64 offset:80
+; GFX7-NEXT: buffer_store_dwordx4 v[18:21], v[31:32], s[4:7], 0 addr64 offset:64
+; GFX7-NEXT: buffer_store_dwordx4 v[14:17], v[31:32], s[4:7], 0 addr64 offset:48
+; GFX7-NEXT: buffer_store_dwordx4 v[6:9], v[31:32], s[4:7], 0 addr64 offset:32
+; GFX7-NEXT: buffer_store_dwordx4 v[10:13], v[31:32], s[4:7], 0 addr64 offset:16
+; GFX7-NEXT: buffer_store_dwordx4 v[2:5], v[31:32], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -1895,6 +2154,7 @@ define void @test_load_store_f32_to_bf16(ptr addrspace(1) %in, ptr addrspace(1)
; GCN-NEXT: s_mov_b32 s5, s6
; GCN-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GCN-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
@@ -1909,6 +2169,7 @@ define void @test_load_store_f32_to_bf16(ptr addrspace(1) %in, ptr addrspace(1)
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: buffer_store_short v0, v[2:3], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -1919,6 +2180,13 @@ define void @test_load_store_f32_to_bf16(ptr addrspace(1) %in, ptr addrspace(1)
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -1928,7 +2196,14 @@ define void @test_load_store_f32_to_bf16(ptr addrspace(1) %in, ptr addrspace(1)
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v0, v[0:1], off
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc
; GFX9-NEXT: global_store_short_d16_hi v[2:3], v0, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1937,7 +2212,13 @@ define void @test_load_store_f32_to_bf16(ptr addrspace(1) %in, ptr addrspace(1)
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v0, v[0:1], off
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo
; GFX10-NEXT: global_store_short_d16_hi v[2:3], v0, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -1945,7 +2226,14 @@ define void @test_load_store_f32_to_bf16(ptr addrspace(1) %in, ptr addrspace(1)
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo
; GFX11-NEXT: global_store_d16_hi_b16 v[2:3], v0, off
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val = load float, ptr addrspace(1) %in
@@ -1990,7 +2278,24 @@ define void @test_load_store_f64_to_bf16(ptr addrspace(1) %in, ptr addrspace(1)
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX8-NEXT: v_cvt_f32_f64_e64 v6, |v[0:1]|
+; GFX8-NEXT: v_and_b32_e32 v7, 0x80000000, v1
+; GFX8-NEXT: v_cvt_f64_f32_e32 v[4:5], v6
+; GFX8-NEXT: v_and_b32_e32 v6, 1, v6
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX8-NEXT: v_cmp_nlg_f64_e64 s[4:5], |v[0:1]|, v[4:5]
+; GFX8-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[0:1]|, v[4:5]
+; GFX8-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v4, -1, 1, s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, 0, s[4:5]
+; GFX8-NEXT: v_or_b32_e32 v5, v4, v7
+; GFX8-NEXT: v_bfe_u32 v4, v4, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v5
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v5
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: flat_store_short v[2:3], v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -2000,8 +2305,25 @@ define void @test_load_store_f64_to_bf16(ptr addrspace(1) %in, ptr addrspace(1)
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT: s_brev_b32 s8, 1
+; GFX9-NEXT: s_movk_i32 s9, 0x7fff
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX9-NEXT: v_cvt_f32_f64_e64 v6, |v[0:1]|
+; GFX9-NEXT: v_cvt_f64_f32_e32 v[4:5], v6
+; GFX9-NEXT: v_and_b32_e32 v6, 1, v6
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX9-NEXT: v_cmp_nlg_f64_e64 s[4:5], |v[0:1]|, v[4:5]
+; GFX9-NEXT: v_cmp_gt_f64_e64 s[6:7], |v[0:1]|, v[4:5]
+; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX9-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v4, -1, 1, s[6:7]
+; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, 0, s[4:5]
+; GFX9-NEXT: v_and_or_b32 v5, v1, s8, v4
+; GFX9-NEXT: v_bfe_u32 v4, v4, 16, 1
+; GFX9-NEXT: v_add3_u32 v4, v4, v5, s9
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v5
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
; GFX9-NEXT: global_store_short_d16_hi v[2:3], v0, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -2011,7 +2333,22 @@ define void @test_load_store_f64_to_bf16(ptr addrspace(1) %in, ptr addrspace(1)
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX10-NEXT: v_cvt_f32_f64_e64 v6, |v[0:1]|
+; GFX10-NEXT: v_cvt_f64_f32_e32 v[4:5], v6
+; GFX10-NEXT: v_and_b32_e32 v6, 1, v6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
+; GFX10-NEXT: v_cmp_nlg_f64_e64 s4, |v[0:1]|, v[4:5]
+; GFX10-NEXT: v_cmp_gt_f64_e64 s5, |v[0:1]|, v[4:5]
+; GFX10-NEXT: s_or_b32 s4, s4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[0:1]
+; GFX10-NEXT: v_cndmask_b32_e64 v4, -1, 1, s5
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, 0, s4
+; GFX10-NEXT: s_mov_b32 s4, 0x400000
+; GFX10-NEXT: v_and_or_b32 v5, 0x80000000, v1, v4
+; GFX10-NEXT: v_bfe_u32 v4, v4, 16, 1
+; GFX10-NEXT: v_add3_u32 v4, v4, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v5, 0x80000000, v5, s4
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
; GFX10-NEXT: global_store_short_d16_hi v[2:3], v0, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -2020,7 +2357,27 @@ define void @test_load_store_f64_to_bf16(ptr addrspace(1) %in, ptr addrspace(1)
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX11-NEXT: v_cvt_f32_f64_e64 v6, |v[0:1]|
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cvt_f64_f32_e32 v[4:5], v6
+; GFX11-NEXT: v_and_b32_e32 v6, 1, v6
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_nlg_f64_e64 s0, |v[0:1]|, v[4:5]
+; GFX11-NEXT: v_cmp_gt_f64_e64 s1, |v[0:1]|, v[4:5]
+; GFX11-NEXT: s_or_b32 s0, s0, vcc_lo
+; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v4, -1, 1, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v4, 0, s0
+; GFX11-NEXT: s_mov_b32 s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, 0x80000000, v1, v4
+; GFX11-NEXT: v_bfe_u32 v4, v4, 16, 1
+; GFX11-NEXT: v_add3_u32 v4, v4, v5, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v5, 0x80000000, v5, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
; GFX11-NEXT: global_store_d16_hi_b16 v[2:3], v0, off
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val = load double, ptr addrspace(1) %in
@@ -2468,11 +2825,12 @@ define void @test_arg_store(bfloat %in, ptr addrspace(1) %out) {
; GCN-LABEL: test_arg_store:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, 0
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, s6
; GCN-NEXT: s_mov_b32 s5, s6
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GCN-NEXT: buffer_store_short v0, v[1:2], s[4:7], 0 addr64
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -2481,10 +2839,11 @@ define void @test_arg_store(bfloat %in, ptr addrspace(1) %out) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: buffer_store_short v0, v[1:2], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -2522,8 +2881,10 @@ define void @test_arg_store_v2bf16(<2 x bfloat> %in, ptr addrspace(1) %out) {
; GCN-LABEL: test_arg_store_v2bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s6, 0
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s4, s6
@@ -2535,7 +2896,9 @@ define void @test_arg_store_v2bf16(<2 x bfloat> %in, ptr addrspace(1) %out) {
; GFX7-LABEL: test_arg_store_v2bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX7-NEXT: s_mov_b32 s7, 0xf000
@@ -2578,13 +2941,16 @@ define void @test_arg_store_v3bf16(<3 x bfloat> %in, ptr addrspace(1) %out) {
; GCN-LABEL: test_arg_store_v3bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, 0
-; GCN-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: s_mov_b32 s4, s6
; GCN-NEXT: s_mov_b32 s5, s6
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GCN-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GCN-NEXT: buffer_store_short v2, v[3:4], s[4:7], 0 addr64 offset:4
; GCN-NEXT: buffer_store_dword v0, v[3:4], s[4:7], 0 addr64
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
@@ -2593,13 +2959,16 @@ define void @test_arg_store_v3bf16(<3 x bfloat> %in, ptr addrspace(1) %out) {
; GFX7-LABEL: test_arg_store_v3bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v2
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: buffer_store_short v1, v[3:4], s[4:7], 0 addr64 offset:4
; GFX7-NEXT: buffer_store_dword v0, v[3:4], s[4:7], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -2645,9 +3014,13 @@ define void @test_arg_store_v4bf16(<4 x bfloat> %in, ptr addrspace(1) %out) {
; GCN-LABEL: test_arg_store_v4bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_alignbit_b32 v1, v3, v2, 16
; GCN-NEXT: v_alignbit_b32 v0, v6, v0, 16
; GCN-NEXT: s_mov_b32 s7, 0xf000
@@ -2660,8 +3033,12 @@ define void @test_arg_store_v4bf16(<4 x bfloat> %in, ptr addrspace(1) %out) {
; GFX7-LABEL: test_arg_store_v4bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: v_alignbit_b32 v2, v3, v2, 16
; GFX7-NEXT: v_alignbit_b32 v1, v1, v0, 16
@@ -2707,28 +3084,44 @@ define void @test_arg_store_v8bf16(<8 x bfloat> %in, ptr addrspace(1) %out) {
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, 0
-; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GCN-NEXT: v_lshrrev_b32_e32 v10, 16, v5
-; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, s6
; GCN-NEXT: s_mov_b32 s5, s6
-; GCN-NEXT: v_alignbit_b32 v5, v7, v6, 16
-; GCN-NEXT: v_alignbit_b32 v4, v10, v4, 16
-; GCN-NEXT: v_alignbit_b32 v3, v3, v2, 16
-; GCN-NEXT: v_alignbit_b32 v2, v1, v0, 16
-; GCN-NEXT: buffer_store_dwordx4 v[2:5], v[8:9], s[4:7], 0 addr64
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v7
+; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GCN-NEXT: v_lshrrev_b32_e32 v11, 16, v1
+; GCN-NEXT: v_alignbit_b32 v3, v2, v6, 16
+; GCN-NEXT: v_alignbit_b32 v2, v5, v4, 16
+; GCN-NEXT: v_alignbit_b32 v1, v7, v10, 16
+; GCN-NEXT: v_alignbit_b32 v0, v11, v0, 16
+; GCN-NEXT: buffer_store_dwordx4 v[0:3], v[8:9], s[4:7], 0 addr64
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: test_arg_store_v8bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
@@ -2773,53 +3166,85 @@ define void @test_arg_store_v16bf16(<16 x bfloat> %in, ptr addrspace(1) %out) {
; GCN-LABEL: test_arg_store_v16bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GCN-NEXT: v_lshrrev_b32_e32 v18, 16, v5
-; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, 0
-; GCN-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; GCN-NEXT: v_lshrrev_b32_e32 v19, 16, v13
-; GCN-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GCN-NEXT: v_alignbit_b32 v5, v7, v6, 16
-; GCN-NEXT: v_alignbit_b32 v4, v18, v4, 16
-; GCN-NEXT: v_alignbit_b32 v3, v3, v2, 16
-; GCN-NEXT: v_alignbit_b32 v2, v1, v0, 16
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v15
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT: v_lshrrev_b32_e32 v15, 16, v3
+; GCN-NEXT: v_lshrrev_b32_e32 v19, 16, v1
; GCN-NEXT: s_mov_b32 s4, s6
; GCN-NEXT: s_mov_b32 s5, s6
-; GCN-NEXT: v_alignbit_b32 v13, v15, v14, 16
-; GCN-NEXT: v_alignbit_b32 v12, v19, v12, 16
-; GCN-NEXT: v_alignbit_b32 v11, v11, v10, 16
-; GCN-NEXT: v_alignbit_b32 v10, v9, v8, 16
-; GCN-NEXT: buffer_store_dwordx4 v[10:13], v[16:17], s[4:7], 0 addr64 offset:16
-; GCN-NEXT: buffer_store_dwordx4 v[2:5], v[16:17], s[4:7], 0 addr64
+; GCN-NEXT: v_lshrrev_b32_e32 v20, 16, v2
+; GCN-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GCN-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT: v_alignbit_b32 v3, v7, v6, 16
+; GCN-NEXT: v_alignbit_b32 v2, v5, v4, 16
+; GCN-NEXT: v_alignbit_b32 v1, v15, v18, 16
+; GCN-NEXT: v_alignbit_b32 v0, v19, v0, 16
+; GCN-NEXT: v_alignbit_b32 v7, v20, v14, 16
+; GCN-NEXT: v_alignbit_b32 v6, v13, v12, 16
+; GCN-NEXT: v_alignbit_b32 v5, v11, v10, 16
+; GCN-NEXT: v_alignbit_b32 v4, v9, v8, 16
+; GCN-NEXT: buffer_store_dwordx4 v[4:7], v[16:17], s[4:7], 0 addr64 offset:16
+; GCN-NEXT: buffer_store_dwordx4 v[0:3], v[16:17], s[4:7], 0 addr64
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: test_arg_store_v16bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_alignbit_b32 v5, v5, v4, 16
; GFX7-NEXT: v_alignbit_b32 v4, v3, v2, 16
; GFX7-NEXT: v_alignbit_b32 v3, v1, v0, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v15
-; GFX7-NEXT: v_alignbit_b32 v14, v0, v14, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v13
-; GFX7-NEXT: v_alignbit_b32 v13, v0, v12, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v11
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v15
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v14
+; GFX7-NEXT: v_alignbit_b32 v14, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v13
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v12
+; GFX7-NEXT: v_alignbit_b32 v13, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v11
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v10
+; GFX7-NEXT: v_alignbit_b32 v12, v0, v1, 16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: v_alignbit_b32 v12, v0, v10, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v9
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v8
; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: v_alignbit_b32 v11, v0, v8, 16
+; GFX7-NEXT: v_alignbit_b32 v11, v0, v1, 16
; GFX7-NEXT: v_alignbit_b32 v6, v7, v6, 16
; GFX7-NEXT: buffer_store_dwordx4 v[11:14], v[16:17], s[4:7], 0 addr64 offset:16
; GFX7-NEXT: buffer_store_dwordx4 v[3:6], v[16:17], s[4:7], 0 addr64
@@ -2867,12 +3292,12 @@ define amdgpu_gfx void @test_inreg_arg_store(bfloat inreg %in, ptr addrspace(1)
; GCN-LABEL: test_inreg_arg_store:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: s_lshr_b32 s34, s4, 16
-; GCN-NEXT: s_mov_b32 s38, 0
; GCN-NEXT: s_mov_b32 s39, 0xf000
+; GCN-NEXT: s_mov_b32 s38, 0
+; GCN-NEXT: v_mul_f32_e64 v2, 1.0, s4
; GCN-NEXT: s_mov_b32 s36, s38
; GCN-NEXT: s_mov_b32 s37, s38
-; GCN-NEXT: v_mov_b32_e32 v2, s34
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GCN-NEXT: buffer_store_short v2, v[0:1], s[36:39], 0 addr64
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -2880,12 +3305,12 @@ define amdgpu_gfx void @test_inreg_arg_store(bfloat inreg %in, ptr addrspace(1)
; GFX7-LABEL: test_inreg_arg_store:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s34, s4, 16
; GFX7-NEXT: s_mov_b32 s38, 0
+; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s4
; GFX7-NEXT: s_mov_b32 s39, 0xf000
; GFX7-NEXT: s_mov_b32 s36, s38
; GFX7-NEXT: s_mov_b32 s37, s38
-; GFX7-NEXT: v_mov_b32_e32 v2, s34
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX7-NEXT: buffer_store_short v2, v[0:1], s[36:39], 0 addr64
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -2927,7 +3352,8 @@ define bfloat @test_byval(ptr addrspace(5) byval(bfloat) %bv, bfloat %val) {
; GCN-LABEL: test_byval:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v0
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: buffer_store_short v1, off, s[0:3], s32
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -2935,7 +3361,8 @@ define bfloat @test_byval(ptr addrspace(5) byval(bfloat) %bv, bfloat %val) {
; GFX7-LABEL: test_byval:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: buffer_store_short v1, off, s[0:3], s32
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -2974,6 +3401,7 @@ define void @test_sret(ptr addrspace(5) sret(bfloat) %sret, bfloat %val) {
; GCN-LABEL: test_sret:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: buffer_store_short v1, v0, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
@@ -2982,6 +3410,7 @@ define void @test_sret(ptr addrspace(5) sret(bfloat) %sret, bfloat %val) {
; GFX7-LABEL: test_sret:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: buffer_store_short v1, v0, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -3371,6 +3800,7 @@ define void @test_call(bfloat %in, ptr addrspace(5) %out) {
; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GCN-NEXT: buffer_store_short v0, v1, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
@@ -3401,6 +3831,7 @@ define void @test_call(bfloat %in, ptr addrspace(5) %out) {
; GFX7-NEXT: v_writelane_b32 v2, s31, 1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: buffer_store_short v0, v1, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -3556,9 +3987,11 @@ define void @test_call_v2bf16(<2 x bfloat> %in, ptr addrspace(5) %out) {
; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_add_i32_e32 v3, vcc, 2, v2
; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GCN-NEXT: v_add_i32_e32 v3, vcc, 2, v2
; GCN-NEXT: buffer_store_short v1, v3, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_store_short v0, v2, s[0:3], 0 offen
@@ -3590,6 +4023,8 @@ define void @test_call_v2bf16(<2 x bfloat> %in, ptr addrspace(5) %out) {
; GFX7-NEXT: v_writelane_b32 v4, s31, 1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 2, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
@@ -3749,9 +4184,12 @@ define void @test_call_v3bf16(<3 x bfloat> %in, ptr addrspace(5) %out) {
; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_add_i32_e32 v4, vcc, 4, v3
; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GCN-NEXT: v_add_i32_e32 v4, vcc, 4, v3
; GCN-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GCN-NEXT: buffer_store_short v2, v4, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
@@ -3784,9 +4222,12 @@ define void @test_call_v3bf16(<3 x bfloat> %in, ptr addrspace(5) %out) {
; GFX7-NEXT: v_writelane_b32 v4, s31, 1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_add_i32_e32 v2, vcc, 4, v3
; GFX7-NEXT: buffer_store_short v1, v2, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -3953,13 +4394,17 @@ define void @test_call_v4bf16(<4 x bfloat> %in, ptr addrspace(5) %out) {
; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_add_i32_e32 v5, vcc, 6, v4
+; GCN-NEXT: v_add_i32_e32 v6, vcc, 4, v4
+; GCN-NEXT: v_add_i32_e32 v7, vcc, 2, v4
; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GCN-NEXT: v_add_i32_e32 v5, vcc, 6, v4
-; GCN-NEXT: v_add_i32_e32 v6, vcc, 4, v4
-; GCN-NEXT: v_add_i32_e32 v7, vcc, 2, v4
; GCN-NEXT: buffer_store_short v3, v5, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_store_short v2, v6, s[0:3], 0 offen
@@ -3995,12 +4440,16 @@ define void @test_call_v4bf16(<4 x bfloat> %in, ptr addrspace(5) %out) {
; GFX7-NEXT: v_writelane_b32 v6, s31, 1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_add_i32_e32 v5, vcc, 6, v4
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX7-NEXT: buffer_store_short v3, v5, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 4, v4
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: buffer_store_short v2, v3, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -4169,6 +4618,21 @@ define void @test_call_v8bf16(<8 x bfloat> %in, ptr addrspace(5) %out) {
; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_add_i32_e32 v9, vcc, 14, v8
+; GCN-NEXT: v_add_i32_e32 v10, vcc, 12, v8
+; GCN-NEXT: v_add_i32_e32 v11, vcc, 10, v8
+; GCN-NEXT: v_add_i32_e32 v12, vcc, 8, v8
+; GCN-NEXT: v_add_i32_e32 v13, vcc, 6, v8
+; GCN-NEXT: v_add_i32_e32 v14, vcc, 4, v8
+; GCN-NEXT: v_add_i32_e32 v15, vcc, 2, v8
; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
@@ -4177,13 +4641,6 @@ define void @test_call_v8bf16(<8 x bfloat> %in, ptr addrspace(5) %out) {
; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GCN-NEXT: v_add_i32_e32 v9, vcc, 14, v8
-; GCN-NEXT: v_add_i32_e32 v10, vcc, 12, v8
-; GCN-NEXT: v_add_i32_e32 v11, vcc, 10, v8
-; GCN-NEXT: v_add_i32_e32 v12, vcc, 8, v8
-; GCN-NEXT: v_add_i32_e32 v13, vcc, 6, v8
-; GCN-NEXT: v_add_i32_e32 v14, vcc, 4, v8
-; GCN-NEXT: v_add_i32_e32 v15, vcc, 2, v8
; GCN-NEXT: buffer_store_short v7, v9, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_store_short v6, v10, s[0:3], 0 offen
@@ -4227,28 +4684,36 @@ define void @test_call_v8bf16(<8 x bfloat> %in, ptr addrspace(5) %out) {
; GFX7-NEXT: v_writelane_b32 v10, s31, 1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; GFX7-NEXT: v_add_i32_e32 v9, vcc, 14, v8
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; GFX7-NEXT: buffer_store_short v7, v9, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v7, vcc, 12, v8
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX7-NEXT: buffer_store_short v6, v7, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v6, vcc, 10, v8
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX7-NEXT: buffer_store_short v5, v6, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v5, vcc, 8, v8
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX7-NEXT: buffer_store_short v4, v5, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 6, v8
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX7-NEXT: buffer_store_short v3, v4, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 4, v8
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: buffer_store_short v2, v3, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -4419,85 +4884,101 @@ define void @test_call_v16bf16(<16 x bfloat> %in, ptr addrspace(5) %out) {
; GCN-NEXT: s_mov_b32 s8, s33
; GCN-NEXT: s_mov_b32 s33, s32
; GCN-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GCN-NEXT: buffer_store_dword v28, off, s[0:3], s33 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v21, off, s[0:3], s33 ; 4-byte Folded Spill
; GCN-NEXT: s_mov_b64 exec, s[4:5]
; GCN-NEXT: s_addk_i32 s32, 0x400
; GCN-NEXT: s_waitcnt expcnt(0)
-; GCN-NEXT: v_writelane_b32 v28, s30, 0
-; GCN-NEXT: v_writelane_b32 v28, s31, 1
+; GCN-NEXT: v_writelane_b32 v21, s30, 0
+; GCN-NEXT: v_writelane_b32 v21, s31, 1
; GCN-NEXT: s_getpc_b64 s[4:5]
; GCN-NEXT: s_add_u32 s4, s4, test_arg_store_v2bf16 at gotpcrel32@lo+4
; GCN-NEXT: s_addc_u32 s5, s5, test_arg_store_v2bf16 at gotpcrel32@hi+12
; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_swappc_b64 s[30:31], s[4:5]
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GCN-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; GCN-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; GCN-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; GCN-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; GCN-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; GCN-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: v_add_i32_e32 v17, vcc, 30, v16
; GCN-NEXT: v_add_i32_e32 v18, vcc, 28, v16
; GCN-NEXT: v_add_i32_e32 v19, vcc, 26, v16
; GCN-NEXT: v_add_i32_e32 v20, vcc, 24, v16
-; GCN-NEXT: v_add_i32_e32 v21, vcc, 22, v16
-; GCN-NEXT: v_add_i32_e32 v22, vcc, 20, v16
-; GCN-NEXT: v_add_i32_e32 v23, vcc, 18, v16
-; GCN-NEXT: v_add_i32_e32 v24, vcc, 16, v16
-; GCN-NEXT: v_add_i32_e32 v25, vcc, 14, v16
-; GCN-NEXT: v_add_i32_e32 v26, vcc, 12, v16
-; GCN-NEXT: v_add_i32_e32 v27, vcc, 10, v16
+; GCN-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; GCN-NEXT: buffer_store_short v15, v17, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GCN-NEXT: v_add_i32_e32 v15, vcc, 8, v16
-; GCN-NEXT: v_add_i32_e32 v17, vcc, 6, v16
+; GCN-NEXT: v_add_i32_e32 v15, vcc, 22, v16
+; GCN-NEXT: v_add_i32_e32 v17, vcc, 20, v16
+; GCN-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; GCN-NEXT: buffer_store_short v14, v18, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GCN-NEXT: v_add_i32_e32 v14, vcc, 4, v16
-; GCN-NEXT: v_add_i32_e32 v18, vcc, 2, v16
+; GCN-NEXT: v_add_i32_e32 v14, vcc, 18, v16
+; GCN-NEXT: v_add_i32_e32 v18, vcc, 16, v16
+; GCN-NEXT: v_lshrrev_b32_e32 v13, 16, v13
; GCN-NEXT: buffer_store_short v13, v19, s[0:3], 0 offen
-; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT: v_add_i32_e32 v13, vcc, 14, v16
+; GCN-NEXT: v_add_i32_e32 v19, vcc, 12, v16
+; GCN-NEXT: v_lshrrev_b32_e32 v12, 16, v12
; GCN-NEXT: buffer_store_short v12, v20, s[0:3], 0 offen
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT: v_add_i32_e32 v12, vcc, 10, v16
+; GCN-NEXT: v_add_i32_e32 v20, vcc, 8, v16
+; GCN-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT: buffer_store_short v11, v15, s[0:3], 0 offen
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT: v_add_i32_e32 v11, vcc, 6, v16
+; GCN-NEXT: v_add_i32_e32 v15, vcc, 4, v16
+; GCN-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GCN-NEXT: buffer_store_short v10, v17, s[0:3], 0 offen
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GCN-NEXT: v_add_i32_e32 v10, vcc, 2, v16
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT: buffer_store_short v9, v14, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_store_short v11, v21, s[0:3], 0 offen
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_store_short v10, v22, s[0:3], 0 offen
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_store_short v9, v23, s[0:3], 0 offen
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_store_short v8, v24, s[0:3], 0 offen
+; GCN-NEXT: buffer_store_short v8, v18, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_store_short v7, v25, s[0:3], 0 offen
+; GCN-NEXT: buffer_store_short v7, v13, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_store_short v6, v26, s[0:3], 0 offen
+; GCN-NEXT: buffer_store_short v6, v19, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_store_short v5, v27, s[0:3], 0 offen
+; GCN-NEXT: buffer_store_short v5, v12, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_store_short v4, v15, s[0:3], 0 offen
+; GCN-NEXT: buffer_store_short v4, v20, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_store_short v3, v17, s[0:3], 0 offen
+; GCN-NEXT: buffer_store_short v3, v11, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_store_short v2, v14, s[0:3], 0 offen
+; GCN-NEXT: buffer_store_short v2, v15, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_store_short v1, v18, s[0:3], 0 offen
+; GCN-NEXT: buffer_store_short v1, v10, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_store_short v0, v16, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_readlane_b32 s31, v28, 1
-; GCN-NEXT: v_readlane_b32 s30, v28, 0
+; GCN-NEXT: v_readlane_b32 s31, v21, 1
+; GCN-NEXT: v_readlane_b32 s30, v21, 0
; GCN-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GCN-NEXT: buffer_load_dword v28, off, s[0:3], s33 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v21, off, s[0:3], s33 ; 4-byte Folded Reload
; GCN-NEXT: s_mov_b64 exec, s[4:5]
; GCN-NEXT: s_addk_i32 s32, 0xfc00
; GCN-NEXT: s_mov_b32 s33, s8
@@ -4521,60 +5002,76 @@ define void @test_call_v16bf16(<16 x bfloat> %in, ptr addrspace(5) %out) {
; GFX7-NEXT: v_writelane_b32 v18, s31, 1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; GFX7-NEXT: v_add_i32_e32 v17, vcc, 30, v16
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; GFX7-NEXT: buffer_store_short v15, v17, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v15, vcc, 28, v16
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GFX7-NEXT: v_lshrrev_b32_e32 v13, 16, v13
; GFX7-NEXT: buffer_store_short v14, v15, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v14, vcc, 26, v16
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v12
; GFX7-NEXT: buffer_store_short v13, v14, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v13, vcc, 24, v16
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GFX7-NEXT: v_lshrrev_b32_e32 v11, 16, v11
; GFX7-NEXT: buffer_store_short v12, v13, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v12, vcc, 22, v16
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; GFX7-NEXT: buffer_store_short v11, v12, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v11, vcc, 20, v16
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GFX7-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; GFX7-NEXT: buffer_store_short v10, v11, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v10, vcc, 18, v16
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; GFX7-NEXT: buffer_store_short v9, v10, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v9, vcc, 16, v16
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; GFX7-NEXT: buffer_store_short v8, v9, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v8, vcc, 14, v16
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; GFX7-NEXT: buffer_store_short v7, v8, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v7, vcc, 12, v16
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX7-NEXT: buffer_store_short v6, v7, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v6, vcc, 10, v16
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX7-NEXT: buffer_store_short v5, v6, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v5, vcc, 8, v16
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX7-NEXT: buffer_store_short v4, v5, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v4, vcc, 6, v16
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX7-NEXT: buffer_store_short v3, v4, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v3, vcc, 4, v16
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: buffer_store_short v2, v3, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -4772,6 +5269,7 @@ define bfloat @test_alloca_load_store_ret(bfloat %in) {
; GCN-LABEL: test_alloca_load_store_ret:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GCN-NEXT: buffer_store_short v0, off, s[0:3], s32
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
@@ -4783,6 +5281,7 @@ define bfloat @test_alloca_load_store_ret(bfloat %in) {
; GFX7-LABEL: test_alloca_load_store_ret:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: buffer_store_short v0, off, s[0:3], s32
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -4870,7 +5369,7 @@ define { <32 x i32>, bfloat } @test_overflow_stack(bfloat %a, <32 x i32> %b) {
; GCN-NEXT: s_waitcnt expcnt(0)
; GCN-NEXT: v_add_i32_e32 v27, vcc, 0x50, v0
; GCN-NEXT: v_add_i32_e32 v30, vcc, 0x4c, v0
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: buffer_store_dword v26, v29, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt expcnt(0)
; GCN-NEXT: v_add_i32_e32 v26, vcc, 0x48, v0
@@ -4908,6 +5407,7 @@ define { <32 x i32>, bfloat } @test_overflow_stack(bfloat %a, <32 x i32> %b) {
; GCN-NEXT: v_add_i32_e32 v18, vcc, 8, v0
; GCN-NEXT: v_add_i32_e32 v25, vcc, 4, v0
; GCN-NEXT: v_add_i32_e32 v0, vcc, 0x80, v0
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: buffer_store_dword v17, v31, s[0:3], 0 offen
; GCN-NEXT: buffer_store_dword v16, v2, s[0:3], 0 offen
; GCN-NEXT: buffer_store_dword v15, v24, s[0:3], 0 offen
@@ -4933,6 +5433,7 @@ define { <32 x i32>, bfloat } @test_overflow_stack(bfloat %a, <32 x i32> %b) {
; GFX7-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GFX7-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:8
; GFX7-NEXT: v_add_i32_e32 v31, vcc, 0x7c, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_store_dword v2, v31, s[0:3], 0 offen
@@ -8466,6 +8967,8 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fadd_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_add_f32_e32 v0, v0, v1
@@ -8475,6 +8978,8 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fadd_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_add_f32_e32 v0, v0, v1
@@ -8487,6 +8992,13 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) {
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -8496,6 +9008,13 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) {
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -8504,7 +9023,13 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -8513,8 +9038,16 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fadd bfloat %a, %b
@@ -8525,6 +9058,10 @@ define <2 x bfloat> @v_fadd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_fadd_v2bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -8538,6 +9075,10 @@ define <2 x bfloat> @v_fadd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX7-LABEL: v_fadd_v2bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -8553,10 +9094,24 @@ define <2 x bfloat> @v_fadd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX8-NEXT: v_add_f32_e32 v2, v3, v2
+; GFX8-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v2
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v2
; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX8-NEXT: v_add_f32_e32 v2, v3, v2
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v2, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -8566,10 +9121,23 @@ define <2 x bfloat> @v_fadd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX9-NEXT: v_add_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_add_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_add3_u32 v3, v3, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v2, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -8581,9 +9149,20 @@ define <2 x bfloat> @v_fadd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_add_f32_e32 v2, v3, v2
; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_and_or_b32 v5, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fadd_v2bf16:
@@ -8593,11 +9172,24 @@ define <2 x bfloat> @v_fadd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
; GFX11-NEXT: v_add_f32_e32 v2, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v2, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_and_or_b32 v5, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fadd <2 x bfloat> %a, %b
ret <2 x bfloat> %op
@@ -8607,6 +9199,12 @@ define <3 x bfloat> @v_fadd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GCN-LABEL: v_fadd_v3bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
@@ -8624,6 +9222,12 @@ define <3 x bfloat> @v_fadd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX7-LABEL: v_fadd_v3bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
@@ -8644,12 +9248,34 @@ define <3 x bfloat> @v_fadd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v3
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
@@ -8661,12 +9287,31 @@ define <3 x bfloat> @v_fadd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_add3_u32 v4, v4, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16
@@ -8675,16 +9320,32 @@ define <3 x bfloat> @v_fadd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX10-LABEL: v_fadd_v3bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_add_f32_e32 v4, v5, v4
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
; GFX10-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
+; GFX10-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v4, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v6, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = fadd <3 x bfloat> %a, %b
@@ -8695,6 +9356,14 @@ define <4 x bfloat> @v_fadd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GCN-LABEL: v_fadd_v4bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
@@ -8716,6 +9385,14 @@ define <4 x bfloat> @v_fadd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX7-LABEL: v_fadd_v4bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
@@ -8739,17 +9416,46 @@ define <4 x bfloat> @v_fadd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX8-NEXT: v_add_f32_e32 v4, v5, v4
+; GFX8-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v4
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_add_f32_e32 v4, v5, v4
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX8-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v3
; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16
@@ -8760,16 +9466,41 @@ define <4 x bfloat> @v_fadd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX9-NEXT: v_add_f32_e32 v4, v5, v4
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_add_f32_e32 v4, v5, v4
+; GFX9-NEXT: v_add3_u32 v5, v5, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; GFX9-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX9-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4
@@ -8781,17 +9512,38 @@ define <4 x bfloat> @v_fadd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX10-NEXT: v_add_f32_e32 v4, v5, v4
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_add_f32_e32 v4, v5, v4
+; GFX10-NEXT: v_add_f32_e32 v1, v1, v3
; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
+; GFX10-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX10-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
-; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX10-NEXT: v_and_or_b32 v6, v4, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v9, v1, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v6, v5, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc_lo
+; GFX10-NEXT: v_perm_b32 v1, v1, v3, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fadd_v4bf16:
@@ -8799,19 +9551,45 @@ define <4 x bfloat> @v_fadd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_add_f32 v0, v0, v2 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_dual_add_f32 v4, v5, v4 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_add_f32 v0, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-NEXT: v_add_f32_e32 v4, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v3, v4, 16, 1
; GFX11-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX11-NEXT: v_and_or_b32 v6, v4, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_and_or_b32 v9, v1, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v6, v5, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v8, v0, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc_lo
+; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
+; GFX11-NEXT: v_perm_b32 v1, v1, v3, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fadd <4 x bfloat> %a, %b
ret <4 x bfloat> %op
@@ -8821,6 +9599,22 @@ define <8 x bfloat> @v_fadd_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GCN-LABEL: v_fadd_v8bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
@@ -8858,6 +9652,22 @@ define <8 x bfloat> @v_fadd_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX7-LABEL: v_fadd_v8bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
@@ -8897,31 +9707,88 @@ define <8 x bfloat> @v_fadd_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX8-NEXT: v_add_f32_e32 v8, v9, v8
+; GFX8-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v8
; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_add_f32_e32 v8, v9, v8
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v8
; GFX8-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, s4, v7
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX8-NEXT: v_add_f32_e32 v7, v9, v7
+; GFX8-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v7
; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: v_add_f32_e32 v7, v9, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v7
; GFX8-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, s4, v6
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v6, v9, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v5
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX8-NEXT: v_add_f32_e32 v6, v9, v6
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_add_f32_e32 v6, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v6
; GFX8-NEXT: v_add_f32_e32 v1, v1, v5
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX8-NEXT: v_add_f32_e32 v5, v9, v5
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v5
; GFX8-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v9, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_add_f32_e32 v5, v9, v5
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v5, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v6, 16
@@ -8934,28 +9801,77 @@ define <8 x bfloat> @v_fadd_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX9-NEXT: v_add_f32_e32 v8, v9, v8
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v8
; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_add_f32_e32 v8, v9, v8
+; GFX9-NEXT: v_add3_u32 v9, v9, v8, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX9-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v7, v7, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX9-NEXT: v_add_f32_e32 v7, v9, v7
+; GFX9-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v7
; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_add_f32_e32 v7, v9, v7
+; GFX9-NEXT: v_add3_u32 v9, v9, v7, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v9, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v5
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX9-NEXT: v_add_f32_e32 v6, v9, v6
+; GFX9-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v6
; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_add_f32_e32 v6, v9, v6
+; GFX9-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX9-NEXT: v_add_f32_e32 v1, v1, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v5, v5, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX9-NEXT: v_add_f32_e32 v5, v9, v5
+; GFX9-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v5
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_add_f32_e32 v5, v9, v5
+; GFX9-NEXT: v_add3_u32 v9, v9, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v9, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v5, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v6, s4
@@ -8968,65 +9884,157 @@ define <8 x bfloat> @v_fadd_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v6
-; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_add_f32_e32 v8, v9, v8
+; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v6
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v7
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX10-NEXT: v_add_f32_e32 v9, v11, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v8, s4, 0x400000
+; GFX10-NEXT: v_add_f32_e32 v9, v11, v9
+; GFX10-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX10-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX10-NEXT: v_bfe_u32 v8, v9, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v5
+; GFX10-NEXT: v_and_or_b32 v12, v9, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX10-NEXT: v_add3_u32 v10, v11, v3, 0x7fff
; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX10-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX10-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v0
+; GFX10-NEXT: v_add_f32_e32 v6, v11, v6
+; GFX10-NEXT: v_add3_u32 v9, v13, v2, 0x7fff
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v11, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v12, v6, 16, 1
; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_add_f32_e32 v10, v11, v10
-; GFX10-NEXT: v_add_f32_e32 v11, v13, v12
-; GFX10-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX10-NEXT: v_add_f32_e32 v1, v1, v5
-; GFX10-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX10-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX10-NEXT: v_perm_b32 v0, v0, v11, 0x7060302
-; GFX10-NEXT: v_perm_b32 v1, v1, v10, 0x7060302
-; GFX10-NEXT: v_perm_b32 v2, v2, v9, 0x7060302
-; GFX10-NEXT: v_perm_b32 v3, v3, v8, 0x7060302
+; GFX10-NEXT: v_add_f32_e32 v5, v15, v13
+; GFX10-NEXT: v_and_or_b32 v14, v3, s4, 0x400000
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v9, v11, vcc_lo
+; GFX10-NEXT: v_add3_u32 v4, v12, v6, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v9, v6, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GFX10-NEXT: v_bfe_u32 v12, v5, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_bfe_u32 v13, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v15, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v6, v11, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v11, v5, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc_lo
+; GFX10-NEXT: v_add3_u32 v9, v12, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v13, v0, s4, 0x400000
+; GFX10-NEXT: v_perm_b32 v2, v2, v8, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v6, v15, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v10, v14, vcc_lo
+; GFX10-NEXT: v_perm_b32 v3, v3, v7, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fadd_v8bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-NEXT: v_dual_add_f32 v8, v9, v8 :: v_dual_add_f32 v9, v11, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v1
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_dual_add_f32 v1, v1, v5 :: v_dual_and_b32 v6, 0xffff0000, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_dual_add_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v9, 16, v6
+; GFX11-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_f32_e32 v9, v11, v9
+; GFX11-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v12, v9, s0, 0x400000
+; GFX11-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v5
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v13, v2, 16, 1
; GFX11-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX11-NEXT: v_dual_add_f32 v10, v11, v10 :: v_dual_add_f32 v11, v13, v12
+; GFX11-NEXT: v_and_or_b32 v7, v8, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v8, v9, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX11-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-NEXT: v_add3_u32 v9, v13, v2, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_and_or_b32 v14, v3, s0, 0x400000
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX11-NEXT: v_add3_u32 v10, v11, v3, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_add_f32 v6, v11, v6 :: v_dual_and_b32 v1, 0xffff0000, v1
+; GFX11-NEXT: v_and_or_b32 v11, v2, s0, 0x400000
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX11-NEXT: v_bfe_u32 v12, v6, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v9, v11, vcc_lo
+; GFX11-NEXT: v_and_or_b32 v9, v6, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: v_add_f32_e32 v1, v1, v5
+; GFX11-NEXT: v_add3_u32 v4, v12, v6, 0x7fff
+; GFX11-NEXT: v_perm_b32 v2, v2, v8, 0x7060302
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_perm_b32 v2, v2, v9, 0x7060302
-; GFX11-NEXT: v_perm_b32 v3, v3, v8, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v1, v1, v10, 0x7060302
-; GFX11-NEXT: v_perm_b32 v0, v0, v11, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc_lo
+; GFX11-NEXT: v_add_f32_e32 v5, v15, v13
+; GFX11-NEXT: v_bfe_u32 v13, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v15, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v6, v11, v1, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v12, v5, 16, 1
+; GFX11-NEXT: v_and_or_b32 v11, v5, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_add3_u32 v9, v12, v5, 0x7fff
+; GFX11-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v13, v0, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v6, v15, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v10, v14, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v3, v3, v7, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fadd <8 x bfloat> %a, %b
ret <8 x bfloat> %op
@@ -9036,36 +10044,67 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GCN-LABEL: v_fadd_v16bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GCN-NEXT: v_add_f32_e32 v14, v14, v30
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GCN-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GCN-NEXT: v_add_f32_e32 v13, v13, v29
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GCN-NEXT: v_add_f32_e32 v12, v12, v28
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GCN-NEXT: v_add_f32_e32 v11, v11, v27
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GCN-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GCN-NEXT: v_add_f32_e32 v10, v10, v26
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GCN-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GCN-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GCN-NEXT: v_add_f32_e32 v9, v9, v25
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GCN-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GCN-NEXT: v_add_f32_e32 v8, v8, v24
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GCN-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_add_f32_e32 v7, v7, v23
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GCN-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GCN-NEXT: v_add_f32_e32 v6, v6, v22
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GCN-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_add_f32_e32 v5, v5, v21
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GCN-NEXT: v_add_f32_e32 v4, v4, v20
@@ -9098,7 +10137,8 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v20
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v20
+; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GCN-NEXT: v_add_f32_e32 v15, v15, v16
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
@@ -9107,12 +10147,41 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-LABEL: v_fadd_v16bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX7-NEXT: v_add_f32_e32 v4, v4, v20
-; GFX7-NEXT: buffer_load_dword v20, off, s[0:3], s32
-; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v22
+; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
@@ -9129,18 +10198,19 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_add_f32_e32 v0, v0, v16
+; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_add_f32_e32 v14, v14, v30
; GFX7-NEXT: v_add_f32_e32 v13, v13, v29
; GFX7-NEXT: v_add_f32_e32 v12, v12, v28
@@ -9149,11 +10219,12 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_add_f32_e32 v9, v9, v25
; GFX7-NEXT: v_add_f32_e32 v8, v8, v24
; GFX7-NEXT: v_add_f32_e32 v7, v7, v23
-; GFX7-NEXT: v_add_f32_e32 v6, v6, v22
; GFX7-NEXT: v_add_f32_e32 v5, v5, v21
+; GFX7-NEXT: v_add_f32_e32 v4, v4, v20
; GFX7-NEXT: v_add_f32_e32 v3, v3, v19
; GFX7-NEXT: v_add_f32_e32 v2, v2, v18
; GFX7-NEXT: v_add_f32_e32 v1, v1, v17
+; GFX7-NEXT: v_add_f32_e32 v0, v0, v16
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -9161,6 +10232,10 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; GFX7-NEXT: v_add_f32_e32 v15, v15, v22
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
@@ -9169,9 +10244,6 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v20
-; GFX7-NEXT: v_add_f32_e32 v15, v15, v16
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -9180,51 +10252,165 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v15
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v7
+; GFX8-NEXT: v_add_f32_e32 v16, v17, v16
+; GFX8-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX8-NEXT: v_add_f32_e32 v16, v17, v16
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v16
; GFX8-NEXT: v_add_f32_e32 v7, v7, v15
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
+; GFX8-NEXT: v_bfe_u32 v15, v7, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, v15, v7
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, s4, v15
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v7
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v15, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX8-NEXT: v_add_f32_e32 v15, v17, v15
+; GFX8-NEXT: v_bfe_u32 v17, v15, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v15
; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX8-NEXT: v_add_f32_e32 v15, v17, v15
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v15
; GFX8-NEXT: v_add_f32_e32 v6, v6, v14
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v15, v15
+; GFX8-NEXT: v_bfe_u32 v14, v6, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v14, vcc, v14, v6
+; GFX8-NEXT: v_add_u32_e32 v14, vcc, s4, v14
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v6
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v13
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX8-NEXT: v_add_f32_e32 v14, v17, v14
+; GFX8-NEXT: v_bfe_u32 v17, v14, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v14
; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX8-NEXT: v_add_f32_e32 v14, v17, v14
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v14
; GFX8-NEXT: v_add_f32_e32 v5, v5, v13
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
+; GFX8-NEXT: v_bfe_u32 v13, v5, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v14, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v13, vcc, v13, v5
+; GFX8-NEXT: v_add_u32_e32 v13, vcc, s4, v13
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v5
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v13, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v12
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v4
+; GFX8-NEXT: v_add_f32_e32 v13, v17, v13
+; GFX8-NEXT: v_bfe_u32 v17, v13, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v13
; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v13, v17, v13
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v13
; GFX8-NEXT: v_add_f32_e32 v4, v4, v12
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX8-NEXT: v_bfe_u32 v12, v4, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v13, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v12, vcc, v12, v4
+; GFX8-NEXT: v_add_u32_e32 v12, vcc, s4, v12
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v4
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v12, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v12, 16, v11
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX8-NEXT: v_add_f32_e32 v12, v17, v12
+; GFX8-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v12
; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_add_f32_e32 v12, v17, v12
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v12
; GFX8-NEXT: v_add_f32_e32 v3, v3, v11
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX8-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v11, vcc, v11, v3
+; GFX8-NEXT: v_add_u32_e32 v11, vcc, s4, v11
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v11, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v2
+; GFX8-NEXT: v_add_f32_e32 v11, v17, v11
+; GFX8-NEXT: v_bfe_u32 v17, v11, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v11
; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: v_add_f32_e32 v11, v17, v11
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v11
; GFX8-NEXT: v_add_f32_e32 v2, v2, v10
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
+; GFX8-NEXT: v_bfe_u32 v10, v2, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v11, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, v10, v2
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, s4, v10
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v10, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v9
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; GFX8-NEXT: v_add_f32_e32 v10, v17, v10
+; GFX8-NEXT: v_bfe_u32 v17, v10, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v10
; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_add_f32_e32 v10, v17, v10
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v10
; GFX8-NEXT: v_add_f32_e32 v1, v1, v9
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
+; GFX8-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v10, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v0
+; GFX8-NEXT: v_add_f32_e32 v9, v17, v9
+; GFX8-NEXT: v_bfe_u32 v17, v9, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v9
; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v9
; GFX8-NEXT: v_add_f32_e32 v0, v0, v8
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v9, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, s4, v8
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v17, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
@@ -9232,7 +10418,6 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_add_f32_e32 v9, v17, v9
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v9, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v10, 16
@@ -9249,52 +10434,149 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v15
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v7
+; GFX9-NEXT: v_add_f32_e32 v16, v17, v16
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v16
; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX9-NEXT: v_add_f32_e32 v16, v17, v16
+; GFX9-NEXT: v_add3_u32 v17, v17, v16, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_add_f32_e32 v7, v7, v15
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v15, v7, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v7
+; GFX9-NEXT: v_add3_u32 v15, v15, v7, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v15, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX9-NEXT: v_add_f32_e32 v15, v17, v15
+; GFX9-NEXT: v_bfe_u32 v17, v15, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v15
; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX9-NEXT: v_add_f32_e32 v15, v17, v15
+; GFX9-NEXT: v_add3_u32 v17, v17, v15, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v15, v15
; GFX9-NEXT: v_add_f32_e32 v6, v6, v14
+; GFX9-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v14, v6, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v6
+; GFX9-NEXT: v_add3_u32 v14, v14, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v14, 16, v13
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX9-NEXT: v_add_f32_e32 v14, v17, v14
+; GFX9-NEXT: v_bfe_u32 v17, v14, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v14
; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX9-NEXT: v_add_f32_e32 v14, v17, v14
+; GFX9-NEXT: v_add3_u32 v17, v17, v14, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
; GFX9-NEXT: v_add_f32_e32 v5, v5, v13
+; GFX9-NEXT: v_cndmask_b32_e32 v14, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v13, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v5
+; GFX9-NEXT: v_add3_u32 v13, v13, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v13, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v13, 16, v12
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v4
+; GFX9-NEXT: v_add_f32_e32 v13, v17, v13
+; GFX9-NEXT: v_bfe_u32 v17, v13, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v13
; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX9-NEXT: v_add_f32_e32 v13, v17, v13
+; GFX9-NEXT: v_add3_u32 v17, v17, v13, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
; GFX9-NEXT: v_add_f32_e32 v4, v4, v12
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v12, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v12, v12, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v12, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v11
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX9-NEXT: v_add_f32_e32 v12, v17, v12
+; GFX9-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v12
; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_add_f32_e32 v12, v17, v12
+; GFX9-NEXT: v_add3_u32 v17, v17, v12, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
; GFX9-NEXT: v_add_f32_e32 v3, v3, v11
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v11, v11, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v11, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v2
+; GFX9-NEXT: v_add_f32_e32 v11, v17, v11
+; GFX9-NEXT: v_bfe_u32 v17, v11, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v11
; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_add_f32_e32 v11, v17, v11
+; GFX9-NEXT: v_add3_u32 v17, v17, v11, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
; GFX9-NEXT: v_add_f32_e32 v2, v2, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v11, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v10, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v10, v10, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v9
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; GFX9-NEXT: v_add_f32_e32 v10, v17, v10
+; GFX9-NEXT: v_bfe_u32 v17, v10, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v10
; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_add_f32_e32 v10, v17, v10
+; GFX9-NEXT: v_add3_u32 v17, v17, v10, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
; GFX9-NEXT: v_add_f32_e32 v1, v1, v9
+; GFX9-NEXT: v_cndmask_b32_e32 v10, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v9, v9, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v0
+; GFX9-NEXT: v_add_f32_e32 v9, v17, v9
+; GFX9-NEXT: v_bfe_u32 v17, v9, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v9
; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_add_f32_e32 v9, v17, v9
+; GFX9-NEXT: v_add3_u32 v17, v17, v9, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
; GFX9-NEXT: v_add_f32_e32 v0, v0, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v8, v8, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v17, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v9, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v10, s4
@@ -9313,119 +10595,297 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v7
; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v5
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v6
; GFX10-NEXT: v_add_f32_e32 v16, v17, v16
-; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v14
; GFX10-NEXT: v_add_f32_e32 v7, v7, v15
-; GFX10-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v12
-; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v4
-; GFX10-NEXT: v_add_f32_e32 v15, v17, v15
+; GFX10-NEXT: v_bfe_u32 v15, v16, 16, 1
+; GFX10-NEXT: v_and_or_b32 v20, v16, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v19, v7, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX10-NEXT: v_add_f32_e32 v17, v18, v17
+; GFX10-NEXT: v_add3_u32 v15, v15, v16, 0x7fff
; GFX10-NEXT: v_add_f32_e32 v6, v6, v14
-; GFX10-NEXT: v_add_f32_e32 v14, v19, v18
+; GFX10-NEXT: v_add3_u32 v18, v19, v7, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v19, v7, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v21, v17, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v15, v15, v20, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v5
+; GFX10-NEXT: v_and_or_b32 v16, v17, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v14, v21, v17, 0x7fff
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v7, v18, v19, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX10-NEXT: v_bfe_u32 v18, v6, 16, 1
+; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT: v_perm_b32 v7, v7, v15, 0x7060302
+; GFX10-NEXT: v_add_f32_e32 v17, v20, v19
+; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v4
; GFX10-NEXT: v_add_f32_e32 v5, v5, v13
-; GFX10-NEXT: v_add_f32_e32 v13, v21, v20
-; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v11
-; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v14, v14, v16, vcc_lo
+; GFX10-NEXT: v_add3_u32 v16, v18, v6, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v13, v6, s4, 0x400000
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX10-NEXT: v_bfe_u32 v20, v17, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_bfe_u32 v21, v5, 16, 1
; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v17, v18, v17
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v16, v13, vcc_lo
+; GFX10-NEXT: v_add_f32_e32 v13, v19, v18
+; GFX10-NEXT: v_add3_u32 v16, v20, v17, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v18, v17, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT: v_add3_u32 v19, v21, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v20, v5, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v21, v13, 16, 1
+; GFX10-NEXT: v_add_f32_e32 v4, v4, v12
+; GFX10-NEXT: v_cndmask_b32_e32 v16, v16, v18, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v11
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v3
+; GFX10-NEXT: v_add3_u32 v17, v21, v13, 0x7fff
; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v19, v20, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v19, v13, s4, 0x400000
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT: v_add_f32_e32 v18, v20, v19
+; GFX10-NEXT: v_add_f32_e32 v12, v18, v12
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v10
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v11
+; GFX10-NEXT: v_and_or_b32 v22, v12, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v13, v17, v19, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v2
+; GFX10-NEXT: v_add3_u32 v11, v20, v4, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v20, v3, 16, 1
; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v9
-; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v1
+; GFX10-NEXT: v_add3_u32 v17, v17, v12, 0x7fff
+; GFX10-NEXT: v_add_f32_e32 v18, v19, v18
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX10-NEXT: v_add3_u32 v19, v20, v3, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v20, v3, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v23, v18, 16, 1
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v10
+; GFX10-NEXT: v_cndmask_b32_e32 v12, v17, v22, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_and_or_b32 v17, v18, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v10, v23, v18, 0x7fff
+; GFX10-NEXT: v_lshlrev_b32_e32 v22, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v19, v20, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v19, v2, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v9
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX10-NEXT: v_and_or_b32 v18, v2, s4, 0x400000
+; GFX10-NEXT: v_and_or_b32 v21, v4, s4, 0x400000
+; GFX10-NEXT: v_perm_b32 v3, v3, v12, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v10, v10, v17, vcc_lo
+; GFX10-NEXT: v_add3_u32 v17, v19, v2, 0x7fff
+; GFX10-NEXT: v_add_f32_e32 v19, v22, v20
+; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v8
; GFX10-NEXT: v_lshlrev_b32_e32 v22, 16, v0
; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_add_f32_e32 v19, v20, v19
-; GFX10-NEXT: v_add_f32_e32 v20, v22, v21
-; GFX10-NEXT: v_add_f32_e32 v0, v0, v8
+; GFX10-NEXT: v_bfe_u32 v23, v19, 16, 1
; GFX10-NEXT: v_add_f32_e32 v1, v1, v9
-; GFX10-NEXT: v_add_f32_e32 v2, v2, v10
-; GFX10-NEXT: v_add_f32_e32 v3, v3, v11
-; GFX10-NEXT: v_add_f32_e32 v4, v4, v12
-; GFX10-NEXT: v_perm_b32 v0, v0, v20, 0x7060302
+; GFX10-NEXT: v_add_f32_e32 v9, v22, v20
+; GFX10-NEXT: v_and_or_b32 v22, v19, s4, 0x400000
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v8
+; GFX10-NEXT: v_add3_u32 v20, v23, v19, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX10-NEXT: v_bfe_u32 v23, v9, 16, 1
+; GFX10-NEXT: v_and_or_b32 v24, v9, s4, 0x400000
+; GFX10-NEXT: v_and_or_b32 v25, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v22, v1, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_bfe_u32 v20, v0, 16, 1
+; GFX10-NEXT: v_add3_u32 v23, v23, v9, 0x7fff
+; GFX10-NEXT: v_perm_b32 v5, v5, v16, 0x7060302
+; GFX10-NEXT: v_perm_b32 v6, v6, v14, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v8, v22, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX10-NEXT: v_add3_u32 v20, v20, v0, 0x7fff
; GFX10-NEXT: v_perm_b32 v1, v1, v19, 0x7060302
-; GFX10-NEXT: v_perm_b32 v2, v2, v18, 0x7060302
-; GFX10-NEXT: v_perm_b32 v3, v3, v17, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v8, v23, v24, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v20, v25, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_perm_b32 v0, v0, v8, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_perm_b32 v2, v2, v10, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v11, v21, vcc_lo
; GFX10-NEXT: v_perm_b32 v4, v4, v13, 0x7060302
-; GFX10-NEXT: v_perm_b32 v5, v5, v14, 0x7060302
-; GFX10-NEXT: v_perm_b32 v6, v6, v15, 0x7060302
-; GFX10-NEXT: v_perm_b32 v7, v7, v16, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fadd_v16bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v0
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_add_f32 v4, v4, v12 :: v_dual_and_b32 v5, 0xffff0000, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v15
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v7
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v13
-; GFX11-NEXT: v_add_f32_e32 v13, v21, v20
-; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v8
-; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-NEXT: v_dual_add_f32 v16, v17, v16 :: v_dual_and_b32 v15, 0xffff0000, v15
-; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_add_f32 v0, v0, v8 :: v_dual_and_b32 v7, 0xffff0000, v7
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-NEXT: v_perm_b32 v4, v4, v13, 0x7060302
-; GFX11-NEXT: v_add_f32_e32 v7, v7, v15
-; GFX11-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_add_f32 v16, v17, v16 :: v_dual_lshlrev_b32 v17, 16, v14
; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v7, v7, v16, 0x7060302
-; GFX11-NEXT: v_add_f32_e32 v15, v17, v15
+; GFX11-NEXT: v_and_or_b32 v20, v16, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_f32_e32 v17, v18, v17
+; GFX11-NEXT: v_add_f32_e32 v6, v6, v14
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_bfe_u32 v21, v17, 16, 1
+; GFX11-NEXT: v_add3_u32 v14, v21, v17, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v7, v7, v15
+; GFX11-NEXT: v_bfe_u32 v15, v16, 16, 1
+; GFX11-NEXT: v_add3_u32 v15, v15, v16, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v16, v17, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v15, v15, v20 :: v_dual_lshlrev_b32 v20, 16, v5
+; GFX11-NEXT: v_bfe_u32 v19, v7, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-NEXT: v_add3_u32 v18, v19, v7, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v19, v7, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v18, v19, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v18, v6, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: v_perm_b32 v7, v7, v15, 0x7060302
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_dual_add_f32 v6, v6, v14 :: v_dual_lshlrev_b32 v17, 16, v11
-; GFX11-NEXT: v_add_f32_e32 v14, v19, v18
+; GFX11-NEXT: v_dual_add_f32 v17, v20, v19 :: v_dual_cndmask_b32 v14, v14, v16
+; GFX11-NEXT: v_add3_u32 v16, v18, v6, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; GFX11-NEXT: v_bfe_u32 v20, v17, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v4, v4, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v11
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX11-NEXT: v_add_f32_e32 v5, v5, v13
+; GFX11-NEXT: v_and_or_b32 v13, v6, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_cndmask_b32 v6, v16, v13 :: v_dual_add_f32 v13, v19, v18
+; GFX11-NEXT: v_add3_u32 v16, v20, v17, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v18, v17, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: v_perm_b32 v6, v6, v14, 0x7060302
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v16, v16, v18, vcc_lo
; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v10
+; GFX11-NEXT: v_bfe_u32 v21, v5, 16, 1
+; GFX11-NEXT: v_and_or_b32 v20, v5, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_add_f32_e32 v12, v18, v12
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v19, v21, v5, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v21, v13, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v10
+; GFX11-NEXT: v_and_or_b32 v22, v12, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v19, v20, vcc_lo
+; GFX11-NEXT: v_add3_u32 v17, v21, v13, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v19, v13, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX11-NEXT: v_and_or_b32 v21, v4, s0, 0x400000
+; GFX11-NEXT: v_perm_b32 v5, v5, v16, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v13, v17, v19, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v2
; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-NEXT: v_add3_u32 v17, v17, v12, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_f32_e32 v18, v19, v18
+; GFX11-NEXT: v_cndmask_b32_e32 v12, v17, v22, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v1
+; GFX11-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_dual_add_f32 v17, v18, v17 :: v_dual_and_b32 v10, 0xffff0000, v10
-; GFX11-NEXT: v_perm_b32 v5, v5, v14, 0x7060302
-; GFX11-NEXT: v_perm_b32 v6, v6, v15, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v23, v18, 16, 1
+; GFX11-NEXT: v_and_or_b32 v17, v18, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_add_f32 v2, v2, v10 :: v_dual_and_b32 v1, 0xffff0000, v1
; GFX11-NEXT: v_add_f32_e32 v3, v3, v11
-; GFX11-NEXT: v_dual_add_f32 v18, v20, v19 :: v_dual_lshlrev_b32 v19, 16, v9
-; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v1
+; GFX11-NEXT: v_add3_u32 v11, v20, v4, 0x7fff
+; GFX11-NEXT: v_add3_u32 v10, v23, v18, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v20, v3, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v19, v20, v3, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v20, v3, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v19, v20, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v19, v2, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v9
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-NEXT: v_and_or_b32 v18, v2, s0, 0x400000
+; GFX11-NEXT: v_perm_b32 v3, v3, v12, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v10, v10, v17, vcc_lo
+; GFX11-NEXT: v_add3_u32 v17, v19, v2, 0x7fff
+; GFX11-NEXT: v_add_f32_e32 v19, v22, v20
+; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v8
+; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX11-NEXT: v_dual_add_f32 v2, v2, v10 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_perm_b32 v3, v3, v17, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_add_f32 v19, v20, v19 :: v_dual_add_f32 v20, v22, v21
-; GFX11-NEXT: v_add_f32_e32 v1, v1, v9
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_perm_b32 v2, v2, v18, 0x7060302
-; GFX11-NEXT: v_perm_b32 v0, v0, v20, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v23, v19, 16, 1
+; GFX11-NEXT: v_dual_add_f32 v0, v0, v8 :: v_dual_add_f32 v1, v1, v9
+; GFX11-NEXT: v_add_f32_e32 v9, v22, v20
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v20, v23, v19, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v22, v19, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v25, v0, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-NEXT: v_bfe_u32 v23, v9, 16, 1
+; GFX11-NEXT: v_and_or_b32 v24, v9, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX11-NEXT: v_and_or_b32 v22, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_bfe_u32 v20, v0, 16, 1
+; GFX11-NEXT: v_add3_u32 v23, v23, v9, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v8, v22, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-NEXT: v_add3_u32 v20, v20, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_perm_b32 v1, v1, v19, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v8, v23, v24, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v20, v25, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v0, v0, v8, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_perm_b32 v2, v2, v10, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v11, v21, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v4, v4, v13, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fadd <16 x bfloat> %a, %b
ret <16 x bfloat> %op
@@ -9435,166 +10895,230 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GCN-LABEL: v_fadd_v32bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:128
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32
+; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128
; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:124
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
-; GCN-NEXT: v_add_f32_e32 v31, v32, v31
+; GCN-NEXT: v_add_f32_e32 v31, v31, v32
+; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:120
; GCN-NEXT: v_add_f32_e32 v30, v30, v32
+; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GCN-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v29, v29, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:116
+; GCN-NEXT: v_add_f32_e32 v29, v29, v32
+; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112
; GCN-NEXT: v_add_f32_e32 v28, v28, v32
+; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v27, v27, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:108
+; GCN-NEXT: v_add_f32_e32 v27, v27, v32
+; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:104
; GCN-NEXT: v_add_f32_e32 v26, v26, v32
+; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GCN-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v25, v25, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:100
+; GCN-NEXT: v_add_f32_e32 v25, v25, v32
+; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GCN-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:96
; GCN-NEXT: v_add_f32_e32 v24, v24, v32
+; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GCN-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v23, v23, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:92
+; GCN-NEXT: v_add_f32_e32 v23, v23, v32
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GCN-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:88
; GCN-NEXT: v_add_f32_e32 v22, v22, v32
+; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GCN-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v21, v21, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:84
+; GCN-NEXT: v_add_f32_e32 v21, v21, v32
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GCN-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:80
; GCN-NEXT: v_add_f32_e32 v20, v20, v32
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GCN-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v19, v19, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:76
+; GCN-NEXT: v_add_f32_e32 v19, v19, v32
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GCN-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:72
; GCN-NEXT: v_add_f32_e32 v18, v18, v32
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GCN-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v17, v17, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:68
+; GCN-NEXT: v_add_f32_e32 v17, v17, v32
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64
; GCN-NEXT: v_add_f32_e32 v16, v16, v32
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v15, v15, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60
+; GCN-NEXT: v_add_f32_e32 v15, v15, v32
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:56
; GCN-NEXT: v_add_f32_e32 v14, v14, v32
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v13, v13, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:52
+; GCN-NEXT: v_add_f32_e32 v13, v13, v32
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:48
; GCN-NEXT: v_add_f32_e32 v12, v12, v32
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v11, v11, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:44
+; GCN-NEXT: v_add_f32_e32 v11, v11, v32
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GCN-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:40
; GCN-NEXT: v_add_f32_e32 v10, v10, v32
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GCN-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v9, v9, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:36
+; GCN-NEXT: v_add_f32_e32 v9, v9, v32
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:32
; GCN-NEXT: v_add_f32_e32 v8, v8, v32
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v7, v7, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:28
+; GCN-NEXT: v_add_f32_e32 v7, v7, v32
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:24
; GCN-NEXT: v_add_f32_e32 v6, v6, v32
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v5, v5, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:20
+; GCN-NEXT: v_add_f32_e32 v5, v5, v32
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:16
; GCN-NEXT: v_add_f32_e32 v4, v4, v32
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v3, v3, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12
+; GCN-NEXT: v_add_f32_e32 v3, v3, v32
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
; GCN-NEXT: v_add_f32_e32 v2, v2, v32
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_add_f32_e32 v1, v1, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:4
+; GCN-NEXT: v_add_f32_e32 v1, v1, v32
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: v_add_f32_e32 v0, v0, v32
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -9634,197 +11158,261 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX7-LABEL: v_fadd_v32bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:128
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128
+; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
-; GFX7-NEXT: v_add_f32_e32 v31, v32, v31
+; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; GFX7-NEXT: v_add_f32_e32 v31, v31, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124
; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v30, v30, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120
; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v29, v29, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v28, v28, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112
; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v27, v27, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108
; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v26, v26, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104
; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v25, v25, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100
; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v24, v24, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96
; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v23, v23, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v22, v22, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88
; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v21, v21, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v20, v20, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80
; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v19, v19, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76
; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v18, v18, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72
; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v17, v17, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68
; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v16, v16, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64
; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v15, v15, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v14, v14, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v13, v13, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52
; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v12, v12, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48
; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v11, v11, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v10, v10, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v9, v9, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v8, v8, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v7, v7, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v6, v6, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v5, v5, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v4, v4, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v3, v3, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v2, v2, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v1, v1, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_add_f32_e32 v0, v0, v32
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -9835,114 +11423,329 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v14
+; GFX8-NEXT: v_add_f32_e32 v31, v32, v31
+; GFX8-NEXT: v_bfe_u32 v32, v31, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v32, vcc, v32, v31
; GFX8-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX8-NEXT: v_add_f32_e32 v31, v32, v31
-; GFX8-NEXT: v_add_f32_e32 v30, v14, v30
-; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v29
+; GFX8-NEXT: v_add_u32_e32 v32, vcc, s4, v32
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v31
+; GFX8-NEXT: v_add_f32_e32 v14, v14, v30
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_bfe_u32 v30, v14, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v32, v33, vcc
+; GFX8-NEXT: v_add_u32_e32 v30, vcc, v30, v14
+; GFX8-NEXT: v_add_u32_e32 v30, vcc, s4, v30
+; GFX8-NEXT: v_and_b32_e32 v32, 0x80000000, v14
+; GFX8-NEXT: v_or_b32_e32 v32, 0x400000, v32
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
+; GFX8-NEXT: v_cndmask_b32_e32 v14, v30, v32, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v30, 16, v29
; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v13
+; GFX8-NEXT: v_add_f32_e32 v32, v32, v30
+; GFX8-NEXT: buffer_load_dword v30, off, s[0:3], s32
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX8-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX8-NEXT: v_add_f32_e32 v14, v32, v14
; GFX8-NEXT: v_add_f32_e32 v13, v13, v29
+; GFX8-NEXT: v_bfe_u32 v29, v13, 16, 1
+; GFX8-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX8-NEXT: v_alignbit_b32 v14, v14, v31, 16
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v34, 16, v30
+; GFX8-NEXT: v_add_f32_e32 v33, v33, v34
+; GFX8-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; GFX8-NEXT: v_add_f32_e32 v30, v15, v30
+; GFX8-NEXT: v_bfe_u32 v15, v33, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, v15, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v33
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, s4, v15
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX8-NEXT: v_bfe_u32 v33, v30, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v15, v15, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v30
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v30
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
+; GFX8-NEXT: v_cndmask_b32_e32 v30, v33, v34, vcc
+; GFX8-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v32
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v32
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
+; GFX8-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v29, vcc, v29, v13
+; GFX8-NEXT: v_add_u32_e32 v29, vcc, s4, v29
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v13
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX8-NEXT: v_cndmask_b32_e32 v13, v29, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v29, 16, v28
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v12
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v12
+; GFX8-NEXT: v_add_f32_e32 v29, v33, v29
+; GFX8-NEXT: v_bfe_u32 v33, v29, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v29
; GFX8-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX8-NEXT: v_add_f32_e32 v29, v32, v29
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v29
; GFX8-NEXT: v_add_f32_e32 v12, v12, v28
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v29, v29
+; GFX8-NEXT: v_bfe_u32 v28, v12, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v29, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v28, vcc, v28, v12
+; GFX8-NEXT: v_add_u32_e32 v28, vcc, s4, v28
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v12
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX8-NEXT: v_cndmask_b32_e32 v12, v28, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v28, 16, v27
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v11
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v11
+; GFX8-NEXT: v_add_f32_e32 v28, v33, v28
+; GFX8-NEXT: v_bfe_u32 v33, v28, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v28
; GFX8-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; GFX8-NEXT: v_add_f32_e32 v28, v32, v28
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v28
; GFX8-NEXT: v_add_f32_e32 v11, v11, v27
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v28, v28
+; GFX8-NEXT: v_bfe_u32 v27, v11, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v28, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v27, vcc, v27, v11
+; GFX8-NEXT: v_add_u32_e32 v27, vcc, s4, v27
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v11
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
+; GFX8-NEXT: v_cndmask_b32_e32 v11, v27, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v27, 16, v26
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v10
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v10
+; GFX8-NEXT: v_add_f32_e32 v27, v33, v27
+; GFX8-NEXT: v_bfe_u32 v33, v27, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v27
; GFX8-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX8-NEXT: v_add_f32_e32 v27, v32, v27
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v27
; GFX8-NEXT: v_add_f32_e32 v10, v10, v26
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v27, v27
+; GFX8-NEXT: v_bfe_u32 v26, v10, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v27, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v26, vcc, v26, v10
+; GFX8-NEXT: v_add_u32_e32 v26, vcc, s4, v26
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v10
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
+; GFX8-NEXT: v_cndmask_b32_e32 v10, v26, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v26, 16, v25
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v9
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v9
+; GFX8-NEXT: v_add_f32_e32 v26, v33, v26
+; GFX8-NEXT: v_bfe_u32 v33, v26, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v26
; GFX8-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX8-NEXT: v_add_f32_e32 v26, v32, v26
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v26
; GFX8-NEXT: v_add_f32_e32 v9, v9, v25
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v26, v26
+; GFX8-NEXT: v_bfe_u32 v25, v9, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v26, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v25, vcc, v25, v9
+; GFX8-NEXT: v_add_u32_e32 v25, vcc, s4, v25
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v9
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX8-NEXT: v_cndmask_b32_e32 v9, v25, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v25, 16, v24
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v8
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v8
+; GFX8-NEXT: v_add_f32_e32 v25, v33, v25
+; GFX8-NEXT: v_bfe_u32 v33, v25, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v25
; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v25
; GFX8-NEXT: v_add_f32_e32 v8, v8, v24
-; GFX8-NEXT: buffer_load_dword v24, off, s[0:3], s32
-; GFX8-NEXT: v_add_f32_e32 v25, v32, v25
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v15
-; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; GFX8-NEXT: v_alignbit_b32 v8, v8, v25, 16
-; GFX8-NEXT: v_alignbit_b32 v9, v9, v26, 16
-; GFX8-NEXT: v_alignbit_b32 v10, v10, v27, 16
-; GFX8-NEXT: v_alignbit_b32 v11, v11, v28, 16
-; GFX8-NEXT: v_alignbit_b32 v12, v12, v29, 16
-; GFX8-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v24
-; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; GFX8-NEXT: v_add_f32_e32 v32, v32, v33
-; GFX8-NEXT: v_add_f32_e32 v15, v15, v24
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v25, v25
+; GFX8-NEXT: v_bfe_u32 v24, v8, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v25, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v24, vcc, v24, v8
+; GFX8-NEXT: v_add_u32_e32 v24, vcc, s4, v24
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v8
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v24, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v24, 16, v23
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v7
+; GFX8-NEXT: v_add_f32_e32 v24, v33, v24
+; GFX8-NEXT: v_bfe_u32 v33, v24, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v24
; GFX8-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX8-NEXT: v_add_f32_e32 v24, v33, v24
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v24
; GFX8-NEXT: v_add_f32_e32 v7, v7, v23
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v24, v24
+; GFX8-NEXT: v_bfe_u32 v23, v7, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v24, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v23, vcc, v23, v7
+; GFX8-NEXT: v_add_u32_e32 v23, vcc, s4, v23
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v7
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v23, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v23, 16, v22
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v6
+; GFX8-NEXT: v_add_f32_e32 v23, v33, v23
+; GFX8-NEXT: v_bfe_u32 v33, v23, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v23
; GFX8-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX8-NEXT: v_add_f32_e32 v23, v33, v23
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v23
; GFX8-NEXT: v_add_f32_e32 v6, v6, v22
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v23, v23
+; GFX8-NEXT: v_bfe_u32 v22, v6, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v23, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v22, vcc, v22, v6
+; GFX8-NEXT: v_add_u32_e32 v22, vcc, s4, v22
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v6
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v22, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v22, 16, v21
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v5
+; GFX8-NEXT: v_add_f32_e32 v22, v33, v22
+; GFX8-NEXT: v_bfe_u32 v33, v22, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v22
; GFX8-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX8-NEXT: v_add_f32_e32 v22, v33, v22
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v22
; GFX8-NEXT: v_add_f32_e32 v5, v5, v21
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v22, v22
+; GFX8-NEXT: v_bfe_u32 v21, v5, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v22, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v21, vcc, v21, v5
+; GFX8-NEXT: v_add_u32_e32 v21, vcc, s4, v21
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v5
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v21, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v21, 16, v20
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v4
+; GFX8-NEXT: v_add_f32_e32 v21, v33, v21
+; GFX8-NEXT: v_bfe_u32 v33, v21, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v21
; GFX8-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v21, v33, v21
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v21
; GFX8-NEXT: v_add_f32_e32 v4, v4, v20
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v21, v21
+; GFX8-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v21, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v20, vcc, v20, v4
+; GFX8-NEXT: v_add_u32_e32 v20, vcc, s4, v20
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v4
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v20, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v20, 16, v19
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v3
+; GFX8-NEXT: v_add_f32_e32 v20, v33, v20
+; GFX8-NEXT: v_bfe_u32 v33, v20, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v20
; GFX8-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_add_f32_e32 v20, v33, v20
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v20
; GFX8-NEXT: v_add_f32_e32 v3, v3, v19
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v20, v20
+; GFX8-NEXT: v_bfe_u32 v19, v3, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v20, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v19, vcc, v19, v3
+; GFX8-NEXT: v_add_u32_e32 v19, vcc, s4, v19
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v19, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v19, 16, v18
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v2
+; GFX8-NEXT: v_add_f32_e32 v19, v33, v19
+; GFX8-NEXT: v_bfe_u32 v33, v19, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v19
; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: v_add_f32_e32 v19, v33, v19
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v19
; GFX8-NEXT: v_add_f32_e32 v2, v2, v18
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v19, v19
+; GFX8-NEXT: v_bfe_u32 v18, v2, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v19, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v18, vcc, v18, v2
+; GFX8-NEXT: v_add_u32_e32 v18, vcc, s4, v18
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v18, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v17
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v1
+; GFX8-NEXT: v_add_f32_e32 v18, v33, v18
+; GFX8-NEXT: v_bfe_u32 v33, v18, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v18
; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_add_f32_e32 v18, v33, v18
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v18
; GFX8-NEXT: v_add_f32_e32 v1, v1, v17
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
+; GFX8-NEXT: v_bfe_u32 v17, v1, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v16
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v0
+; GFX8-NEXT: v_add_f32_e32 v17, v33, v17
+; GFX8-NEXT: v_bfe_u32 v33, v17, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v17
; GFX8-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v17
; GFX8-NEXT: v_add_f32_e32 v0, v0, v16
-; GFX8-NEXT: v_add_f32_e32 v17, v33, v17
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
+; GFX8-NEXT: v_bfe_u32 v16, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v16, vcc, v16, v0
+; GFX8-NEXT: v_add_u32_e32 v16, vcc, s4, v16
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v16, v33, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
@@ -9951,8 +11754,13 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v30
+; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11
; GFX8-NEXT: v_alignbit_b32 v0, v0, v17, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v18, 16
; GFX8-NEXT: v_alignbit_b32 v2, v2, v19, 16
@@ -9961,8 +11769,13 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX8-NEXT: v_alignbit_b32 v5, v5, v22, 16
; GFX8-NEXT: v_alignbit_b32 v6, v6, v23, 16
; GFX8-NEXT: v_alignbit_b32 v7, v7, v24, 16
-; GFX8-NEXT: v_alignbit_b32 v14, v16, v31, 16
-; GFX8-NEXT: v_alignbit_b32 v15, v15, v32, 16
+; GFX8-NEXT: v_alignbit_b32 v8, v8, v25, 16
+; GFX8-NEXT: v_alignbit_b32 v9, v9, v26, 16
+; GFX8-NEXT: v_alignbit_b32 v10, v10, v27, 16
+; GFX8-NEXT: v_alignbit_b32 v11, v11, v28, 16
+; GFX8-NEXT: v_alignbit_b32 v12, v12, v29, 16
+; GFX8-NEXT: v_alignbit_b32 v13, v13, v32, 16
+; GFX8-NEXT: v_alignbit_b32 v15, v16, v15, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fadd_v32bf16:
@@ -9970,110 +11783,296 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v14
+; GFX9-NEXT: v_add_f32_e32 v31, v32, v31
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_bfe_u32 v32, v31, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v31
; GFX9-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX9-NEXT: v_add_f32_e32 v31, v32, v31
+; GFX9-NEXT: v_add3_u32 v32, v32, v31, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
; GFX9-NEXT: v_add_f32_e32 v14, v14, v30
+; GFX9-NEXT: v_cndmask_b32_e32 v31, v32, v33, vcc
+; GFX9-NEXT: v_bfe_u32 v30, v14, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v32, 0x80000000, v14
+; GFX9-NEXT: v_add3_u32 v30, v30, v14, s4
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
+; GFX9-NEXT: v_cndmask_b32_e32 v14, v30, v32, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v29
; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v13
; GFX9-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX9-NEXT: v_add_f32_e32 v30, v32, v30
; GFX9-NEXT: v_add_f32_e32 v13, v13, v29
-; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v28
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v12
+; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32
+; GFX9-NEXT: v_add_f32_e32 v30, v32, v30
+; GFX9-NEXT: v_bfe_u32 v32, v30, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v30
+; GFX9-NEXT: v_add3_u32 v32, v32, v30, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
+; GFX9-NEXT: v_cndmask_b32_e32 v30, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v15
+; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v29
+; GFX9-NEXT: v_add_f32_e32 v32, v32, v33
+; GFX9-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX9-NEXT: v_add_f32_e32 v29, v15, v29
+; GFX9-NEXT: v_bfe_u32 v15, v32, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v32
+; GFX9-NEXT: v_add3_u32 v15, v15, v32, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v15, v15, v33, vcc
+; GFX9-NEXT: v_bfe_u32 v32, v29, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v29
+; GFX9-NEXT: v_add3_u32 v32, v32, v29, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v29, v29
+; GFX9-NEXT: v_cndmask_b32_e32 v29, v32, v33, vcc
+; GFX9-NEXT: v_bfe_u32 v32, v13, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v13
+; GFX9-NEXT: v_add3_u32 v32, v32, v13, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v12
+; GFX9-NEXT: v_add_f32_e32 v32, v33, v32
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v32
; GFX9-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX9-NEXT: v_add_f32_e32 v29, v32, v29
+; GFX9-NEXT: v_add3_u32 v33, v33, v32, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_add_f32_e32 v12, v12, v28
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v28, v12, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v12
+; GFX9-NEXT: v_add3_u32 v28, v28, v12, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v28, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v27
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v11
+; GFX9-NEXT: v_add_f32_e32 v28, v33, v28
+; GFX9-NEXT: v_bfe_u32 v33, v28, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v28
; GFX9-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; GFX9-NEXT: v_add_f32_e32 v28, v32, v28
+; GFX9-NEXT: v_add3_u32 v33, v33, v28, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v28, v28
; GFX9-NEXT: v_add_f32_e32 v11, v11, v27
+; GFX9-NEXT: v_cndmask_b32_e32 v28, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v27, v11, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v11
+; GFX9-NEXT: v_add3_u32 v27, v27, v11, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
+; GFX9-NEXT: v_cndmask_b32_e32 v11, v27, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v26
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v10
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v10
+; GFX9-NEXT: v_add_f32_e32 v27, v33, v27
+; GFX9-NEXT: v_bfe_u32 v33, v27, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v27
; GFX9-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX9-NEXT: v_add_f32_e32 v27, v32, v27
+; GFX9-NEXT: v_add3_u32 v33, v33, v27, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v27, v27
; GFX9-NEXT: v_add_f32_e32 v10, v10, v26
+; GFX9-NEXT: v_cndmask_b32_e32 v27, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v26, v10, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v10
+; GFX9-NEXT: v_add3_u32 v26, v26, v10, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v10, v26, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v25
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v9
+; GFX9-NEXT: v_add_f32_e32 v26, v33, v26
+; GFX9-NEXT: v_bfe_u32 v33, v26, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v26
; GFX9-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX9-NEXT: v_add_f32_e32 v26, v32, v26
+; GFX9-NEXT: v_add3_u32 v33, v33, v26, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v26, v26
; GFX9-NEXT: v_add_f32_e32 v9, v9, v25
+; GFX9-NEXT: v_cndmask_b32_e32 v26, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v25, v9, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v9
+; GFX9-NEXT: v_add3_u32 v25, v25, v9, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v25, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v24
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v8
+; GFX9-NEXT: v_add_f32_e32 v25, v33, v25
+; GFX9-NEXT: v_bfe_u32 v33, v25, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v25
; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; GFX9-NEXT: v_add3_u32 v33, v33, v25, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v25, v25
; GFX9-NEXT: v_add_f32_e32 v8, v8, v24
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32
-; GFX9-NEXT: v_add_f32_e32 v25, v32, v25
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v15
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: v_perm_b32 v8, v8, v25, s4
-; GFX9-NEXT: v_perm_b32 v9, v9, v26, s4
-; GFX9-NEXT: v_perm_b32 v10, v10, v27, s4
-; GFX9-NEXT: v_perm_b32 v11, v11, v28, s4
-; GFX9-NEXT: v_perm_b32 v12, v12, v29, s4
-; GFX9-NEXT: v_perm_b32 v13, v13, v30, s4
-; GFX9-NEXT: v_perm_b32 v14, v14, v31, s4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v24
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; GFX9-NEXT: v_add_f32_e32 v32, v32, v33
-; GFX9-NEXT: v_add_f32_e32 v15, v15, v24
+; GFX9-NEXT: v_cndmask_b32_e32 v25, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v24, v8, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v8
+; GFX9-NEXT: v_add3_u32 v24, v24, v8, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v24, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v23
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v7
+; GFX9-NEXT: v_add_f32_e32 v24, v33, v24
+; GFX9-NEXT: v_bfe_u32 v33, v24, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v24
; GFX9-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX9-NEXT: v_add_f32_e32 v24, v33, v24
+; GFX9-NEXT: v_add3_u32 v33, v33, v24, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v24, v24
; GFX9-NEXT: v_add_f32_e32 v7, v7, v23
+; GFX9-NEXT: v_cndmask_b32_e32 v24, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v23, v7, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v7
+; GFX9-NEXT: v_add3_u32 v23, v23, v7, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v23, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v22
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v6
+; GFX9-NEXT: v_add_f32_e32 v23, v33, v23
+; GFX9-NEXT: v_bfe_u32 v33, v23, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v23
; GFX9-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX9-NEXT: v_add_f32_e32 v23, v33, v23
+; GFX9-NEXT: v_add3_u32 v33, v33, v23, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v23, v23
; GFX9-NEXT: v_add_f32_e32 v6, v6, v22
+; GFX9-NEXT: v_cndmask_b32_e32 v23, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v22, v6, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v6
+; GFX9-NEXT: v_add3_u32 v22, v22, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v22, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v21
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v5
+; GFX9-NEXT: v_add_f32_e32 v22, v33, v22
+; GFX9-NEXT: v_bfe_u32 v33, v22, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v22
; GFX9-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX9-NEXT: v_add_f32_e32 v22, v33, v22
+; GFX9-NEXT: v_add3_u32 v33, v33, v22, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v22, v22
; GFX9-NEXT: v_add_f32_e32 v5, v5, v21
+; GFX9-NEXT: v_cndmask_b32_e32 v22, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v21, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v5
+; GFX9-NEXT: v_add3_u32 v21, v21, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v21, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v20
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v4
+; GFX9-NEXT: v_add_f32_e32 v21, v33, v21
+; GFX9-NEXT: v_bfe_u32 v33, v21, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v21
; GFX9-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX9-NEXT: v_add_f32_e32 v21, v33, v21
+; GFX9-NEXT: v_add3_u32 v33, v33, v21, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v21, v21
; GFX9-NEXT: v_add_f32_e32 v4, v4, v20
+; GFX9-NEXT: v_cndmask_b32_e32 v21, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v20, v20, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v20, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v19
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v3
+; GFX9-NEXT: v_add_f32_e32 v20, v33, v20
+; GFX9-NEXT: v_bfe_u32 v33, v20, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v20
; GFX9-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_add_f32_e32 v20, v33, v20
+; GFX9-NEXT: v_add3_u32 v33, v33, v20, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v20, v20
; GFX9-NEXT: v_add_f32_e32 v3, v3, v19
+; GFX9-NEXT: v_cndmask_b32_e32 v20, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v19, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v19, v19, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v19, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v18
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v2
+; GFX9-NEXT: v_add_f32_e32 v19, v33, v19
+; GFX9-NEXT: v_bfe_u32 v33, v19, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v19
; GFX9-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_add_f32_e32 v19, v33, v19
+; GFX9-NEXT: v_add3_u32 v33, v33, v19, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v19, v19
; GFX9-NEXT: v_add_f32_e32 v2, v2, v18
+; GFX9-NEXT: v_cndmask_b32_e32 v19, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v18, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v18, v18, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v18, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v1
+; GFX9-NEXT: v_add_f32_e32 v18, v33, v18
+; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v18
; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_add_f32_e32 v18, v33, v18
+; GFX9-NEXT: v_add3_u32 v33, v33, v18, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
; GFX9-NEXT: v_add_f32_e32 v1, v1, v17
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v17, v17, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v16
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v0
+; GFX9-NEXT: v_add_f32_e32 v17, v33, v17
+; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v17
; GFX9-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_add_f32_e32 v17, v33, v17
+; GFX9-NEXT: v_add3_u32 v33, v33, v17, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_add_f32_e32 v0, v0, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v16, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v16, v16, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v16, v33, vcc
+; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v17, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v18, s4
; GFX9-NEXT: v_perm_b32 v2, v2, v19, s4
@@ -10082,7 +12081,14 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX9-NEXT: v_perm_b32 v5, v5, v22, s4
; GFX9-NEXT: v_perm_b32 v6, v6, v23, s4
; GFX9-NEXT: v_perm_b32 v7, v7, v24, s4
-; GFX9-NEXT: v_perm_b32 v15, v15, v32, s4
+; GFX9-NEXT: v_perm_b32 v8, v8, v25, s4
+; GFX9-NEXT: v_perm_b32 v9, v9, v26, s4
+; GFX9-NEXT: v_perm_b32 v10, v10, v27, s4
+; GFX9-NEXT: v_perm_b32 v11, v11, v28, s4
+; GFX9-NEXT: v_perm_b32 v12, v12, v32, s4
+; GFX9-NEXT: v_perm_b32 v13, v13, v30, s4
+; GFX9-NEXT: v_perm_b32 v14, v14, v31, s4
+; GFX9-NEXT: v_perm_b32 v15, v29, v15, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fadd_v32bf16:
@@ -10097,32 +12103,10 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v10
; GFX10-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v30
-; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v14
-; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
-; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v29
-; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v13
-; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v28
; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v12
; GFX10-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX10-NEXT: v_add_f32_e32 v39, v48, v39
-; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v17
-; GFX10-NEXT: v_add_f32_e32 v11, v11, v27
-; GFX10-NEXT: v_lshlrev_b32_e32 v27, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_add_f32_e32 v49, v50, v49
-; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v16
-; GFX10-NEXT: v_add_f32_e32 v10, v10, v26
-; GFX10-NEXT: v_lshlrev_b32_e32 v26, 16, v0
-; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v15
-; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX10-NEXT: v_lshlrev_b32_e32 v51, 16, v25
; GFX10-NEXT: v_lshlrev_b32_e32 v52, 16, v9
; GFX10-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
@@ -10141,29 +12125,28 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX10-NEXT: v_lshlrev_b32_e32 v67, 16, v21
; GFX10-NEXT: v_lshlrev_b32_e32 v68, 16, v5
-; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT: v_add_f32_e32 v33, v34, v33
-; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v20
-; GFX10-NEXT: v_add_f32_e32 v14, v14, v30
-; GFX10-NEXT: v_lshlrev_b32_e32 v30, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v35, v36, v35
-; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v19
-; GFX10-NEXT: v_add_f32_e32 v13, v13, v29
-; GFX10-NEXT: v_lshlrev_b32_e32 v29, 16, v3
-; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v39, v48, v39
+; GFX10-NEXT: v_add_f32_e32 v11, v11, v27
+; GFX10-NEXT: v_add_f32_e32 v49, v50, v49
+; GFX10-NEXT: v_add_f32_e32 v10, v10, v26
+; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v29
+; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v13
+; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX10-NEXT: v_add_f32_e32 v37, v38, v37
; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v18
; GFX10-NEXT: v_add_f32_e32 v12, v12, v28
; GFX10-NEXT: v_lshlrev_b32_e32 v28, 16, v2
; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_add_f32_e32 v0, v0, v16
-; GFX10-NEXT: v_add_f32_e32 v1, v1, v17
-; GFX10-NEXT: v_add_f32_e32 v51, v52, v51
+; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v17
+; GFX10-NEXT: v_lshlrev_b32_e32 v27, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v16
+; GFX10-NEXT: v_lshlrev_b32_e32 v26, 16, v0
+; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX10-NEXT: v_add_f32_e32 v9, v9, v25
; GFX10-NEXT: v_add_f32_e32 v25, v54, v53
; GFX10-NEXT: v_add_f32_e32 v8, v8, v24
@@ -10172,36 +12155,220 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX10-NEXT: v_add_f32_e32 v23, v66, v65
; GFX10-NEXT: v_add_f32_e32 v6, v6, v22
; GFX10-NEXT: v_add_f32_e32 v22, v68, v67
-; GFX10-NEXT: v_add_f32_e32 v5, v5, v21
-; GFX10-NEXT: v_add_f32_e32 v21, v30, v34
-; GFX10-NEXT: v_add_f32_e32 v29, v29, v36
-; GFX10-NEXT: v_add_f32_e32 v28, v28, v38
-; GFX10-NEXT: v_add_f32_e32 v27, v27, v48
-; GFX10-NEXT: v_add_f32_e32 v26, v26, v50
+; GFX10-NEXT: v_bfe_u32 v53, v39, 16, 1
+; GFX10-NEXT: v_bfe_u32 v55, v11, 16, 1
+; GFX10-NEXT: v_bfe_u32 v65, v49, 16, 1
+; GFX10-NEXT: v_bfe_u32 v67, v10, 16, 1
+; GFX10-NEXT: s_brev_b32 s23, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v30
+; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v14
+; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; GFX10-NEXT: v_add_f32_e32 v35, v36, v35
+; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v19
+; GFX10-NEXT: v_add_f32_e32 v13, v13, v29
+; GFX10-NEXT: v_lshlrev_b32_e32 v29, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX10-NEXT: v_add_f32_e32 v2, v2, v18
+; GFX10-NEXT: v_add_f32_e32 v18, v27, v48
+; GFX10-NEXT: v_add_f32_e32 v1, v1, v17
+; GFX10-NEXT: v_add_f32_e32 v17, v26, v50
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v16
+; GFX10-NEXT: v_and_or_b32 v54, v39, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v64, v11, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v66, v49, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v68, v10, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s9, v39, v39
+; GFX10-NEXT: v_add3_u32 v39, v53, v39, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s10, v11, v11
+; GFX10-NEXT: v_add3_u32 v11, v55, v11, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s11, v49, v49
+; GFX10-NEXT: v_add3_u32 v49, v65, v49, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s12, v10, v10
+; GFX10-NEXT: v_add3_u32 v10, v67, v10, 0x7fff
+; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX10-NEXT: v_add_f32_e32 v33, v34, v33
+; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v20
+; GFX10-NEXT: v_add_f32_e32 v14, v14, v30
+; GFX10-NEXT: v_lshlrev_b32_e32 v30, 16, v4
+; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX10-NEXT: v_add_f32_e32 v3, v3, v19
+; GFX10-NEXT: v_add_f32_e32 v19, v28, v38
+; GFX10-NEXT: v_bfe_u32 v38, v37, 16, 1
+; GFX10-NEXT: v_bfe_u32 v50, v12, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v39, v39, v54, s9
+; GFX10-NEXT: v_bfe_u32 v54, v18, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v11, v11, v64, s10
+; GFX10-NEXT: v_bfe_u32 v64, v1, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v49, v49, v66, s11
+; GFX10-NEXT: v_bfe_u32 v66, v17, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v68, s12
+; GFX10-NEXT: v_bfe_u32 v68, v0, 16, 1
+; GFX10-NEXT: v_add_f32_e32 v51, v52, v51
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v21
+; GFX10-NEXT: v_add_f32_e32 v21, v30, v34
; GFX10-NEXT: v_add_f32_e32 v4, v4, v20
-; GFX10-NEXT: v_perm_b32 v1, v1, v27, 0x7060302
-; GFX10-NEXT: v_perm_b32 v0, v0, v26, 0x7060302
-; GFX10-NEXT: v_perm_b32 v2, v2, v28, 0x7060302
-; GFX10-NEXT: v_perm_b32 v3, v3, v29, 0x7060302
+; GFX10-NEXT: v_add_f32_e32 v20, v29, v36
+; GFX10-NEXT: v_bfe_u32 v16, v33, 16, 1
+; GFX10-NEXT: v_bfe_u32 v27, v14, 16, 1
+; GFX10-NEXT: v_bfe_u32 v29, v35, 16, 1
+; GFX10-NEXT: v_bfe_u32 v34, v13, 16, 1
+; GFX10-NEXT: v_and_or_b32 v48, v37, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v52, v12, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s7, v37, v37
+; GFX10-NEXT: v_add3_u32 v37, v38, v37, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s8, v12, v12
+; GFX10-NEXT: v_add3_u32 v12, v50, v12, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s10, v18, v18
+; GFX10-NEXT: v_add3_u32 v54, v54, v18, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v18, v18, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s11, v1, v1
+; GFX10-NEXT: v_add3_u32 v64, v64, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v1, v1, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s12, v17, v17
+; GFX10-NEXT: v_add3_u32 v66, v66, v17, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v17, v17, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s22, v0, v0
+; GFX10-NEXT: v_add3_u32 v68, v68, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v0, v0, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v26, v33, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v28, v14, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v30, v35, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v36, v13, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX10-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v33, v51, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v14, v14
+; GFX10-NEXT: v_add3_u32 v14, v27, v14, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s5, v35, v35
+; GFX10-NEXT: v_add3_u32 v29, v29, v35, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s6, v13, v13
+; GFX10-NEXT: v_add3_u32 v13, v34, v13, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v65, v24, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v37, v37, v48, s7
+; GFX10-NEXT: v_bfe_u32 v48, v19, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v12, v12, v52, s8
+; GFX10-NEXT: v_bfe_u32 v52, v2, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v18, v54, v18, s10
+; GFX10-NEXT: v_cndmask_b32_e64 v17, v66, v17, s12
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v68, v0, s22
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v64, v1, s11
+; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v15
+; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX10-NEXT: v_and_or_b32 v27, v51, s23, 0x400000
+; GFX10-NEXT: v_bfe_u32 v35, v9, 16, 1
+; GFX10-NEXT: v_bfe_u32 v38, v25, 16, 1
+; GFX10-NEXT: v_and_or_b32 v67, v24, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s13, v51, v51
+; GFX10-NEXT: v_add3_u32 v33, v33, v51, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v51, v7, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s17, v24, v24
+; GFX10-NEXT: v_add3_u32 v24, v65, v24, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v65, v6, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v16, v16, v26, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v26, v21, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v14, v14, v28, s4
+; GFX10-NEXT: v_bfe_u32 v28, v4, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v29, v29, v30, s5
+; GFX10-NEXT: v_bfe_u32 v30, v20, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v13, v13, v36, s6
+; GFX10-NEXT: v_bfe_u32 v36, v3, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s8, v19, v19
+; GFX10-NEXT: v_add3_u32 v48, v48, v19, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v19, v19, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s9, v2, v2
+; GFX10-NEXT: v_add3_u32 v52, v52, v2, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v2, v2, s23, 0x400000
+; GFX10-NEXT: v_perm_b32 v0, v0, v17, 0x7060302
+; GFX10-NEXT: v_perm_b32 v1, v1, v18, 0x7060302
+; GFX10-NEXT: v_and_or_b32 v34, v9, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v50, v25, s23, 0x400000
+; GFX10-NEXT: v_bfe_u32 v53, v8, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s14, v9, v9
+; GFX10-NEXT: v_add3_u32 v9, v35, v9, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v35, v7, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s15, v25, v25
+; GFX10-NEXT: v_add3_u32 v25, v38, v25, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v38, v23, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s18, v7, v7
+; GFX10-NEXT: v_add3_u32 v7, v51, v7, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v51, v6, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s20, v6, v6
+; GFX10-NEXT: v_add3_u32 v6, v65, v6, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v65, v5, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v21, v21
+; GFX10-NEXT: v_add3_u32 v26, v26, v21, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v21, v21, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s5, v4, v4
+; GFX10-NEXT: v_add3_u32 v28, v28, v4, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v4, v4, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s6, v20, v20
+; GFX10-NEXT: v_add3_u32 v30, v30, v20, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v20, v20, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s7, v3, v3
+; GFX10-NEXT: v_add3_u32 v36, v36, v3, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v3, v3, s23, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e64 v19, v48, v19, s8
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v52, v2, s9
+; GFX10-NEXT: v_and_or_b32 v55, v8, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s16, v8, v8
+; GFX10-NEXT: v_add3_u32 v8, v53, v8, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v53, v23, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s19, v23, v23
+; GFX10-NEXT: v_add3_u32 v23, v38, v23, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v38, v22, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v65, v65, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v5, v5, s23, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e64 v21, v26, v21, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v28, v4, s5
+; GFX10-NEXT: v_cndmask_b32_e64 v20, v30, v20, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v36, v3, s7
+; GFX10-NEXT: v_perm_b32 v2, v2, v19, 0x7060302
+; GFX10-NEXT: v_cmp_u_f32_e64 s21, v22, v22
+; GFX10-NEXT: v_add3_u32 v38, v38, v22, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v22, v22, s23, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v65, v5, vcc_lo
+; GFX10-NEXT: v_perm_b32 v3, v3, v20, 0x7060302
; GFX10-NEXT: v_perm_b32 v4, v4, v21, 0x7060302
-; GFX10-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
-; GFX10-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
-; GFX10-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e64 v27, v33, v27, s13
+; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v34, s14
+; GFX10-NEXT: v_cndmask_b32_e64 v25, v25, v50, s15
+; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, v55, s16
+; GFX10-NEXT: v_cndmask_b32_e64 v24, v24, v67, s17
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v35, s18
+; GFX10-NEXT: v_cndmask_b32_e64 v23, v23, v53, s19
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v51, s20
+; GFX10-NEXT: v_cndmask_b32_e64 v22, v38, v22, s21
; GFX10-NEXT: v_perm_b32 v8, v8, v25, 0x7060302
-; GFX10-NEXT: v_perm_b32 v9, v9, v51, 0x7060302
+; GFX10-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
+; GFX10-NEXT: v_perm_b32 v9, v9, v27, 0x7060302
+; GFX10-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
+; GFX10-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
; GFX10-NEXT: v_perm_b32 v10, v10, v49, 0x7060302
; GFX10-NEXT: v_perm_b32 v11, v11, v39, 0x7060302
; GFX10-NEXT: v_perm_b32 v12, v12, v37, 0x7060302
-; GFX10-NEXT: v_perm_b32 v13, v13, v35, 0x7060302
-; GFX10-NEXT: v_perm_b32 v14, v14, v33, 0x7060302
+; GFX10-NEXT: v_perm_b32 v13, v13, v29, 0x7060302
+; GFX10-NEXT: v_perm_b32 v14, v14, v16, 0x7060302
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v31
-; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v31
-; GFX10-NEXT: v_add_f32_e32 v16, v32, v16
-; GFX10-NEXT: v_add_f32_e32 v15, v15, v17
-; GFX10-NEXT: v_perm_b32 v15, v15, v16, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v31
+; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v31
+; GFX10-NEXT: v_add_f32_e32 v17, v32, v17
+; GFX10-NEXT: v_add_f32_e32 v15, v15, v18
+; GFX10-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX10-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX10-NEXT: v_and_or_b32 v20, v17, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v21, v15, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v15, v15
+; GFX10-NEXT: v_add3_u32 v17, v18, v17, 0x7fff
+; GFX10-NEXT: v_add3_u32 v15, v19, v15, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v17, v17, v20, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v15, v15, v21, s4
+; GFX10-NEXT: v_perm_b32 v15, v15, v17, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fadd_v32bf16:
@@ -10212,102 +12379,269 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX11-NEXT: v_lshlrev_b32_e32 v84, 16, v1
; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v24
+; GFX11-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_dual_add_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v64, 16, v7
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v18
; GFX11-NEXT: v_lshlrev_b32_e32 v85, 16, v16
; GFX11-NEXT: v_lshlrev_b32_e32 v86, 16, v0
+; GFX11-NEXT: v_bfe_u32 v135, v1, 16, 1
; GFX11-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v23
+; GFX11-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; GFX11-NEXT: v_and_or_b32 v144, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v54, 16, v8
-; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v7
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: v_dual_add_f32 v17, v86, v85 :: v_dual_and_b32 v8, 0xffff0000, v8
+; GFX11-NEXT: v_dual_add_f32 v7, v7, v23 :: v_dual_lshlrev_b32 v36, 16, v13
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_add_f32 v8, v8, v24 :: v_dual_lshlrev_b32 v39, 16, v27
+; GFX11-NEXT: v_dual_add_f32 v0, v0, v16 :: v_dual_lshlrev_b32 v49, 16, v26
+; GFX11-NEXT: v_add_f32_e32 v24, v64, v55
+; GFX11-NEXT: v_bfe_u32 v87, v7, 16, 1
; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v22
; GFX11-NEXT: v_lshlrev_b32_e32 v66, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v11
-; GFX11-NEXT: v_dual_add_f32 v0, v0, v16 :: v_dual_and_b32 v11, 0xffff0000, v11
; GFX11-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX11-NEXT: v_bfe_u32 v85, v24, 16, 1
; GFX11-NEXT: v_lshlrev_b32_e32 v67, 16, v21
; GFX11-NEXT: v_lshlrev_b32_e32 v68, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v51, 16, v25
; GFX11-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v20
; GFX11-NEXT: v_lshlrev_b32_e32 v70, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v23
-; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v19
-; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v3
-; GFX11-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v9
-; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v18
-; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v2
+; GFX11-NEXT: v_and_or_b32 v86, v24, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v96, v7, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v85, v85, v24, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v20
+; GFX11-NEXT: v_add3_u32 v87, v87, v7, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX11-NEXT: v_dual_add_f32 v23, v66, v65 :: v_dual_lshlrev_b32 v48, 16, v11
+; GFX11-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; GFX11-NEXT: v_dual_add_f32 v5, v5, v21 :: v_dual_lshlrev_b32 v50, 16, v10
+; GFX11-NEXT: v_dual_add_f32 v21, v70, v69 :: v_dual_and_b32 v26, 0xffff0000, v26
; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v24
-; GFX11-NEXT: v_dual_add_f32 v1, v1, v17 :: v_dual_and_b32 v24, 0xffff0000, v24
-; GFX11-NEXT: v_dual_add_f32 v5, v5, v21 :: v_dual_lshlrev_b32 v50, 16, v10
-; GFX11-NEXT: v_dual_add_f32 v21, v70, v69 :: v_dual_and_b32 v10, 0xffff0000, v10
-; GFX11-NEXT: v_dual_add_f32 v2, v2, v18 :: v_dual_add_f32 v3, v3, v19
-; GFX11-NEXT: v_dual_add_f32 v4, v4, v20 :: v_dual_lshlrev_b32 v49, 16, v26
-; GFX11-NEXT: v_dual_add_f32 v9, v9, v25 :: v_dual_and_b32 v26, 0xffff0000, v26
; GFX11-NEXT: v_add_f32_e32 v6, v6, v22
-; GFX11-NEXT: v_dual_add_f32 v22, v68, v67 :: v_dual_lshlrev_b32 v37, 16, v28
-; GFX11-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_f32_e32 v10, v10, v26
-; GFX11-NEXT: v_add_f32_e32 v26, v52, v51
-; GFX11-NEXT: v_perm_b32 v4, v4, v21, 0x7060302
-; GFX11-NEXT: v_add_f32_e32 v25, v54, v53
-; GFX11-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
-; GFX11-NEXT: v_perm_b32 v9, v9, v26, 0x7060302
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v31
-; GFX11-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v31
-; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-NEXT: v_lshlrev_b32_e32 v36, 16, v13
-; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX11-NEXT: v_lshlrev_b32_e32 v39, 16, v27
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_dual_add_f32 v8, v8, v24 :: v_dual_and_b32 v27, 0xffff0000, v27
-; GFX11-NEXT: v_add_f32_e32 v24, v64, v55
+; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v9
+; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; GFX11-NEXT: v_bfe_u32 v97, v23, 16, 1
+; GFX11-NEXT: v_add_f32_e32 v2, v2, v18
+; GFX11-NEXT: v_add_f32_e32 v18, v84, v83
+; GFX11-NEXT: v_bfe_u32 v83, v8, 16, 1
+; GFX11-NEXT: v_bfe_u32 v99, v6, 16, 1
+; GFX11-NEXT: v_bfe_u32 v103, v5, 16, 1
+; GFX11-NEXT: v_bfe_u32 v113, v21, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v19
+; GFX11-NEXT: v_and_or_b32 v84, v8, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v98, v23, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v100, v6, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v112, v5, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v114, v21, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v83, v83, v8, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; GFX11-NEXT: v_add3_u32 v97, v97, v23, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; GFX11-NEXT: v_add3_u32 v99, v99, v6, 0x7fff
+; GFX11-NEXT: v_add3_u32 v103, v103, v5, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-NEXT: v_add3_u32 v113, v113, v21, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v12
-; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX11-NEXT: v_dual_add_f32 v3, v3, v19 :: v_dual_and_b32 v10, 0xffff0000, v10
+; GFX11-NEXT: v_dual_add_f32 v22, v68, v67 :: v_dual_lshlrev_b32 v51, 16, v25
+; GFX11-NEXT: v_lshlrev_b32_e32 v37, 16, v28
+; GFX11-NEXT: v_dual_add_f32 v4, v4, v20 :: v_dual_and_b32 v25, 0xffff0000, v25
+; GFX11-NEXT: v_add_f32_e32 v20, v80, v71
+; GFX11-NEXT: v_dual_add_f32 v19, v82, v81 :: v_dual_and_b32 v28, 0xffff0000, v28
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_dual_add_f32 v9, v9, v25 :: v_dual_and_b32 v12, 0xffff0000, v12
+; GFX11-NEXT: v_add_f32_e32 v25, v54, v53
; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v29
-; GFX11-NEXT: v_add_f32_e32 v7, v7, v23
-; GFX11-NEXT: v_add_f32_e32 v23, v66, v65
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_add_f32 v12, v12, v28 :: v_dual_and_b32 v29, 0xffff0000, v29
-; GFX11-NEXT: v_dual_add_f32 v28, v48, v39 :: v_dual_lshlrev_b32 v33, 16, v30
+; GFX11-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX11-NEXT: v_dual_add_f32 v10, v10, v26 :: v_dual_and_b32 v13, 0xffff0000, v13
+; GFX11-NEXT: v_dual_add_f32 v12, v12, v28 :: v_dual_lshlrev_b32 v33, 16, v30
+; GFX11-NEXT: v_add_f32_e32 v28, v48, v39
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_dual_add_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v34, 16, v14
-; GFX11-NEXT: v_lshlrev_b32_e32 v32, 16, v15
; GFX11-NEXT: v_dual_add_f32 v11, v11, v27 :: v_dual_and_b32 v14, 0xffff0000, v14
-; GFX11-NEXT: v_dual_add_f32 v27, v50, v49 :: v_dual_and_b32 v30, 0xffff0000, v30
-; GFX11-NEXT: v_add_f32_e32 v29, v38, v37
+; GFX11-NEXT: v_dual_add_f32 v27, v50, v49 :: v_dual_add_f32 v26, v52, v51
+; GFX11-NEXT: v_dual_add_f32 v29, v38, v37 :: v_dual_and_b32 v30, 0xffff0000, v30
+; GFX11-NEXT: v_lshlrev_b32_e32 v32, 16, v15
; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX11-NEXT: v_add_f32_e32 v37, v86, v85
-; GFX11-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v39, v13, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_add_f32_e32 v14, v14, v30
; GFX11-NEXT: v_dual_add_f32 v30, v36, v35 :: v_dual_add_f32 v33, v34, v33
-; GFX11-NEXT: v_dual_add_f32 v34, v80, v71 :: v_dual_add_f32 v35, v82, v81
-; GFX11-NEXT: v_add_f32_e32 v36, v84, v83
-; GFX11-NEXT: v_dual_add_f32 v16, v32, v16 :: v_dual_add_f32 v15, v15, v17
-; GFX11-NEXT: v_perm_b32 v0, v0, v37, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v2, v2, v35, 0x7060302
-; GFX11-NEXT: v_perm_b32 v1, v1, v36, 0x7060302
-; GFX11-NEXT: v_perm_b32 v3, v3, v34, 0x7060302
-; GFX11-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
-; GFX11-NEXT: v_perm_b32 v8, v8, v25, 0x7060302
+; GFX11-NEXT: v_and_or_b32 v48, v13, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v49, v29, 16, 1
+; GFX11-NEXT: v_bfe_u32 v35, v14, 16, 1
+; GFX11-NEXT: v_and_or_b32 v36, v14, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v16, v33, 16, 1
+; GFX11-NEXT: v_and_or_b32 v34, v33, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX11-NEXT: v_bfe_u32 v37, v30, 16, 1
+; GFX11-NEXT: v_add3_u32 v35, v35, v14, 0x7fff
+; GFX11-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v38, v30, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v39, v39, v13, 0x7fff
+; GFX11-NEXT: v_add3_u32 v37, v37, v30, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v50, v29, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v16, v16, v34, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-NEXT: v_bfe_u32 v51, v12, 16, 1
+; GFX11-NEXT: v_add3_u32 v49, v49, v29, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v52, v12, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v53, v28, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v14, v35, v36, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
+; GFX11-NEXT: v_add3_u32 v51, v51, v12, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v54, v28, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v55, v11, 16, 1
+; GFX11-NEXT: v_add3_u32 v53, v53, v28, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v30, v37, v38, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-NEXT: v_and_or_b32 v64, v11, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v65, v27, 16, 1
+; GFX11-NEXT: v_add3_u32 v55, v55, v11, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v66, v27, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v13, v39, v48, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-NEXT: v_bfe_u32 v67, v10, 16, 1
+; GFX11-NEXT: v_add3_u32 v65, v65, v27, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v68, v10, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v69, v26, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v29, v49, v50, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-NEXT: v_add3_u32 v67, v67, v10, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v70, v26, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v71, v9, 16, 1
+; GFX11-NEXT: v_add3_u32 v69, v69, v26, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v12, v51, v52, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX11-NEXT: v_and_or_b32 v80, v9, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v81, v25, 16, 1
+; GFX11-NEXT: v_add3_u32 v71, v71, v9, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v82, v25, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v28, v53, v54, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-NEXT: v_add3_u32 v81, v81, v25, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v101, v22, 16, 1
+; GFX11-NEXT: v_and_or_b32 v102, v22, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v115, v4, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v11, v55, v64, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX11-NEXT: v_add3_u32 v101, v101, v22, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v116, v4, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v117, v20, 16, 1
+; GFX11-NEXT: v_add3_u32 v115, v115, v4, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v27, v65, v66, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-NEXT: v_and_or_b32 v118, v20, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v129, v19, 16, 1
+; GFX11-NEXT: v_add3_u32 v117, v117, v20, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v130, v19, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v10, v67, v68, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-NEXT: v_bfe_u32 v133, v18, 16, 1
+; GFX11-NEXT: v_add3_u32 v129, v129, v19, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v134, v18, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v145, v17, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v26, v69, v70, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-NEXT: v_add3_u32 v133, v133, v18, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v146, v17, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v147, v0, 16, 1
+; GFX11-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v9, v71, v80, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-NEXT: v_bfe_u32 v131, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v33, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v147, v147, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v132, v2, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v25, v81, v82, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-NEXT: v_add3_u32 v131, v131, v2, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v119, v3, 16, 1
+; GFX11-NEXT: v_and_or_b32 v128, v3, s0, 0x400000
+; GFX11-NEXT: v_perm_b32 v9, v9, v26, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v8, v83, v84, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-NEXT: v_add3_u32 v119, v119, v3, 0x7fff
; GFX11-NEXT: v_perm_b32 v10, v10, v27, 0x7060302
; GFX11-NEXT: v_perm_b32 v11, v11, v28, 0x7060302
+; GFX11-NEXT: v_perm_b32 v8, v8, v25, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v24, v85, v86, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11-NEXT: v_perm_b32 v12, v12, v29, 0x7060302
; GFX11-NEXT: v_perm_b32 v13, v13, v30, 0x7060302
-; GFX11-NEXT: v_perm_b32 v14, v14, v33, 0x7060302
-; GFX11-NEXT: v_perm_b32 v15, v15, v16, 0x7060302
+; GFX11-NEXT: v_perm_b32 v14, v14, v16, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v87, v96, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v23, v97, v98, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: v_cndmask_b32_e32 v6, v99, v100, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v22, v101, v102, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v103, v112, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v21, v113, v114, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v115, v116, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-NEXT: v_perm_b32 v4, v4, v21, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v20, v117, v118, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-NEXT: v_cndmask_b32_e32 v19, v129, v130, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-NEXT: v_cndmask_b32_e32 v18, v133, v134, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v135, v144, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, v1, v18, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v17, v145, v146, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v147, v33, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_perm_b32 v0, v0, v17, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v131, v132, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v119, v128, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_perm_b32 v3, v3, v20, 0x7060302
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v31
+; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v31
+; GFX11-NEXT: v_perm_b32 v2, v2, v19, 0x7060302
+; GFX11-NEXT: v_add_f32_e32 v17, v32, v17
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_f32_e32 v15, v15, v18
+; GFX11-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11-NEXT: v_and_or_b32 v20, v17, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: v_and_or_b32 v21, v15, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v18, v18, v17, 0x7fff
+; GFX11-NEXT: v_add3_u32 v19, v19, v15, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v17, v18, v20, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-NEXT: v_cndmask_b32_e32 v15, v19, v21, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v15, v15, v17, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fadd <32 x bfloat> %a, %b
ret <32 x bfloat> %op
@@ -10317,6 +12651,7 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) {
; GCN-LABEL: v_fadd_bf16_fpimm_0:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_add_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -10325,6 +12660,7 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) {
; GFX7-LABEL: v_fadd_bf16_fpimm_0:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_add_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -10335,6 +12671,13 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -10343,6 +12686,13 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -10350,7 +12700,13 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -10358,8 +12714,16 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%add = fadd bfloat %arg0, 1.0
@@ -10370,6 +12734,7 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) {
; GCN-LABEL: v_fadd_bf16_fpimm_1:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_add_f32_e32 v0, 0x42280000, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -10378,6 +12743,7 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) {
; GFX7-LABEL: v_fadd_bf16_fpimm_1:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_add_f32_e32 v0, 0x42280000, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -10388,6 +12754,13 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_add_f32_e32 v0, 0x42280000, v0
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -10396,6 +12769,13 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_add_f32_e32 v0, 0x42280000, v0
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -10403,7 +12783,13 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_add_f32_e32 v0, 0x42280000, v0
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -10411,8 +12797,16 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_add_f32_e32 v0, 0x42280000, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%add = fadd bfloat %arg0, 42.0
@@ -10423,6 +12817,8 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fsub_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_sub_f32_e32 v0, v0, v1
@@ -10432,6 +12828,8 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fsub_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_sub_f32_e32 v0, v0, v1
@@ -10444,6 +12842,13 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) {
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -10453,6 +12858,13 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) {
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -10461,7 +12873,13 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -10470,8 +12888,16 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fsub bfloat %a, %b
@@ -10482,6 +12908,10 @@ define <2 x bfloat> @v_fsub_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_fsub_v2bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -10495,6 +12925,10 @@ define <2 x bfloat> @v_fsub_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX7-LABEL: v_fsub_v2bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -10510,10 +12944,24 @@ define <2 x bfloat> @v_fsub_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX8-NEXT: v_sub_f32_e32 v2, v3, v2
+; GFX8-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v2
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v2
; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX8-NEXT: v_sub_f32_e32 v2, v3, v2
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v2, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -10523,10 +12971,23 @@ define <2 x bfloat> @v_fsub_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX9-NEXT: v_sub_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_sub_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_add3_u32 v3, v3, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v2, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -10538,9 +12999,20 @@ define <2 x bfloat> @v_fsub_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_sub_f32_e32 v2, v3, v2
; GFX10-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_and_or_b32 v5, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fsub_v2bf16:
@@ -10550,11 +13022,24 @@ define <2 x bfloat> @v_fsub_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1
; GFX11-NEXT: v_sub_f32_e32 v2, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v2, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_and_or_b32 v5, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fsub <2 x bfloat> %a, %b
ret <2 x bfloat> %op
@@ -10564,6 +13049,12 @@ define <3 x bfloat> @v_fsub_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GCN-LABEL: v_fsub_v3bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
@@ -10581,6 +13072,12 @@ define <3 x bfloat> @v_fsub_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX7-LABEL: v_fsub_v3bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
@@ -10601,12 +13098,34 @@ define <3 x bfloat> @v_fsub_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_sub_f32_e32 v3, v4, v3
+; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v3
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX8-NEXT: v_sub_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_sub_f32_e32 v3, v4, v3
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
@@ -10618,12 +13137,31 @@ define <3 x bfloat> @v_fsub_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_sub_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_sub_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_add3_u32 v4, v4, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16
@@ -10632,16 +13170,32 @@ define <3 x bfloat> @v_fsub_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX10-LABEL: v_fsub_v3bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_sub_f32_e32 v4, v5, v4
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_sub_f32_e32 v0, v0, v2
; GFX10-NEXT: v_sub_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
+; GFX10-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v4, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v6, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = fsub <3 x bfloat> %a, %b
@@ -10652,6 +13206,14 @@ define <4 x bfloat> @v_fsub_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GCN-LABEL: v_fsub_v4bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
@@ -10673,6 +13235,14 @@ define <4 x bfloat> @v_fsub_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX7-LABEL: v_fsub_v4bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
@@ -10696,17 +13266,46 @@ define <4 x bfloat> @v_fsub_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX8-NEXT: v_sub_f32_e32 v4, v5, v4
+; GFX8-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v4
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_sub_f32_e32 v4, v5, v4
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX8-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX8-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v3
; GFX8-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_sub_f32_e32 v3, v5, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16
@@ -10717,16 +13316,41 @@ define <4 x bfloat> @v_fsub_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX9-NEXT: v_sub_f32_e32 v4, v5, v4
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_sub_f32_e32 v4, v5, v4
+; GFX9-NEXT: v_add3_u32 v5, v5, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; GFX9-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX9-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4
@@ -10738,17 +13362,38 @@ define <4 x bfloat> @v_fsub_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX10-NEXT: v_sub_f32_e32 v4, v5, v4
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_sub_f32_e32 v4, v5, v4
+; GFX10-NEXT: v_sub_f32_e32 v1, v1, v3
; GFX10-NEXT: v_sub_f32_e32 v5, v7, v6
+; GFX10-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_sub_f32_e32 v0, v0, v2
-; GFX10-NEXT: v_sub_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
-; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX10-NEXT: v_and_or_b32 v6, v4, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v9, v1, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v6, v5, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc_lo
+; GFX10-NEXT: v_perm_b32 v1, v1, v3, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fsub_v4bf16:
@@ -10756,19 +13401,45 @@ define <4 x bfloat> @v_fsub_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_sub_f32 v0, v0, v2 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_dual_sub_f32 v4, v5, v4 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_sub_f32 v0, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-NEXT: v_sub_f32_e32 v4, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v3, v4, 16, 1
; GFX11-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX11-NEXT: v_and_or_b32 v6, v4, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_and_or_b32 v9, v1, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v6, v5, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v8, v0, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc_lo
+; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
+; GFX11-NEXT: v_perm_b32 v1, v1, v3, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fsub <4 x bfloat> %a, %b
ret <4 x bfloat> %op
@@ -10778,6 +13449,8 @@ define bfloat @v_fmul_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fmul_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
@@ -10787,6 +13460,8 @@ define bfloat @v_fmul_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fmul_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
@@ -10799,6 +13474,13 @@ define bfloat @v_fmul_bf16(bfloat %a, bfloat %b) {
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -10808,6 +13490,13 @@ define bfloat @v_fmul_bf16(bfloat %a, bfloat %b) {
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -10816,7 +13505,13 @@ define bfloat @v_fmul_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -10825,8 +13520,16 @@ define bfloat @v_fmul_bf16(bfloat %a, bfloat %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fmul bfloat %a, %b
@@ -10837,6 +13540,10 @@ define <2 x bfloat> @v_fmul_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_fmul_v2bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -10850,6 +13557,10 @@ define <2 x bfloat> @v_fmul_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX7-LABEL: v_fmul_v2bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -10865,10 +13576,24 @@ define <2 x bfloat> @v_fmul_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX8-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX8-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v2
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v2
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX8-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v2, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -10878,10 +13603,23 @@ define <2 x bfloat> @v_fmul_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_add3_u32 v3, v3, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v2, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -10893,9 +13631,20 @@ define <2 x bfloat> @v_fmul_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_mul_f32_e32 v2, v3, v2
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_and_or_b32 v5, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fmul_v2bf16:
@@ -10905,11 +13654,24 @@ define <2 x bfloat> @v_fmul_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v2, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_and_or_b32 v5, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fmul <2 x bfloat> %a, %b
ret <2 x bfloat> %op
@@ -10919,6 +13681,12 @@ define <3 x bfloat> @v_fmul_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GCN-LABEL: v_fmul_v3bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
@@ -10936,6 +13704,12 @@ define <3 x bfloat> @v_fmul_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX7-LABEL: v_fmul_v3bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
@@ -10956,12 +13730,34 @@ define <3 x bfloat> @v_fmul_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v3
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
@@ -10973,12 +13769,31 @@ define <3 x bfloat> @v_fmul_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_add3_u32 v4, v4, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16
@@ -10987,16 +13802,32 @@ define <3 x bfloat> @v_fmul_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX10-LABEL: v_fmul_v3bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_mul_f32_e32 v4, v5, v4
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v2
; GFX10-NEXT: v_mul_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
+; GFX10-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v4, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v6, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = fmul <3 x bfloat> %a, %b
@@ -11007,6 +13838,14 @@ define <4 x bfloat> @v_fmul_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GCN-LABEL: v_fmul_v4bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
@@ -11028,6 +13867,14 @@ define <4 x bfloat> @v_fmul_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX7-LABEL: v_fmul_v4bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
@@ -11051,17 +13898,46 @@ define <4 x bfloat> @v_fmul_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX8-NEXT: v_mul_f32_e32 v4, v5, v4
+; GFX8-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v4
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_mul_f32_e32 v4, v5, v4
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX8-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX8-NEXT: v_mul_f32_e32 v3, v5, v3
+; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v3
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_mul_f32_e32 v3, v5, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16
@@ -11072,16 +13948,41 @@ define <4 x bfloat> @v_fmul_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX9-NEXT: v_mul_f32_e32 v4, v5, v4
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_mul_f32_e32 v4, v5, v4
+; GFX9-NEXT: v_add3_u32 v5, v5, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX9-NEXT: v_mul_f32_e32 v3, v5, v3
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_mul_f32_e32 v3, v5, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4
@@ -11093,17 +13994,38 @@ define <4 x bfloat> @v_fmul_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX10-NEXT: v_mul_f32_e32 v4, v5, v4
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_mul_f32_e32 v4, v5, v4
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v3
; GFX10-NEXT: v_mul_f32_e32 v5, v7, v6
+; GFX10-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX10-NEXT: v_mul_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
-; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX10-NEXT: v_and_or_b32 v6, v4, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v9, v1, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v6, v5, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc_lo
+; GFX10-NEXT: v_perm_b32 v1, v1, v3, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fmul_v4bf16:
@@ -11111,19 +14033,45 @@ define <4 x bfloat> @v_fmul_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_dual_mul_f32 v4, v5, v4 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-NEXT: v_mul_f32_e32 v4, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v3, v4, 16, 1
; GFX11-NEXT: v_mul_f32_e32 v5, v7, v6
-; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX11-NEXT: v_and_or_b32 v6, v4, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_and_or_b32 v9, v1, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v6, v5, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v8, v0, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc_lo
+; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
+; GFX11-NEXT: v_perm_b32 v1, v1, v3, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fmul <4 x bfloat> %a, %b
ret <4 x bfloat> %op
@@ -11133,6 +14081,22 @@ define <8 x bfloat> @v_fmul_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GCN-LABEL: v_fmul_v8bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
@@ -11170,6 +14134,22 @@ define <8 x bfloat> @v_fmul_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX7-LABEL: v_fmul_v8bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
@@ -11209,31 +14189,88 @@ define <8 x bfloat> @v_fmul_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX8-NEXT: v_mul_f32_e32 v8, v9, v8
+; GFX8-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v8
; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_mul_f32_e32 v8, v9, v8
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v8
; GFX8-NEXT: v_mul_f32_e32 v3, v3, v7
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, s4, v7
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX8-NEXT: v_mul_f32_e32 v7, v9, v7
+; GFX8-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v7
; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: v_mul_f32_e32 v7, v9, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v7
; GFX8-NEXT: v_mul_f32_e32 v2, v2, v6
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, s4, v6
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v6, v9, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v5
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX8-NEXT: v_mul_f32_e32 v6, v9, v6
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_mul_f32_e32 v6, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v6
; GFX8-NEXT: v_mul_f32_e32 v1, v1, v5
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX8-NEXT: v_mul_f32_e32 v5, v9, v5
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v5
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v9, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_mul_f32_e32 v5, v9, v5
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v5, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v6, 16
@@ -11246,28 +14283,77 @@ define <8 x bfloat> @v_fmul_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX9-NEXT: v_mul_f32_e32 v8, v9, v8
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v8
; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_mul_f32_e32 v8, v9, v8
+; GFX9-NEXT: v_add3_u32 v9, v9, v8, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX9-NEXT: v_mul_f32_e32 v3, v3, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v7, v7, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX9-NEXT: v_mul_f32_e32 v7, v9, v7
+; GFX9-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v7
; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_mul_f32_e32 v7, v9, v7
+; GFX9-NEXT: v_add3_u32 v9, v9, v7, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_mul_f32_e32 v2, v2, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v9, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v5
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX9-NEXT: v_mul_f32_e32 v6, v9, v6
+; GFX9-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v6
; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_mul_f32_e32 v6, v9, v6
+; GFX9-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX9-NEXT: v_mul_f32_e32 v1, v1, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v5, v5, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX9-NEXT: v_mul_f32_e32 v5, v9, v5
+; GFX9-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v5
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_mul_f32_e32 v5, v9, v5
+; GFX9-NEXT: v_add3_u32 v9, v9, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v9, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v5, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v6, s4
@@ -11280,65 +14366,157 @@ define <8 x bfloat> @v_fmul_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v6
-; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_mul_f32_e32 v8, v9, v8
+; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v6
+; GFX10-NEXT: v_mul_f32_e32 v3, v3, v7
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX10-NEXT: v_mul_f32_e32 v9, v11, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v8, s4, 0x400000
+; GFX10-NEXT: v_mul_f32_e32 v9, v11, v9
+; GFX10-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX10-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX10-NEXT: v_mul_f32_e32 v2, v2, v6
+; GFX10-NEXT: v_bfe_u32 v8, v9, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v5
+; GFX10-NEXT: v_and_or_b32 v12, v9, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX10-NEXT: v_add3_u32 v10, v11, v3, 0x7fff
; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX10-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX10-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v0
+; GFX10-NEXT: v_mul_f32_e32 v6, v11, v6
+; GFX10-NEXT: v_add3_u32 v9, v13, v2, 0x7fff
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v11, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v12, v6, 16, 1
; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_mul_f32_e32 v10, v11, v10
-; GFX10-NEXT: v_mul_f32_e32 v11, v13, v12
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX10-NEXT: v_mul_f32_e32 v1, v1, v5
-; GFX10-NEXT: v_mul_f32_e32 v2, v2, v6
-; GFX10-NEXT: v_mul_f32_e32 v3, v3, v7
-; GFX10-NEXT: v_perm_b32 v0, v0, v11, 0x7060302
-; GFX10-NEXT: v_perm_b32 v1, v1, v10, 0x7060302
-; GFX10-NEXT: v_perm_b32 v2, v2, v9, 0x7060302
-; GFX10-NEXT: v_perm_b32 v3, v3, v8, 0x7060302
+; GFX10-NEXT: v_mul_f32_e32 v5, v15, v13
+; GFX10-NEXT: v_and_or_b32 v14, v3, s4, 0x400000
+; GFX10-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v9, v11, vcc_lo
+; GFX10-NEXT: v_add3_u32 v4, v12, v6, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v9, v6, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GFX10-NEXT: v_bfe_u32 v12, v5, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_bfe_u32 v13, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v15, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v6, v11, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v11, v5, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc_lo
+; GFX10-NEXT: v_add3_u32 v9, v12, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v13, v0, s4, 0x400000
+; GFX10-NEXT: v_perm_b32 v2, v2, v8, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v6, v15, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v10, v14, vcc_lo
+; GFX10-NEXT: v_perm_b32 v3, v3, v7, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fmul_v8bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-NEXT: v_dual_mul_f32 v8, v9, v8 :: v_dual_mul_f32 v9, v11, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mul_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v9, 16, v6
+; GFX11-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_mul_f32_e32 v9, v11, v9
+; GFX11-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v12, v9, s0, 0x400000
+; GFX11-NEXT: v_mul_f32_e32 v2, v2, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v5
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX11-NEXT: v_mul_f32_e32 v3, v3, v7
+; GFX11-NEXT: v_and_or_b32 v7, v8, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v8, v9, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX11-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-NEXT: v_add3_u32 v9, v13, v2, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_and_or_b32 v14, v3, s0, 0x400000
; GFX11-NEXT: v_mul_f32_e32 v0, v0, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-NEXT: v_add3_u32 v10, v11, v3, 0x7fff
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_mul_f32 v6, v11, v6 :: v_dual_and_b32 v1, 0xffff0000, v1
+; GFX11-NEXT: v_and_or_b32 v11, v2, s0, 0x400000
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_dual_mul_f32 v1, v1, v5 :: v_dual_and_b32 v6, 0xffff0000, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mul_f32 v2, v2, v6 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11-NEXT: v_mul_f32_e32 v3, v3, v7
-; GFX11-NEXT: v_dual_mul_f32 v10, v11, v10 :: v_dual_mul_f32 v11, v13, v12
+; GFX11-NEXT: v_bfe_u32 v12, v6, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v9, v11, vcc_lo
+; GFX11-NEXT: v_and_or_b32 v9, v6, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: v_mul_f32_e32 v1, v1, v5
+; GFX11-NEXT: v_add3_u32 v4, v12, v6, 0x7fff
+; GFX11-NEXT: v_perm_b32 v2, v2, v8, 0x7060302
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_perm_b32 v2, v2, v9, 0x7060302
-; GFX11-NEXT: v_perm_b32 v3, v3, v8, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v1, v1, v10, 0x7060302
-; GFX11-NEXT: v_perm_b32 v0, v0, v11, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc_lo
+; GFX11-NEXT: v_mul_f32_e32 v5, v15, v13
+; GFX11-NEXT: v_bfe_u32 v13, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v15, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v6, v11, v1, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v12, v5, 16, 1
+; GFX11-NEXT: v_and_or_b32 v11, v5, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_add3_u32 v9, v12, v5, 0x7fff
+; GFX11-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v13, v0, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v6, v15, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v10, v14, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v3, v3, v7, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fmul <8 x bfloat> %a, %b
ret <8 x bfloat> %op
@@ -11348,36 +14526,67 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GCN-LABEL: v_fmul_v16bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GCN-NEXT: v_mul_f32_e32 v14, v14, v30
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GCN-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GCN-NEXT: v_mul_f32_e32 v13, v13, v29
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GCN-NEXT: v_mul_f32_e32 v12, v12, v28
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GCN-NEXT: v_mul_f32_e32 v11, v11, v27
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GCN-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GCN-NEXT: v_mul_f32_e32 v10, v10, v26
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GCN-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GCN-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GCN-NEXT: v_mul_f32_e32 v9, v9, v25
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GCN-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GCN-NEXT: v_mul_f32_e32 v8, v8, v24
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GCN-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_mul_f32_e32 v7, v7, v23
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GCN-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GCN-NEXT: v_mul_f32_e32 v6, v6, v22
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GCN-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_mul_f32_e32 v5, v5, v21
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GCN-NEXT: v_mul_f32_e32 v4, v4, v20
@@ -11410,7 +14619,8 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v20
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v20
+; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GCN-NEXT: v_mul_f32_e32 v15, v15, v16
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
@@ -11419,12 +14629,41 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-LABEL: v_fmul_v16bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX7-NEXT: v_mul_f32_e32 v4, v4, v20
-; GFX7-NEXT: buffer_load_dword v20, off, s[0:3], s32
-; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX7-NEXT: v_mul_f32_e32 v6, v6, v22
+; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
@@ -11441,18 +14680,19 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v16
+; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_mul_f32_e32 v14, v14, v30
; GFX7-NEXT: v_mul_f32_e32 v13, v13, v29
; GFX7-NEXT: v_mul_f32_e32 v12, v12, v28
@@ -11461,11 +14701,12 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_mul_f32_e32 v9, v9, v25
; GFX7-NEXT: v_mul_f32_e32 v8, v8, v24
; GFX7-NEXT: v_mul_f32_e32 v7, v7, v23
-; GFX7-NEXT: v_mul_f32_e32 v6, v6, v22
; GFX7-NEXT: v_mul_f32_e32 v5, v5, v21
+; GFX7-NEXT: v_mul_f32_e32 v4, v4, v20
; GFX7-NEXT: v_mul_f32_e32 v3, v3, v19
; GFX7-NEXT: v_mul_f32_e32 v2, v2, v18
; GFX7-NEXT: v_mul_f32_e32 v1, v1, v17
+; GFX7-NEXT: v_mul_f32_e32 v0, v0, v16
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -11473,6 +14714,10 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; GFX7-NEXT: v_mul_f32_e32 v15, v15, v22
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
@@ -11481,9 +14726,6 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v20
-; GFX7-NEXT: v_mul_f32_e32 v15, v15, v16
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -11492,51 +14734,165 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v15
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v7
+; GFX8-NEXT: v_mul_f32_e32 v16, v17, v16
+; GFX8-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX8-NEXT: v_mul_f32_e32 v16, v17, v16
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v16
; GFX8-NEXT: v_mul_f32_e32 v7, v7, v15
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
+; GFX8-NEXT: v_bfe_u32 v15, v7, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, v15, v7
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, s4, v15
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v7
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v15, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX8-NEXT: v_mul_f32_e32 v15, v17, v15
+; GFX8-NEXT: v_bfe_u32 v17, v15, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v15
; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX8-NEXT: v_mul_f32_e32 v15, v17, v15
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v15
; GFX8-NEXT: v_mul_f32_e32 v6, v6, v14
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v15, v15
+; GFX8-NEXT: v_bfe_u32 v14, v6, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v14, vcc, v14, v6
+; GFX8-NEXT: v_add_u32_e32 v14, vcc, s4, v14
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v6
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v13
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX8-NEXT: v_mul_f32_e32 v14, v17, v14
+; GFX8-NEXT: v_bfe_u32 v17, v14, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v14
; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX8-NEXT: v_mul_f32_e32 v14, v17, v14
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v14
; GFX8-NEXT: v_mul_f32_e32 v5, v5, v13
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
+; GFX8-NEXT: v_bfe_u32 v13, v5, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v14, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v13, vcc, v13, v5
+; GFX8-NEXT: v_add_u32_e32 v13, vcc, s4, v13
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v5
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v13, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v12
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v4
+; GFX8-NEXT: v_mul_f32_e32 v13, v17, v13
+; GFX8-NEXT: v_bfe_u32 v17, v13, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v13
; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX8-NEXT: v_mul_f32_e32 v13, v17, v13
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v13
; GFX8-NEXT: v_mul_f32_e32 v4, v4, v12
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX8-NEXT: v_bfe_u32 v12, v4, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v13, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v12, vcc, v12, v4
+; GFX8-NEXT: v_add_u32_e32 v12, vcc, s4, v12
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v4
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v12, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v12, 16, v11
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX8-NEXT: v_mul_f32_e32 v12, v17, v12
+; GFX8-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v12
; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_mul_f32_e32 v12, v17, v12
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v12
; GFX8-NEXT: v_mul_f32_e32 v3, v3, v11
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX8-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v11, vcc, v11, v3
+; GFX8-NEXT: v_add_u32_e32 v11, vcc, s4, v11
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v11, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v2
+; GFX8-NEXT: v_mul_f32_e32 v11, v17, v11
+; GFX8-NEXT: v_bfe_u32 v17, v11, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v11
; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: v_mul_f32_e32 v11, v17, v11
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v11
; GFX8-NEXT: v_mul_f32_e32 v2, v2, v10
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
+; GFX8-NEXT: v_bfe_u32 v10, v2, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v11, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, v10, v2
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, s4, v10
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v10, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v9
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; GFX8-NEXT: v_mul_f32_e32 v10, v17, v10
+; GFX8-NEXT: v_bfe_u32 v17, v10, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v10
; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_mul_f32_e32 v10, v17, v10
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v10
; GFX8-NEXT: v_mul_f32_e32 v1, v1, v9
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
+; GFX8-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v10, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v0
+; GFX8-NEXT: v_mul_f32_e32 v9, v17, v9
+; GFX8-NEXT: v_bfe_u32 v17, v9, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v9
; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v9
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v8
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v9, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, s4, v8
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v17, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
@@ -11544,7 +14900,6 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_mul_f32_e32 v9, v17, v9
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v9, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v10, 16
@@ -11561,52 +14916,149 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v15
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v7
+; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v16
; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16
+; GFX9-NEXT: v_add3_u32 v17, v17, v16, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_mul_f32_e32 v7, v7, v15
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v15, v7, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v7
+; GFX9-NEXT: v_add3_u32 v15, v15, v7, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v15, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX9-NEXT: v_mul_f32_e32 v15, v17, v15
+; GFX9-NEXT: v_bfe_u32 v17, v15, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v15
; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX9-NEXT: v_mul_f32_e32 v15, v17, v15
+; GFX9-NEXT: v_add3_u32 v17, v17, v15, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v15, v15
; GFX9-NEXT: v_mul_f32_e32 v6, v6, v14
+; GFX9-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v14, v6, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v6
+; GFX9-NEXT: v_add3_u32 v14, v14, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v14, 16, v13
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX9-NEXT: v_mul_f32_e32 v14, v17, v14
+; GFX9-NEXT: v_bfe_u32 v17, v14, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v14
; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX9-NEXT: v_mul_f32_e32 v14, v17, v14
+; GFX9-NEXT: v_add3_u32 v17, v17, v14, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
; GFX9-NEXT: v_mul_f32_e32 v5, v5, v13
+; GFX9-NEXT: v_cndmask_b32_e32 v14, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v13, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v5
+; GFX9-NEXT: v_add3_u32 v13, v13, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v13, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v13, 16, v12
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v4
+; GFX9-NEXT: v_mul_f32_e32 v13, v17, v13
+; GFX9-NEXT: v_bfe_u32 v17, v13, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v13
; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX9-NEXT: v_mul_f32_e32 v13, v17, v13
+; GFX9-NEXT: v_add3_u32 v17, v17, v13, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
; GFX9-NEXT: v_mul_f32_e32 v4, v4, v12
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v12, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v12, v12, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v12, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v11
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX9-NEXT: v_mul_f32_e32 v12, v17, v12
+; GFX9-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v12
; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_mul_f32_e32 v12, v17, v12
+; GFX9-NEXT: v_add3_u32 v17, v17, v12, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
; GFX9-NEXT: v_mul_f32_e32 v3, v3, v11
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v11, v11, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v11, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v2
+; GFX9-NEXT: v_mul_f32_e32 v11, v17, v11
+; GFX9-NEXT: v_bfe_u32 v17, v11, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v11
; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_mul_f32_e32 v11, v17, v11
+; GFX9-NEXT: v_add3_u32 v17, v17, v11, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
; GFX9-NEXT: v_mul_f32_e32 v2, v2, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v11, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v10, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v10, v10, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v9
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; GFX9-NEXT: v_mul_f32_e32 v10, v17, v10
+; GFX9-NEXT: v_bfe_u32 v17, v10, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v10
; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_mul_f32_e32 v10, v17, v10
+; GFX9-NEXT: v_add3_u32 v17, v17, v10, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
; GFX9-NEXT: v_mul_f32_e32 v1, v1, v9
+; GFX9-NEXT: v_cndmask_b32_e32 v10, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v9, v9, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v0
+; GFX9-NEXT: v_mul_f32_e32 v9, v17, v9
+; GFX9-NEXT: v_bfe_u32 v17, v9, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v9
; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_mul_f32_e32 v9, v17, v9
+; GFX9-NEXT: v_add3_u32 v17, v17, v9, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
; GFX9-NEXT: v_mul_f32_e32 v0, v0, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v8, v8, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v17, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v9, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v10, s4
@@ -11625,119 +15077,297 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v7
; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v5
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v6
; GFX10-NEXT: v_mul_f32_e32 v16, v17, v16
-; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v14
; GFX10-NEXT: v_mul_f32_e32 v7, v7, v15
-; GFX10-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v12
-; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v4
-; GFX10-NEXT: v_mul_f32_e32 v15, v17, v15
+; GFX10-NEXT: v_bfe_u32 v15, v16, 16, 1
+; GFX10-NEXT: v_and_or_b32 v20, v16, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v19, v7, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX10-NEXT: v_mul_f32_e32 v17, v18, v17
+; GFX10-NEXT: v_add3_u32 v15, v15, v16, 0x7fff
; GFX10-NEXT: v_mul_f32_e32 v6, v6, v14
-; GFX10-NEXT: v_mul_f32_e32 v14, v19, v18
+; GFX10-NEXT: v_add3_u32 v18, v19, v7, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v19, v7, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v21, v17, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v15, v15, v20, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v5
+; GFX10-NEXT: v_and_or_b32 v16, v17, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v14, v21, v17, 0x7fff
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v7, v18, v19, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX10-NEXT: v_bfe_u32 v18, v6, 16, 1
+; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT: v_perm_b32 v7, v7, v15, 0x7060302
+; GFX10-NEXT: v_mul_f32_e32 v17, v20, v19
+; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v4
; GFX10-NEXT: v_mul_f32_e32 v5, v5, v13
-; GFX10-NEXT: v_mul_f32_e32 v13, v21, v20
-; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v11
-; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v14, v14, v16, vcc_lo
+; GFX10-NEXT: v_add3_u32 v16, v18, v6, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v13, v6, s4, 0x400000
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX10-NEXT: v_bfe_u32 v20, v17, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_bfe_u32 v21, v5, 16, 1
; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT: v_mul_f32_e32 v17, v18, v17
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v16, v13, vcc_lo
+; GFX10-NEXT: v_mul_f32_e32 v13, v19, v18
+; GFX10-NEXT: v_add3_u32 v16, v20, v17, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v18, v17, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT: v_add3_u32 v19, v21, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v20, v5, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v21, v13, 16, 1
+; GFX10-NEXT: v_mul_f32_e32 v4, v4, v12
+; GFX10-NEXT: v_cndmask_b32_e32 v16, v16, v18, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v11
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v3
+; GFX10-NEXT: v_add3_u32 v17, v21, v13, 0x7fff
; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v19, v20, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v19, v13, s4, 0x400000
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT: v_mul_f32_e32 v18, v20, v19
+; GFX10-NEXT: v_mul_f32_e32 v12, v18, v12
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v10
+; GFX10-NEXT: v_mul_f32_e32 v3, v3, v11
+; GFX10-NEXT: v_and_or_b32 v22, v12, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v13, v17, v19, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v2
+; GFX10-NEXT: v_add3_u32 v11, v20, v4, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v20, v3, 16, 1
; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v9
-; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v1
+; GFX10-NEXT: v_add3_u32 v17, v17, v12, 0x7fff
+; GFX10-NEXT: v_mul_f32_e32 v18, v19, v18
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX10-NEXT: v_add3_u32 v19, v20, v3, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v20, v3, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v23, v18, 16, 1
+; GFX10-NEXT: v_mul_f32_e32 v2, v2, v10
+; GFX10-NEXT: v_cndmask_b32_e32 v12, v17, v22, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_and_or_b32 v17, v18, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v10, v23, v18, 0x7fff
+; GFX10-NEXT: v_lshlrev_b32_e32 v22, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v19, v20, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v19, v2, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v9
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX10-NEXT: v_and_or_b32 v18, v2, s4, 0x400000
+; GFX10-NEXT: v_and_or_b32 v21, v4, s4, 0x400000
+; GFX10-NEXT: v_perm_b32 v3, v3, v12, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v10, v10, v17, vcc_lo
+; GFX10-NEXT: v_add3_u32 v17, v19, v2, 0x7fff
+; GFX10-NEXT: v_mul_f32_e32 v19, v22, v20
+; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v8
; GFX10-NEXT: v_lshlrev_b32_e32 v22, 16, v0
; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_mul_f32_e32 v19, v20, v19
-; GFX10-NEXT: v_mul_f32_e32 v20, v22, v21
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v8
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: v_bfe_u32 v23, v19, 16, 1
; GFX10-NEXT: v_mul_f32_e32 v1, v1, v9
-; GFX10-NEXT: v_mul_f32_e32 v2, v2, v10
-; GFX10-NEXT: v_mul_f32_e32 v3, v3, v11
-; GFX10-NEXT: v_mul_f32_e32 v4, v4, v12
-; GFX10-NEXT: v_perm_b32 v0, v0, v20, 0x7060302
+; GFX10-NEXT: v_mul_f32_e32 v9, v22, v20
+; GFX10-NEXT: v_and_or_b32 v22, v19, s4, 0x400000
+; GFX10-NEXT: v_mul_f32_e32 v0, v0, v8
+; GFX10-NEXT: v_add3_u32 v20, v23, v19, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX10-NEXT: v_bfe_u32 v23, v9, 16, 1
+; GFX10-NEXT: v_and_or_b32 v24, v9, s4, 0x400000
+; GFX10-NEXT: v_and_or_b32 v25, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v22, v1, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_bfe_u32 v20, v0, 16, 1
+; GFX10-NEXT: v_add3_u32 v23, v23, v9, 0x7fff
+; GFX10-NEXT: v_perm_b32 v5, v5, v16, 0x7060302
+; GFX10-NEXT: v_perm_b32 v6, v6, v14, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v8, v22, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX10-NEXT: v_add3_u32 v20, v20, v0, 0x7fff
; GFX10-NEXT: v_perm_b32 v1, v1, v19, 0x7060302
-; GFX10-NEXT: v_perm_b32 v2, v2, v18, 0x7060302
-; GFX10-NEXT: v_perm_b32 v3, v3, v17, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v8, v23, v24, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v20, v25, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_perm_b32 v0, v0, v8, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_perm_b32 v2, v2, v10, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v11, v21, vcc_lo
; GFX10-NEXT: v_perm_b32 v4, v4, v13, 0x7060302
-; GFX10-NEXT: v_perm_b32 v5, v5, v14, 0x7060302
-; GFX10-NEXT: v_perm_b32 v6, v6, v15, 0x7060302
-; GFX10-NEXT: v_perm_b32 v7, v7, v16, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fmul_v16bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v0
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mul_f32 v4, v4, v12 :: v_dual_and_b32 v5, 0xffff0000, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v15
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v7
-; GFX11-NEXT: v_mul_f32_e32 v5, v5, v13
-; GFX11-NEXT: v_mul_f32_e32 v13, v21, v20
-; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v8
-; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-NEXT: v_dual_mul_f32 v16, v17, v16 :: v_dual_and_b32 v15, 0xffff0000, v15
-; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mul_f32 v0, v0, v8 :: v_dual_and_b32 v7, 0xffff0000, v7
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-NEXT: v_perm_b32 v4, v4, v13, 0x7060302
-; GFX11-NEXT: v_mul_f32_e32 v7, v7, v15
-; GFX11-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mul_f32 v16, v17, v16 :: v_dual_lshlrev_b32 v17, 16, v14
; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v7, v7, v16, 0x7060302
-; GFX11-NEXT: v_mul_f32_e32 v15, v17, v15
+; GFX11-NEXT: v_and_or_b32 v20, v16, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_mul_f32_e32 v17, v18, v17
+; GFX11-NEXT: v_mul_f32_e32 v6, v6, v14
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_bfe_u32 v21, v17, 16, 1
+; GFX11-NEXT: v_add3_u32 v14, v21, v17, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_f32_e32 v7, v7, v15
+; GFX11-NEXT: v_bfe_u32 v15, v16, 16, 1
+; GFX11-NEXT: v_add3_u32 v15, v15, v16, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v16, v17, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v15, v15, v20 :: v_dual_lshlrev_b32 v20, 16, v5
+; GFX11-NEXT: v_bfe_u32 v19, v7, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-NEXT: v_add3_u32 v18, v19, v7, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v19, v7, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v18, v19, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v18, v6, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: v_perm_b32 v7, v7, v15, 0x7060302
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mul_f32 v6, v6, v14 :: v_dual_lshlrev_b32 v17, 16, v11
-; GFX11-NEXT: v_mul_f32_e32 v14, v19, v18
+; GFX11-NEXT: v_dual_mul_f32 v17, v20, v19 :: v_dual_cndmask_b32 v14, v14, v16
+; GFX11-NEXT: v_add3_u32 v16, v18, v6, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; GFX11-NEXT: v_bfe_u32 v20, v17, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_f32_e32 v4, v4, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v11
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX11-NEXT: v_mul_f32_e32 v5, v5, v13
+; GFX11-NEXT: v_and_or_b32 v13, v6, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_cndmask_b32 v6, v16, v13 :: v_dual_mul_f32 v13, v19, v18
+; GFX11-NEXT: v_add3_u32 v16, v20, v17, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v18, v17, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: v_perm_b32 v6, v6, v14, 0x7060302
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v16, v16, v18, vcc_lo
; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v10
+; GFX11-NEXT: v_bfe_u32 v21, v5, 16, 1
+; GFX11-NEXT: v_and_or_b32 v20, v5, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_mul_f32_e32 v12, v18, v12
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v19, v21, v5, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v21, v13, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v10
+; GFX11-NEXT: v_and_or_b32 v22, v12, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v19, v20, vcc_lo
+; GFX11-NEXT: v_add3_u32 v17, v21, v13, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v19, v13, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX11-NEXT: v_and_or_b32 v21, v4, s0, 0x400000
+; GFX11-NEXT: v_perm_b32 v5, v5, v16, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v13, v17, v19, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v2
; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-NEXT: v_add3_u32 v17, v17, v12, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mul_f32_e32 v18, v19, v18
+; GFX11-NEXT: v_cndmask_b32_e32 v12, v17, v22, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v1
+; GFX11-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_dual_mul_f32 v17, v18, v17 :: v_dual_and_b32 v10, 0xffff0000, v10
-; GFX11-NEXT: v_perm_b32 v5, v5, v14, 0x7060302
-; GFX11-NEXT: v_perm_b32 v6, v6, v15, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v23, v18, 16, 1
+; GFX11-NEXT: v_and_or_b32 v17, v18, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_mul_f32 v2, v2, v10 :: v_dual_and_b32 v1, 0xffff0000, v1
; GFX11-NEXT: v_mul_f32_e32 v3, v3, v11
-; GFX11-NEXT: v_dual_mul_f32 v18, v20, v19 :: v_dual_lshlrev_b32 v19, 16, v9
-; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v1
+; GFX11-NEXT: v_add3_u32 v11, v20, v4, 0x7fff
+; GFX11-NEXT: v_add3_u32 v10, v23, v18, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v20, v3, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v19, v20, v3, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v20, v3, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v19, v20, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v19, v2, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v9
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-NEXT: v_and_or_b32 v18, v2, s0, 0x400000
+; GFX11-NEXT: v_perm_b32 v3, v3, v12, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v10, v10, v17, vcc_lo
+; GFX11-NEXT: v_add3_u32 v17, v19, v2, 0x7fff
+; GFX11-NEXT: v_mul_f32_e32 v19, v22, v20
+; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v8
+; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX11-NEXT: v_dual_mul_f32 v2, v2, v10 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_perm_b32 v3, v3, v17, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mul_f32 v19, v20, v19 :: v_dual_mul_f32 v20, v22, v21
-; GFX11-NEXT: v_mul_f32_e32 v1, v1, v9
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_perm_b32 v2, v2, v18, 0x7060302
-; GFX11-NEXT: v_perm_b32 v0, v0, v20, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v23, v19, 16, 1
+; GFX11-NEXT: v_dual_mul_f32 v0, v0, v8 :: v_dual_mul_f32 v1, v1, v9
+; GFX11-NEXT: v_mul_f32_e32 v9, v22, v20
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v20, v23, v19, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v22, v19, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v25, v0, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-NEXT: v_bfe_u32 v23, v9, 16, 1
+; GFX11-NEXT: v_and_or_b32 v24, v9, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX11-NEXT: v_and_or_b32 v22, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_bfe_u32 v20, v0, 16, 1
+; GFX11-NEXT: v_add3_u32 v23, v23, v9, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v8, v22, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-NEXT: v_add3_u32 v20, v20, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_perm_b32 v1, v1, v19, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v8, v23, v24, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v20, v25, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v0, v0, v8, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_perm_b32 v2, v2, v10, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v11, v21, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v4, v4, v13, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fmul <16 x bfloat> %a, %b
ret <16 x bfloat> %op
@@ -11747,166 +15377,230 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GCN-LABEL: v_fmul_v32bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:128
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32
+; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128
; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:124
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
-; GCN-NEXT: v_mul_f32_e32 v31, v32, v31
+; GCN-NEXT: v_mul_f32_e32 v31, v31, v32
+; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:120
; GCN-NEXT: v_mul_f32_e32 v30, v30, v32
+; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GCN-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v29, v29, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:116
+; GCN-NEXT: v_mul_f32_e32 v29, v29, v32
+; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112
; GCN-NEXT: v_mul_f32_e32 v28, v28, v32
+; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v27, v27, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:108
+; GCN-NEXT: v_mul_f32_e32 v27, v27, v32
+; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:104
; GCN-NEXT: v_mul_f32_e32 v26, v26, v32
+; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GCN-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v25, v25, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:100
+; GCN-NEXT: v_mul_f32_e32 v25, v25, v32
+; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GCN-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:96
; GCN-NEXT: v_mul_f32_e32 v24, v24, v32
+; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GCN-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v23, v23, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:92
+; GCN-NEXT: v_mul_f32_e32 v23, v23, v32
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GCN-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:88
; GCN-NEXT: v_mul_f32_e32 v22, v22, v32
+; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GCN-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v21, v21, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:84
+; GCN-NEXT: v_mul_f32_e32 v21, v21, v32
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GCN-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:80
; GCN-NEXT: v_mul_f32_e32 v20, v20, v32
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GCN-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v19, v19, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:76
+; GCN-NEXT: v_mul_f32_e32 v19, v19, v32
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GCN-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:72
; GCN-NEXT: v_mul_f32_e32 v18, v18, v32
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GCN-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v17, v17, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:68
+; GCN-NEXT: v_mul_f32_e32 v17, v17, v32
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64
; GCN-NEXT: v_mul_f32_e32 v16, v16, v32
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v15, v15, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60
+; GCN-NEXT: v_mul_f32_e32 v15, v15, v32
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:56
; GCN-NEXT: v_mul_f32_e32 v14, v14, v32
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v13, v13, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:52
+; GCN-NEXT: v_mul_f32_e32 v13, v13, v32
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:48
; GCN-NEXT: v_mul_f32_e32 v12, v12, v32
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v11, v11, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:44
+; GCN-NEXT: v_mul_f32_e32 v11, v11, v32
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GCN-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:40
; GCN-NEXT: v_mul_f32_e32 v10, v10, v32
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GCN-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v9, v9, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:36
+; GCN-NEXT: v_mul_f32_e32 v9, v9, v32
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:32
; GCN-NEXT: v_mul_f32_e32 v8, v8, v32
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v7, v7, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:28
+; GCN-NEXT: v_mul_f32_e32 v7, v7, v32
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:24
; GCN-NEXT: v_mul_f32_e32 v6, v6, v32
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v5, v5, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:20
+; GCN-NEXT: v_mul_f32_e32 v5, v5, v32
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:16
; GCN-NEXT: v_mul_f32_e32 v4, v4, v32
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v3, v3, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12
+; GCN-NEXT: v_mul_f32_e32 v3, v3, v32
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
; GCN-NEXT: v_mul_f32_e32 v2, v2, v32
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; GCN-NEXT: v_mul_f32_e32 v1, v1, v33
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:4
+; GCN-NEXT: v_mul_f32_e32 v1, v1, v32
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: v_mul_f32_e32 v0, v0, v32
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -11946,197 +15640,261 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX7-LABEL: v_fmul_v32bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:128
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128
+; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
-; GFX7-NEXT: v_mul_f32_e32 v31, v32, v31
+; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; GFX7-NEXT: v_mul_f32_e32 v31, v31, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124
; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v30, v30, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120
; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v29, v29, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v28, v28, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112
; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v27, v27, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108
; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v26, v26, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104
; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v25, v25, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100
; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v24, v24, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96
; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v23, v23, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v22, v22, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88
; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v21, v21, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v20, v20, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80
; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v19, v19, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76
; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v18, v18, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72
; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v17, v17, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68
; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v16, v16, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64
; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v15, v15, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v14, v14, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v13, v13, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52
; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v12, v12, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48
; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v11, v11, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v10, v10, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v9, v9, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v8, v8, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v7, v7, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v6, v6, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v5, v5, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v4, v4, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v3, v3, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v2, v2, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v1, v1, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v0, v0, v32
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -12147,114 +15905,329 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v14
+; GFX8-NEXT: v_mul_f32_e32 v31, v32, v31
+; GFX8-NEXT: v_bfe_u32 v32, v31, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v32, vcc, v32, v31
; GFX8-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX8-NEXT: v_mul_f32_e32 v31, v32, v31
-; GFX8-NEXT: v_mul_f32_e32 v30, v14, v30
-; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v29
+; GFX8-NEXT: v_add_u32_e32 v32, vcc, s4, v32
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v31
+; GFX8-NEXT: v_mul_f32_e32 v14, v14, v30
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_bfe_u32 v30, v14, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v32, v33, vcc
+; GFX8-NEXT: v_add_u32_e32 v30, vcc, v30, v14
+; GFX8-NEXT: v_add_u32_e32 v30, vcc, s4, v30
+; GFX8-NEXT: v_and_b32_e32 v32, 0x80000000, v14
+; GFX8-NEXT: v_or_b32_e32 v32, 0x400000, v32
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
+; GFX8-NEXT: v_cndmask_b32_e32 v14, v30, v32, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v30, 16, v29
; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v13
+; GFX8-NEXT: v_mul_f32_e32 v32, v32, v30
+; GFX8-NEXT: buffer_load_dword v30, off, s[0:3], s32
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX8-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX8-NEXT: v_mul_f32_e32 v14, v32, v14
; GFX8-NEXT: v_mul_f32_e32 v13, v13, v29
+; GFX8-NEXT: v_bfe_u32 v29, v13, 16, 1
+; GFX8-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX8-NEXT: v_alignbit_b32 v14, v14, v31, 16
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v34, 16, v30
+; GFX8-NEXT: v_mul_f32_e32 v33, v33, v34
+; GFX8-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; GFX8-NEXT: v_mul_f32_e32 v30, v15, v30
+; GFX8-NEXT: v_bfe_u32 v15, v33, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, v15, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v33
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, s4, v15
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX8-NEXT: v_bfe_u32 v33, v30, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v15, v15, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v30
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v30
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
+; GFX8-NEXT: v_cndmask_b32_e32 v30, v33, v34, vcc
+; GFX8-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v32
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v32
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
+; GFX8-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v29, vcc, v29, v13
+; GFX8-NEXT: v_add_u32_e32 v29, vcc, s4, v29
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v13
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX8-NEXT: v_cndmask_b32_e32 v13, v29, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v29, 16, v28
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v12
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v12
+; GFX8-NEXT: v_mul_f32_e32 v29, v33, v29
+; GFX8-NEXT: v_bfe_u32 v33, v29, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v29
; GFX8-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX8-NEXT: v_mul_f32_e32 v29, v32, v29
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v29
; GFX8-NEXT: v_mul_f32_e32 v12, v12, v28
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v29, v29
+; GFX8-NEXT: v_bfe_u32 v28, v12, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v29, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v28, vcc, v28, v12
+; GFX8-NEXT: v_add_u32_e32 v28, vcc, s4, v28
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v12
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX8-NEXT: v_cndmask_b32_e32 v12, v28, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v28, 16, v27
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v11
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v11
+; GFX8-NEXT: v_mul_f32_e32 v28, v33, v28
+; GFX8-NEXT: v_bfe_u32 v33, v28, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v28
; GFX8-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; GFX8-NEXT: v_mul_f32_e32 v28, v32, v28
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v28
; GFX8-NEXT: v_mul_f32_e32 v11, v11, v27
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v28, v28
+; GFX8-NEXT: v_bfe_u32 v27, v11, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v28, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v27, vcc, v27, v11
+; GFX8-NEXT: v_add_u32_e32 v27, vcc, s4, v27
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v11
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
+; GFX8-NEXT: v_cndmask_b32_e32 v11, v27, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v27, 16, v26
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v10
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v10
+; GFX8-NEXT: v_mul_f32_e32 v27, v33, v27
+; GFX8-NEXT: v_bfe_u32 v33, v27, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v27
; GFX8-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX8-NEXT: v_mul_f32_e32 v27, v32, v27
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v27
; GFX8-NEXT: v_mul_f32_e32 v10, v10, v26
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v27, v27
+; GFX8-NEXT: v_bfe_u32 v26, v10, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v27, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v26, vcc, v26, v10
+; GFX8-NEXT: v_add_u32_e32 v26, vcc, s4, v26
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v10
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
+; GFX8-NEXT: v_cndmask_b32_e32 v10, v26, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v26, 16, v25
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v9
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v9
+; GFX8-NEXT: v_mul_f32_e32 v26, v33, v26
+; GFX8-NEXT: v_bfe_u32 v33, v26, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v26
; GFX8-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX8-NEXT: v_mul_f32_e32 v26, v32, v26
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v26
; GFX8-NEXT: v_mul_f32_e32 v9, v9, v25
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v26, v26
+; GFX8-NEXT: v_bfe_u32 v25, v9, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v26, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v25, vcc, v25, v9
+; GFX8-NEXT: v_add_u32_e32 v25, vcc, s4, v25
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v9
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX8-NEXT: v_cndmask_b32_e32 v9, v25, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v25, 16, v24
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v8
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v8
+; GFX8-NEXT: v_mul_f32_e32 v25, v33, v25
+; GFX8-NEXT: v_bfe_u32 v33, v25, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v25
; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX8-NEXT: v_mul_f32_e32 v8, v8, v24
-; GFX8-NEXT: buffer_load_dword v24, off, s[0:3], s32
-; GFX8-NEXT: v_mul_f32_e32 v25, v32, v25
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v15
-; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; GFX8-NEXT: v_alignbit_b32 v8, v8, v25, 16
-; GFX8-NEXT: v_alignbit_b32 v9, v9, v26, 16
-; GFX8-NEXT: v_alignbit_b32 v10, v10, v27, 16
-; GFX8-NEXT: v_alignbit_b32 v11, v11, v28, 16
-; GFX8-NEXT: v_alignbit_b32 v12, v12, v29, 16
-; GFX8-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v24
-; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; GFX8-NEXT: v_mul_f32_e32 v32, v32, v33
-; GFX8-NEXT: v_mul_f32_e32 v15, v15, v24
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v25
+; GFX8-NEXT: v_mul_f32_e32 v8, v8, v24
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v25, v25
+; GFX8-NEXT: v_bfe_u32 v24, v8, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v25, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v24, vcc, v24, v8
+; GFX8-NEXT: v_add_u32_e32 v24, vcc, s4, v24
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v8
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v24, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v24, 16, v23
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v7
+; GFX8-NEXT: v_mul_f32_e32 v24, v33, v24
+; GFX8-NEXT: v_bfe_u32 v33, v24, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v24
; GFX8-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX8-NEXT: v_mul_f32_e32 v24, v33, v24
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v24
; GFX8-NEXT: v_mul_f32_e32 v7, v7, v23
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v24, v24
+; GFX8-NEXT: v_bfe_u32 v23, v7, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v24, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v23, vcc, v23, v7
+; GFX8-NEXT: v_add_u32_e32 v23, vcc, s4, v23
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v7
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v23, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v23, 16, v22
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v6
+; GFX8-NEXT: v_mul_f32_e32 v23, v33, v23
+; GFX8-NEXT: v_bfe_u32 v33, v23, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v23
; GFX8-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX8-NEXT: v_mul_f32_e32 v23, v33, v23
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v23
; GFX8-NEXT: v_mul_f32_e32 v6, v6, v22
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v23, v23
+; GFX8-NEXT: v_bfe_u32 v22, v6, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v23, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v22, vcc, v22, v6
+; GFX8-NEXT: v_add_u32_e32 v22, vcc, s4, v22
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v6
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v22, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v22, 16, v21
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v5
+; GFX8-NEXT: v_mul_f32_e32 v22, v33, v22
+; GFX8-NEXT: v_bfe_u32 v33, v22, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v22
; GFX8-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX8-NEXT: v_mul_f32_e32 v22, v33, v22
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v22
; GFX8-NEXT: v_mul_f32_e32 v5, v5, v21
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v22, v22
+; GFX8-NEXT: v_bfe_u32 v21, v5, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v22, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v21, vcc, v21, v5
+; GFX8-NEXT: v_add_u32_e32 v21, vcc, s4, v21
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v5
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v21, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v21, 16, v20
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v4
+; GFX8-NEXT: v_mul_f32_e32 v21, v33, v21
+; GFX8-NEXT: v_bfe_u32 v33, v21, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v21
; GFX8-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX8-NEXT: v_mul_f32_e32 v21, v33, v21
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v21
; GFX8-NEXT: v_mul_f32_e32 v4, v4, v20
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v21, v21
+; GFX8-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v21, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v20, vcc, v20, v4
+; GFX8-NEXT: v_add_u32_e32 v20, vcc, s4, v20
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v4
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v20, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v20, 16, v19
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v3
+; GFX8-NEXT: v_mul_f32_e32 v20, v33, v20
+; GFX8-NEXT: v_bfe_u32 v33, v20, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v20
; GFX8-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_mul_f32_e32 v20, v33, v20
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v20
; GFX8-NEXT: v_mul_f32_e32 v3, v3, v19
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v20, v20
+; GFX8-NEXT: v_bfe_u32 v19, v3, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v20, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v19, vcc, v19, v3
+; GFX8-NEXT: v_add_u32_e32 v19, vcc, s4, v19
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v19, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v19, 16, v18
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v2
+; GFX8-NEXT: v_mul_f32_e32 v19, v33, v19
+; GFX8-NEXT: v_bfe_u32 v33, v19, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v19
; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: v_mul_f32_e32 v19, v33, v19
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v19
; GFX8-NEXT: v_mul_f32_e32 v2, v2, v18
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v19, v19
+; GFX8-NEXT: v_bfe_u32 v18, v2, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v19, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v18, vcc, v18, v2
+; GFX8-NEXT: v_add_u32_e32 v18, vcc, s4, v18
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v18, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v17
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v1
+; GFX8-NEXT: v_mul_f32_e32 v18, v33, v18
+; GFX8-NEXT: v_bfe_u32 v33, v18, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v18
; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_mul_f32_e32 v18, v33, v18
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v18
; GFX8-NEXT: v_mul_f32_e32 v1, v1, v17
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
+; GFX8-NEXT: v_bfe_u32 v17, v1, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v16
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v0
+; GFX8-NEXT: v_mul_f32_e32 v17, v33, v17
+; GFX8-NEXT: v_bfe_u32 v33, v17, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v17
; GFX8-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v17
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v16
-; GFX8-NEXT: v_mul_f32_e32 v17, v33, v17
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
+; GFX8-NEXT: v_bfe_u32 v16, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v16, vcc, v16, v0
+; GFX8-NEXT: v_add_u32_e32 v16, vcc, s4, v16
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v16, v33, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
@@ -12263,8 +16236,13 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v30
+; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11
; GFX8-NEXT: v_alignbit_b32 v0, v0, v17, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v18, 16
; GFX8-NEXT: v_alignbit_b32 v2, v2, v19, 16
@@ -12273,8 +16251,13 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX8-NEXT: v_alignbit_b32 v5, v5, v22, 16
; GFX8-NEXT: v_alignbit_b32 v6, v6, v23, 16
; GFX8-NEXT: v_alignbit_b32 v7, v7, v24, 16
-; GFX8-NEXT: v_alignbit_b32 v14, v16, v31, 16
-; GFX8-NEXT: v_alignbit_b32 v15, v15, v32, 16
+; GFX8-NEXT: v_alignbit_b32 v8, v8, v25, 16
+; GFX8-NEXT: v_alignbit_b32 v9, v9, v26, 16
+; GFX8-NEXT: v_alignbit_b32 v10, v10, v27, 16
+; GFX8-NEXT: v_alignbit_b32 v11, v11, v28, 16
+; GFX8-NEXT: v_alignbit_b32 v12, v12, v29, 16
+; GFX8-NEXT: v_alignbit_b32 v13, v13, v32, 16
+; GFX8-NEXT: v_alignbit_b32 v15, v16, v15, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fmul_v32bf16:
@@ -12282,110 +16265,296 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v14
+; GFX9-NEXT: v_mul_f32_e32 v31, v32, v31
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_bfe_u32 v32, v31, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v31
; GFX9-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX9-NEXT: v_mul_f32_e32 v31, v32, v31
+; GFX9-NEXT: v_add3_u32 v32, v32, v31, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
; GFX9-NEXT: v_mul_f32_e32 v14, v14, v30
+; GFX9-NEXT: v_cndmask_b32_e32 v31, v32, v33, vcc
+; GFX9-NEXT: v_bfe_u32 v30, v14, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v32, 0x80000000, v14
+; GFX9-NEXT: v_add3_u32 v30, v30, v14, s4
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
+; GFX9-NEXT: v_cndmask_b32_e32 v14, v30, v32, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v29
; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v13
; GFX9-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX9-NEXT: v_mul_f32_e32 v30, v32, v30
; GFX9-NEXT: v_mul_f32_e32 v13, v13, v29
-; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v28
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v12
+; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32
+; GFX9-NEXT: v_mul_f32_e32 v30, v32, v30
+; GFX9-NEXT: v_bfe_u32 v32, v30, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v30
+; GFX9-NEXT: v_add3_u32 v32, v32, v30, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
+; GFX9-NEXT: v_cndmask_b32_e32 v30, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v15
+; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v29
+; GFX9-NEXT: v_mul_f32_e32 v32, v32, v33
+; GFX9-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX9-NEXT: v_mul_f32_e32 v29, v15, v29
+; GFX9-NEXT: v_bfe_u32 v15, v32, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v32
+; GFX9-NEXT: v_add3_u32 v15, v15, v32, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v15, v15, v33, vcc
+; GFX9-NEXT: v_bfe_u32 v32, v29, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v29
+; GFX9-NEXT: v_add3_u32 v32, v32, v29, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v29, v29
+; GFX9-NEXT: v_cndmask_b32_e32 v29, v32, v33, vcc
+; GFX9-NEXT: v_bfe_u32 v32, v13, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v13
+; GFX9-NEXT: v_add3_u32 v32, v32, v13, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v12
+; GFX9-NEXT: v_mul_f32_e32 v32, v33, v32
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v32
; GFX9-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX9-NEXT: v_mul_f32_e32 v29, v32, v29
+; GFX9-NEXT: v_add3_u32 v33, v33, v32, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_mul_f32_e32 v12, v12, v28
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v28, v12, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v12
+; GFX9-NEXT: v_add3_u32 v28, v28, v12, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v28, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v27
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v11
+; GFX9-NEXT: v_mul_f32_e32 v28, v33, v28
+; GFX9-NEXT: v_bfe_u32 v33, v28, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v28
; GFX9-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; GFX9-NEXT: v_mul_f32_e32 v28, v32, v28
+; GFX9-NEXT: v_add3_u32 v33, v33, v28, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v28, v28
; GFX9-NEXT: v_mul_f32_e32 v11, v11, v27
+; GFX9-NEXT: v_cndmask_b32_e32 v28, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v27, v11, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v11
+; GFX9-NEXT: v_add3_u32 v27, v27, v11, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
+; GFX9-NEXT: v_cndmask_b32_e32 v11, v27, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v26
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v10
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v10
+; GFX9-NEXT: v_mul_f32_e32 v27, v33, v27
+; GFX9-NEXT: v_bfe_u32 v33, v27, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v27
; GFX9-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX9-NEXT: v_mul_f32_e32 v27, v32, v27
+; GFX9-NEXT: v_add3_u32 v33, v33, v27, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v27, v27
; GFX9-NEXT: v_mul_f32_e32 v10, v10, v26
+; GFX9-NEXT: v_cndmask_b32_e32 v27, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v26, v10, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v10
+; GFX9-NEXT: v_add3_u32 v26, v26, v10, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v10, v26, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v25
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v9
+; GFX9-NEXT: v_mul_f32_e32 v26, v33, v26
+; GFX9-NEXT: v_bfe_u32 v33, v26, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v26
; GFX9-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX9-NEXT: v_mul_f32_e32 v26, v32, v26
+; GFX9-NEXT: v_add3_u32 v33, v33, v26, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v26, v26
; GFX9-NEXT: v_mul_f32_e32 v9, v9, v25
+; GFX9-NEXT: v_cndmask_b32_e32 v26, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v25, v9, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v9
+; GFX9-NEXT: v_add3_u32 v25, v25, v9, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v25, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v24
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v8
+; GFX9-NEXT: v_mul_f32_e32 v25, v33, v25
+; GFX9-NEXT: v_bfe_u32 v33, v25, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v25
; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; GFX9-NEXT: v_add3_u32 v33, v33, v25, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v25, v25
; GFX9-NEXT: v_mul_f32_e32 v8, v8, v24
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32
-; GFX9-NEXT: v_mul_f32_e32 v25, v32, v25
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v15
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: v_perm_b32 v8, v8, v25, s4
-; GFX9-NEXT: v_perm_b32 v9, v9, v26, s4
-; GFX9-NEXT: v_perm_b32 v10, v10, v27, s4
-; GFX9-NEXT: v_perm_b32 v11, v11, v28, s4
-; GFX9-NEXT: v_perm_b32 v12, v12, v29, s4
-; GFX9-NEXT: v_perm_b32 v13, v13, v30, s4
-; GFX9-NEXT: v_perm_b32 v14, v14, v31, s4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v24
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; GFX9-NEXT: v_mul_f32_e32 v32, v32, v33
-; GFX9-NEXT: v_mul_f32_e32 v15, v15, v24
+; GFX9-NEXT: v_cndmask_b32_e32 v25, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v24, v8, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v8
+; GFX9-NEXT: v_add3_u32 v24, v24, v8, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v24, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v23
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v7
+; GFX9-NEXT: v_mul_f32_e32 v24, v33, v24
+; GFX9-NEXT: v_bfe_u32 v33, v24, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v24
; GFX9-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX9-NEXT: v_mul_f32_e32 v24, v33, v24
+; GFX9-NEXT: v_add3_u32 v33, v33, v24, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v24, v24
; GFX9-NEXT: v_mul_f32_e32 v7, v7, v23
+; GFX9-NEXT: v_cndmask_b32_e32 v24, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v23, v7, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v7
+; GFX9-NEXT: v_add3_u32 v23, v23, v7, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v23, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v22
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v6
+; GFX9-NEXT: v_mul_f32_e32 v23, v33, v23
+; GFX9-NEXT: v_bfe_u32 v33, v23, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v23
; GFX9-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX9-NEXT: v_mul_f32_e32 v23, v33, v23
+; GFX9-NEXT: v_add3_u32 v33, v33, v23, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v23, v23
; GFX9-NEXT: v_mul_f32_e32 v6, v6, v22
+; GFX9-NEXT: v_cndmask_b32_e32 v23, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v22, v6, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v6
+; GFX9-NEXT: v_add3_u32 v22, v22, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v22, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v21
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v5
+; GFX9-NEXT: v_mul_f32_e32 v22, v33, v22
+; GFX9-NEXT: v_bfe_u32 v33, v22, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v22
; GFX9-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX9-NEXT: v_mul_f32_e32 v22, v33, v22
+; GFX9-NEXT: v_add3_u32 v33, v33, v22, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v22, v22
; GFX9-NEXT: v_mul_f32_e32 v5, v5, v21
+; GFX9-NEXT: v_cndmask_b32_e32 v22, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v21, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v5
+; GFX9-NEXT: v_add3_u32 v21, v21, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v21, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v20
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v4
+; GFX9-NEXT: v_mul_f32_e32 v21, v33, v21
+; GFX9-NEXT: v_bfe_u32 v33, v21, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v21
; GFX9-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX9-NEXT: v_mul_f32_e32 v21, v33, v21
+; GFX9-NEXT: v_add3_u32 v33, v33, v21, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v21, v21
; GFX9-NEXT: v_mul_f32_e32 v4, v4, v20
+; GFX9-NEXT: v_cndmask_b32_e32 v21, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v20, v20, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v20, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v19
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v3
+; GFX9-NEXT: v_mul_f32_e32 v20, v33, v20
+; GFX9-NEXT: v_bfe_u32 v33, v20, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v20
; GFX9-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_mul_f32_e32 v20, v33, v20
+; GFX9-NEXT: v_add3_u32 v33, v33, v20, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v20, v20
; GFX9-NEXT: v_mul_f32_e32 v3, v3, v19
+; GFX9-NEXT: v_cndmask_b32_e32 v20, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v19, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v19, v19, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v19, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v18
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v2
+; GFX9-NEXT: v_mul_f32_e32 v19, v33, v19
+; GFX9-NEXT: v_bfe_u32 v33, v19, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v19
; GFX9-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_mul_f32_e32 v19, v33, v19
+; GFX9-NEXT: v_add3_u32 v33, v33, v19, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v19, v19
; GFX9-NEXT: v_mul_f32_e32 v2, v2, v18
+; GFX9-NEXT: v_cndmask_b32_e32 v19, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v18, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v18, v18, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v18, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v1
+; GFX9-NEXT: v_mul_f32_e32 v18, v33, v18
+; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v18
; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_mul_f32_e32 v18, v33, v18
+; GFX9-NEXT: v_add3_u32 v33, v33, v18, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
; GFX9-NEXT: v_mul_f32_e32 v1, v1, v17
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v17, v17, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v16
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v0
+; GFX9-NEXT: v_mul_f32_e32 v17, v33, v17
+; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v17
; GFX9-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_mul_f32_e32 v17, v33, v17
+; GFX9-NEXT: v_add3_u32 v33, v33, v17, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_mul_f32_e32 v0, v0, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v16, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v16, v16, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v16, v33, vcc
+; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v17, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v18, s4
; GFX9-NEXT: v_perm_b32 v2, v2, v19, s4
@@ -12394,7 +16563,14 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX9-NEXT: v_perm_b32 v5, v5, v22, s4
; GFX9-NEXT: v_perm_b32 v6, v6, v23, s4
; GFX9-NEXT: v_perm_b32 v7, v7, v24, s4
-; GFX9-NEXT: v_perm_b32 v15, v15, v32, s4
+; GFX9-NEXT: v_perm_b32 v8, v8, v25, s4
+; GFX9-NEXT: v_perm_b32 v9, v9, v26, s4
+; GFX9-NEXT: v_perm_b32 v10, v10, v27, s4
+; GFX9-NEXT: v_perm_b32 v11, v11, v28, s4
+; GFX9-NEXT: v_perm_b32 v12, v12, v32, s4
+; GFX9-NEXT: v_perm_b32 v13, v13, v30, s4
+; GFX9-NEXT: v_perm_b32 v14, v14, v31, s4
+; GFX9-NEXT: v_perm_b32 v15, v29, v15, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v32bf16:
@@ -12409,32 +16585,10 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v10
; GFX10-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v30
-; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v14
-; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
-; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v29
-; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v13
-; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v28
; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v12
; GFX10-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX10-NEXT: v_mul_f32_e32 v39, v48, v39
-; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v17
-; GFX10-NEXT: v_mul_f32_e32 v11, v11, v27
-; GFX10-NEXT: v_lshlrev_b32_e32 v27, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_mul_f32_e32 v49, v50, v49
-; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v16
-; GFX10-NEXT: v_mul_f32_e32 v10, v10, v26
-; GFX10-NEXT: v_lshlrev_b32_e32 v26, 16, v0
-; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v15
-; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX10-NEXT: v_lshlrev_b32_e32 v51, 16, v25
; GFX10-NEXT: v_lshlrev_b32_e32 v52, 16, v9
; GFX10-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
@@ -12453,29 +16607,28 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX10-NEXT: v_lshlrev_b32_e32 v67, 16, v21
; GFX10-NEXT: v_lshlrev_b32_e32 v68, 16, v5
-; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT: v_mul_f32_e32 v33, v34, v33
-; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v20
-; GFX10-NEXT: v_mul_f32_e32 v14, v14, v30
-; GFX10-NEXT: v_lshlrev_b32_e32 v30, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT: v_mul_f32_e32 v35, v36, v35
-; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v19
-; GFX10-NEXT: v_mul_f32_e32 v13, v13, v29
-; GFX10-NEXT: v_lshlrev_b32_e32 v29, 16, v3
-; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_mul_f32_e32 v39, v48, v39
+; GFX10-NEXT: v_mul_f32_e32 v11, v11, v27
+; GFX10-NEXT: v_mul_f32_e32 v49, v50, v49
+; GFX10-NEXT: v_mul_f32_e32 v10, v10, v26
+; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v29
+; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v13
+; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX10-NEXT: v_mul_f32_e32 v37, v38, v37
; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v18
; GFX10-NEXT: v_mul_f32_e32 v12, v12, v28
; GFX10-NEXT: v_lshlrev_b32_e32 v28, 16, v2
; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_mul_f32_e32 v0, v0, v16
-; GFX10-NEXT: v_mul_f32_e32 v1, v1, v17
-; GFX10-NEXT: v_mul_f32_e32 v51, v52, v51
+; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v17
+; GFX10-NEXT: v_lshlrev_b32_e32 v27, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v16
+; GFX10-NEXT: v_lshlrev_b32_e32 v26, 16, v0
+; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX10-NEXT: v_mul_f32_e32 v9, v9, v25
; GFX10-NEXT: v_mul_f32_e32 v25, v54, v53
; GFX10-NEXT: v_mul_f32_e32 v8, v8, v24
@@ -12484,36 +16637,220 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX10-NEXT: v_mul_f32_e32 v23, v66, v65
; GFX10-NEXT: v_mul_f32_e32 v6, v6, v22
; GFX10-NEXT: v_mul_f32_e32 v22, v68, v67
-; GFX10-NEXT: v_mul_f32_e32 v5, v5, v21
-; GFX10-NEXT: v_mul_f32_e32 v21, v30, v34
-; GFX10-NEXT: v_mul_f32_e32 v29, v29, v36
-; GFX10-NEXT: v_mul_f32_e32 v28, v28, v38
-; GFX10-NEXT: v_mul_f32_e32 v27, v27, v48
-; GFX10-NEXT: v_mul_f32_e32 v26, v26, v50
+; GFX10-NEXT: v_bfe_u32 v53, v39, 16, 1
+; GFX10-NEXT: v_bfe_u32 v55, v11, 16, 1
+; GFX10-NEXT: v_bfe_u32 v65, v49, 16, 1
+; GFX10-NEXT: v_bfe_u32 v67, v10, 16, 1
+; GFX10-NEXT: s_brev_b32 s23, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v30
+; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v14
+; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; GFX10-NEXT: v_mul_f32_e32 v35, v36, v35
+; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v19
+; GFX10-NEXT: v_mul_f32_e32 v13, v13, v29
+; GFX10-NEXT: v_lshlrev_b32_e32 v29, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX10-NEXT: v_mul_f32_e32 v2, v2, v18
+; GFX10-NEXT: v_mul_f32_e32 v18, v27, v48
+; GFX10-NEXT: v_mul_f32_e32 v1, v1, v17
+; GFX10-NEXT: v_mul_f32_e32 v17, v26, v50
+; GFX10-NEXT: v_mul_f32_e32 v0, v0, v16
+; GFX10-NEXT: v_and_or_b32 v54, v39, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v64, v11, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v66, v49, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v68, v10, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s9, v39, v39
+; GFX10-NEXT: v_add3_u32 v39, v53, v39, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s10, v11, v11
+; GFX10-NEXT: v_add3_u32 v11, v55, v11, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s11, v49, v49
+; GFX10-NEXT: v_add3_u32 v49, v65, v49, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s12, v10, v10
+; GFX10-NEXT: v_add3_u32 v10, v67, v10, 0x7fff
+; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX10-NEXT: v_mul_f32_e32 v33, v34, v33
+; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v20
+; GFX10-NEXT: v_mul_f32_e32 v14, v14, v30
+; GFX10-NEXT: v_lshlrev_b32_e32 v30, 16, v4
+; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX10-NEXT: v_mul_f32_e32 v3, v3, v19
+; GFX10-NEXT: v_mul_f32_e32 v19, v28, v38
+; GFX10-NEXT: v_bfe_u32 v38, v37, 16, 1
+; GFX10-NEXT: v_bfe_u32 v50, v12, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v39, v39, v54, s9
+; GFX10-NEXT: v_bfe_u32 v54, v18, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v11, v11, v64, s10
+; GFX10-NEXT: v_bfe_u32 v64, v1, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v49, v49, v66, s11
+; GFX10-NEXT: v_bfe_u32 v66, v17, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v68, s12
+; GFX10-NEXT: v_bfe_u32 v68, v0, 16, 1
+; GFX10-NEXT: v_mul_f32_e32 v51, v52, v51
+; GFX10-NEXT: v_mul_f32_e32 v5, v5, v21
+; GFX10-NEXT: v_mul_f32_e32 v21, v30, v34
; GFX10-NEXT: v_mul_f32_e32 v4, v4, v20
-; GFX10-NEXT: v_perm_b32 v1, v1, v27, 0x7060302
-; GFX10-NEXT: v_perm_b32 v0, v0, v26, 0x7060302
-; GFX10-NEXT: v_perm_b32 v2, v2, v28, 0x7060302
-; GFX10-NEXT: v_perm_b32 v3, v3, v29, 0x7060302
+; GFX10-NEXT: v_mul_f32_e32 v20, v29, v36
+; GFX10-NEXT: v_bfe_u32 v16, v33, 16, 1
+; GFX10-NEXT: v_bfe_u32 v27, v14, 16, 1
+; GFX10-NEXT: v_bfe_u32 v29, v35, 16, 1
+; GFX10-NEXT: v_bfe_u32 v34, v13, 16, 1
+; GFX10-NEXT: v_and_or_b32 v48, v37, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v52, v12, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s7, v37, v37
+; GFX10-NEXT: v_add3_u32 v37, v38, v37, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s8, v12, v12
+; GFX10-NEXT: v_add3_u32 v12, v50, v12, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s10, v18, v18
+; GFX10-NEXT: v_add3_u32 v54, v54, v18, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v18, v18, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s11, v1, v1
+; GFX10-NEXT: v_add3_u32 v64, v64, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v1, v1, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s12, v17, v17
+; GFX10-NEXT: v_add3_u32 v66, v66, v17, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v17, v17, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s22, v0, v0
+; GFX10-NEXT: v_add3_u32 v68, v68, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v0, v0, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v26, v33, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v28, v14, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v30, v35, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v36, v13, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX10-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v33, v51, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v14, v14
+; GFX10-NEXT: v_add3_u32 v14, v27, v14, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s5, v35, v35
+; GFX10-NEXT: v_add3_u32 v29, v29, v35, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s6, v13, v13
+; GFX10-NEXT: v_add3_u32 v13, v34, v13, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v65, v24, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v37, v37, v48, s7
+; GFX10-NEXT: v_bfe_u32 v48, v19, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v12, v12, v52, s8
+; GFX10-NEXT: v_bfe_u32 v52, v2, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v18, v54, v18, s10
+; GFX10-NEXT: v_cndmask_b32_e64 v17, v66, v17, s12
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v68, v0, s22
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v64, v1, s11
+; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v15
+; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX10-NEXT: v_and_or_b32 v27, v51, s23, 0x400000
+; GFX10-NEXT: v_bfe_u32 v35, v9, 16, 1
+; GFX10-NEXT: v_bfe_u32 v38, v25, 16, 1
+; GFX10-NEXT: v_and_or_b32 v67, v24, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s13, v51, v51
+; GFX10-NEXT: v_add3_u32 v33, v33, v51, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v51, v7, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s17, v24, v24
+; GFX10-NEXT: v_add3_u32 v24, v65, v24, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v65, v6, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v16, v16, v26, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v26, v21, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v14, v14, v28, s4
+; GFX10-NEXT: v_bfe_u32 v28, v4, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v29, v29, v30, s5
+; GFX10-NEXT: v_bfe_u32 v30, v20, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v13, v13, v36, s6
+; GFX10-NEXT: v_bfe_u32 v36, v3, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s8, v19, v19
+; GFX10-NEXT: v_add3_u32 v48, v48, v19, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v19, v19, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s9, v2, v2
+; GFX10-NEXT: v_add3_u32 v52, v52, v2, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v2, v2, s23, 0x400000
+; GFX10-NEXT: v_perm_b32 v0, v0, v17, 0x7060302
+; GFX10-NEXT: v_perm_b32 v1, v1, v18, 0x7060302
+; GFX10-NEXT: v_and_or_b32 v34, v9, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v50, v25, s23, 0x400000
+; GFX10-NEXT: v_bfe_u32 v53, v8, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s14, v9, v9
+; GFX10-NEXT: v_add3_u32 v9, v35, v9, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v35, v7, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s15, v25, v25
+; GFX10-NEXT: v_add3_u32 v25, v38, v25, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v38, v23, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s18, v7, v7
+; GFX10-NEXT: v_add3_u32 v7, v51, v7, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v51, v6, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s20, v6, v6
+; GFX10-NEXT: v_add3_u32 v6, v65, v6, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v65, v5, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v21, v21
+; GFX10-NEXT: v_add3_u32 v26, v26, v21, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v21, v21, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s5, v4, v4
+; GFX10-NEXT: v_add3_u32 v28, v28, v4, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v4, v4, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s6, v20, v20
+; GFX10-NEXT: v_add3_u32 v30, v30, v20, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v20, v20, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s7, v3, v3
+; GFX10-NEXT: v_add3_u32 v36, v36, v3, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v3, v3, s23, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e64 v19, v48, v19, s8
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v52, v2, s9
+; GFX10-NEXT: v_and_or_b32 v55, v8, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s16, v8, v8
+; GFX10-NEXT: v_add3_u32 v8, v53, v8, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v53, v23, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s19, v23, v23
+; GFX10-NEXT: v_add3_u32 v23, v38, v23, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v38, v22, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v65, v65, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v5, v5, s23, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e64 v21, v26, v21, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v28, v4, s5
+; GFX10-NEXT: v_cndmask_b32_e64 v20, v30, v20, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v36, v3, s7
+; GFX10-NEXT: v_perm_b32 v2, v2, v19, 0x7060302
+; GFX10-NEXT: v_cmp_u_f32_e64 s21, v22, v22
+; GFX10-NEXT: v_add3_u32 v38, v38, v22, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v22, v22, s23, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v65, v5, vcc_lo
+; GFX10-NEXT: v_perm_b32 v3, v3, v20, 0x7060302
; GFX10-NEXT: v_perm_b32 v4, v4, v21, 0x7060302
-; GFX10-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
-; GFX10-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
-; GFX10-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e64 v27, v33, v27, s13
+; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v34, s14
+; GFX10-NEXT: v_cndmask_b32_e64 v25, v25, v50, s15
+; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, v55, s16
+; GFX10-NEXT: v_cndmask_b32_e64 v24, v24, v67, s17
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v35, s18
+; GFX10-NEXT: v_cndmask_b32_e64 v23, v23, v53, s19
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v51, s20
+; GFX10-NEXT: v_cndmask_b32_e64 v22, v38, v22, s21
; GFX10-NEXT: v_perm_b32 v8, v8, v25, 0x7060302
-; GFX10-NEXT: v_perm_b32 v9, v9, v51, 0x7060302
+; GFX10-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
+; GFX10-NEXT: v_perm_b32 v9, v9, v27, 0x7060302
+; GFX10-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
+; GFX10-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
; GFX10-NEXT: v_perm_b32 v10, v10, v49, 0x7060302
; GFX10-NEXT: v_perm_b32 v11, v11, v39, 0x7060302
; GFX10-NEXT: v_perm_b32 v12, v12, v37, 0x7060302
-; GFX10-NEXT: v_perm_b32 v13, v13, v35, 0x7060302
-; GFX10-NEXT: v_perm_b32 v14, v14, v33, 0x7060302
+; GFX10-NEXT: v_perm_b32 v13, v13, v29, 0x7060302
+; GFX10-NEXT: v_perm_b32 v14, v14, v16, 0x7060302
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v31
-; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v31
-; GFX10-NEXT: v_mul_f32_e32 v16, v32, v16
-; GFX10-NEXT: v_mul_f32_e32 v15, v15, v17
-; GFX10-NEXT: v_perm_b32 v15, v15, v16, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v31
+; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v31
+; GFX10-NEXT: v_mul_f32_e32 v17, v32, v17
+; GFX10-NEXT: v_mul_f32_e32 v15, v15, v18
+; GFX10-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX10-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX10-NEXT: v_and_or_b32 v20, v17, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v21, v15, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v15, v15
+; GFX10-NEXT: v_add3_u32 v17, v18, v17, 0x7fff
+; GFX10-NEXT: v_add3_u32 v15, v19, v15, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v17, v17, v20, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v15, v15, v21, s4
+; GFX10-NEXT: v_perm_b32 v15, v15, v17, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fmul_v32bf16:
@@ -12524,102 +16861,269 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX11-NEXT: v_lshlrev_b32_e32 v84, 16, v1
; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v24
+; GFX11-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_dual_mul_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v64, 16, v7
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v18
; GFX11-NEXT: v_lshlrev_b32_e32 v85, 16, v16
; GFX11-NEXT: v_lshlrev_b32_e32 v86, 16, v0
+; GFX11-NEXT: v_bfe_u32 v135, v1, 16, 1
; GFX11-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v23
+; GFX11-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; GFX11-NEXT: v_and_or_b32 v144, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v54, 16, v8
-; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v7
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: v_dual_mul_f32 v17, v86, v85 :: v_dual_and_b32 v8, 0xffff0000, v8
+; GFX11-NEXT: v_dual_mul_f32 v7, v7, v23 :: v_dual_lshlrev_b32 v36, 16, v13
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_mul_f32 v8, v8, v24 :: v_dual_lshlrev_b32 v39, 16, v27
+; GFX11-NEXT: v_dual_mul_f32 v0, v0, v16 :: v_dual_lshlrev_b32 v49, 16, v26
+; GFX11-NEXT: v_mul_f32_e32 v24, v64, v55
+; GFX11-NEXT: v_bfe_u32 v87, v7, 16, 1
; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v22
; GFX11-NEXT: v_lshlrev_b32_e32 v66, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v11
-; GFX11-NEXT: v_dual_mul_f32 v0, v0, v16 :: v_dual_and_b32 v11, 0xffff0000, v11
; GFX11-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX11-NEXT: v_bfe_u32 v85, v24, 16, 1
; GFX11-NEXT: v_lshlrev_b32_e32 v67, 16, v21
; GFX11-NEXT: v_lshlrev_b32_e32 v68, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v51, 16, v25
; GFX11-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v20
; GFX11-NEXT: v_lshlrev_b32_e32 v70, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v23
-; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v19
-; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v3
-; GFX11-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v9
-; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v18
-; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v2
+; GFX11-NEXT: v_and_or_b32 v86, v24, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v96, v7, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v85, v85, v24, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v20
+; GFX11-NEXT: v_add3_u32 v87, v87, v7, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX11-NEXT: v_dual_mul_f32 v23, v66, v65 :: v_dual_lshlrev_b32 v48, 16, v11
+; GFX11-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; GFX11-NEXT: v_dual_mul_f32 v5, v5, v21 :: v_dual_lshlrev_b32 v50, 16, v10
+; GFX11-NEXT: v_dual_mul_f32 v21, v70, v69 :: v_dual_and_b32 v26, 0xffff0000, v26
; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v24
-; GFX11-NEXT: v_dual_mul_f32 v1, v1, v17 :: v_dual_and_b32 v24, 0xffff0000, v24
-; GFX11-NEXT: v_dual_mul_f32 v5, v5, v21 :: v_dual_lshlrev_b32 v50, 16, v10
-; GFX11-NEXT: v_dual_mul_f32 v21, v70, v69 :: v_dual_and_b32 v10, 0xffff0000, v10
-; GFX11-NEXT: v_dual_mul_f32 v2, v2, v18 :: v_dual_mul_f32 v3, v3, v19
-; GFX11-NEXT: v_dual_mul_f32 v4, v4, v20 :: v_dual_lshlrev_b32 v49, 16, v26
-; GFX11-NEXT: v_dual_mul_f32 v9, v9, v25 :: v_dual_and_b32 v26, 0xffff0000, v26
; GFX11-NEXT: v_mul_f32_e32 v6, v6, v22
-; GFX11-NEXT: v_dual_mul_f32 v22, v68, v67 :: v_dual_lshlrev_b32 v37, 16, v28
-; GFX11-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_mul_f32_e32 v10, v10, v26
-; GFX11-NEXT: v_mul_f32_e32 v26, v52, v51
-; GFX11-NEXT: v_perm_b32 v4, v4, v21, 0x7060302
-; GFX11-NEXT: v_mul_f32_e32 v25, v54, v53
-; GFX11-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
-; GFX11-NEXT: v_perm_b32 v9, v9, v26, 0x7060302
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v31
-; GFX11-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v31
-; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-NEXT: v_lshlrev_b32_e32 v36, 16, v13
-; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX11-NEXT: v_lshlrev_b32_e32 v39, 16, v27
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_dual_mul_f32 v8, v8, v24 :: v_dual_and_b32 v27, 0xffff0000, v27
-; GFX11-NEXT: v_mul_f32_e32 v24, v64, v55
+; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v9
+; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; GFX11-NEXT: v_bfe_u32 v97, v23, 16, 1
+; GFX11-NEXT: v_mul_f32_e32 v2, v2, v18
+; GFX11-NEXT: v_mul_f32_e32 v18, v84, v83
+; GFX11-NEXT: v_bfe_u32 v83, v8, 16, 1
+; GFX11-NEXT: v_bfe_u32 v99, v6, 16, 1
+; GFX11-NEXT: v_bfe_u32 v103, v5, 16, 1
+; GFX11-NEXT: v_bfe_u32 v113, v21, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v19
+; GFX11-NEXT: v_and_or_b32 v84, v8, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v98, v23, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v100, v6, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v112, v5, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v114, v21, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v83, v83, v8, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; GFX11-NEXT: v_add3_u32 v97, v97, v23, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; GFX11-NEXT: v_add3_u32 v99, v99, v6, 0x7fff
+; GFX11-NEXT: v_add3_u32 v103, v103, v5, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-NEXT: v_add3_u32 v113, v113, v21, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v12
-; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX11-NEXT: v_dual_mul_f32 v3, v3, v19 :: v_dual_and_b32 v10, 0xffff0000, v10
+; GFX11-NEXT: v_dual_mul_f32 v22, v68, v67 :: v_dual_lshlrev_b32 v51, 16, v25
+; GFX11-NEXT: v_lshlrev_b32_e32 v37, 16, v28
+; GFX11-NEXT: v_dual_mul_f32 v4, v4, v20 :: v_dual_and_b32 v25, 0xffff0000, v25
+; GFX11-NEXT: v_mul_f32_e32 v20, v80, v71
+; GFX11-NEXT: v_dual_mul_f32 v19, v82, v81 :: v_dual_and_b32 v28, 0xffff0000, v28
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_dual_mul_f32 v9, v9, v25 :: v_dual_and_b32 v12, 0xffff0000, v12
+; GFX11-NEXT: v_mul_f32_e32 v25, v54, v53
; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v29
-; GFX11-NEXT: v_mul_f32_e32 v7, v7, v23
-; GFX11-NEXT: v_mul_f32_e32 v23, v66, v65
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mul_f32 v12, v12, v28 :: v_dual_and_b32 v29, 0xffff0000, v29
-; GFX11-NEXT: v_dual_mul_f32 v28, v48, v39 :: v_dual_lshlrev_b32 v33, 16, v30
+; GFX11-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX11-NEXT: v_dual_mul_f32 v10, v10, v26 :: v_dual_and_b32 v13, 0xffff0000, v13
+; GFX11-NEXT: v_dual_mul_f32 v12, v12, v28 :: v_dual_lshlrev_b32 v33, 16, v30
+; GFX11-NEXT: v_mul_f32_e32 v28, v48, v39
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_dual_mul_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v34, 16, v14
-; GFX11-NEXT: v_lshlrev_b32_e32 v32, 16, v15
; GFX11-NEXT: v_dual_mul_f32 v11, v11, v27 :: v_dual_and_b32 v14, 0xffff0000, v14
-; GFX11-NEXT: v_dual_mul_f32 v27, v50, v49 :: v_dual_and_b32 v30, 0xffff0000, v30
-; GFX11-NEXT: v_mul_f32_e32 v29, v38, v37
+; GFX11-NEXT: v_dual_mul_f32 v27, v50, v49 :: v_dual_mul_f32 v26, v52, v51
+; GFX11-NEXT: v_dual_mul_f32 v29, v38, v37 :: v_dual_and_b32 v30, 0xffff0000, v30
+; GFX11-NEXT: v_lshlrev_b32_e32 v32, 16, v15
; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX11-NEXT: v_mul_f32_e32 v37, v86, v85
-; GFX11-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v39, v13, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_mul_f32_e32 v14, v14, v30
; GFX11-NEXT: v_dual_mul_f32 v30, v36, v35 :: v_dual_mul_f32 v33, v34, v33
-; GFX11-NEXT: v_dual_mul_f32 v34, v80, v71 :: v_dual_mul_f32 v35, v82, v81
-; GFX11-NEXT: v_mul_f32_e32 v36, v84, v83
-; GFX11-NEXT: v_dual_mul_f32 v16, v32, v16 :: v_dual_mul_f32 v15, v15, v17
-; GFX11-NEXT: v_perm_b32 v0, v0, v37, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v2, v2, v35, 0x7060302
-; GFX11-NEXT: v_perm_b32 v1, v1, v36, 0x7060302
-; GFX11-NEXT: v_perm_b32 v3, v3, v34, 0x7060302
-; GFX11-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
-; GFX11-NEXT: v_perm_b32 v8, v8, v25, 0x7060302
+; GFX11-NEXT: v_and_or_b32 v48, v13, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v49, v29, 16, 1
+; GFX11-NEXT: v_bfe_u32 v35, v14, 16, 1
+; GFX11-NEXT: v_and_or_b32 v36, v14, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v16, v33, 16, 1
+; GFX11-NEXT: v_and_or_b32 v34, v33, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX11-NEXT: v_bfe_u32 v37, v30, 16, 1
+; GFX11-NEXT: v_add3_u32 v35, v35, v14, 0x7fff
+; GFX11-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v38, v30, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v39, v39, v13, 0x7fff
+; GFX11-NEXT: v_add3_u32 v37, v37, v30, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v50, v29, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v16, v16, v34, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-NEXT: v_bfe_u32 v51, v12, 16, 1
+; GFX11-NEXT: v_add3_u32 v49, v49, v29, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v52, v12, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v53, v28, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v14, v35, v36, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
+; GFX11-NEXT: v_add3_u32 v51, v51, v12, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v54, v28, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v55, v11, 16, 1
+; GFX11-NEXT: v_add3_u32 v53, v53, v28, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v30, v37, v38, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-NEXT: v_and_or_b32 v64, v11, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v65, v27, 16, 1
+; GFX11-NEXT: v_add3_u32 v55, v55, v11, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v66, v27, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v13, v39, v48, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-NEXT: v_bfe_u32 v67, v10, 16, 1
+; GFX11-NEXT: v_add3_u32 v65, v65, v27, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v68, v10, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v69, v26, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v29, v49, v50, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-NEXT: v_add3_u32 v67, v67, v10, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v70, v26, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v71, v9, 16, 1
+; GFX11-NEXT: v_add3_u32 v69, v69, v26, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v12, v51, v52, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX11-NEXT: v_and_or_b32 v80, v9, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v81, v25, 16, 1
+; GFX11-NEXT: v_add3_u32 v71, v71, v9, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v82, v25, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v28, v53, v54, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-NEXT: v_add3_u32 v81, v81, v25, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v101, v22, 16, 1
+; GFX11-NEXT: v_and_or_b32 v102, v22, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v115, v4, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v11, v55, v64, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX11-NEXT: v_add3_u32 v101, v101, v22, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v116, v4, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v117, v20, 16, 1
+; GFX11-NEXT: v_add3_u32 v115, v115, v4, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v27, v65, v66, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-NEXT: v_and_or_b32 v118, v20, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v129, v19, 16, 1
+; GFX11-NEXT: v_add3_u32 v117, v117, v20, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v130, v19, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v10, v67, v68, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-NEXT: v_bfe_u32 v133, v18, 16, 1
+; GFX11-NEXT: v_add3_u32 v129, v129, v19, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v134, v18, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v145, v17, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v26, v69, v70, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-NEXT: v_add3_u32 v133, v133, v18, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v146, v17, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v147, v0, 16, 1
+; GFX11-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v9, v71, v80, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-NEXT: v_bfe_u32 v131, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v33, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v147, v147, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v132, v2, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v25, v81, v82, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-NEXT: v_add3_u32 v131, v131, v2, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v119, v3, 16, 1
+; GFX11-NEXT: v_and_or_b32 v128, v3, s0, 0x400000
+; GFX11-NEXT: v_perm_b32 v9, v9, v26, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v8, v83, v84, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-NEXT: v_add3_u32 v119, v119, v3, 0x7fff
; GFX11-NEXT: v_perm_b32 v10, v10, v27, 0x7060302
; GFX11-NEXT: v_perm_b32 v11, v11, v28, 0x7060302
+; GFX11-NEXT: v_perm_b32 v8, v8, v25, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v24, v85, v86, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11-NEXT: v_perm_b32 v12, v12, v29, 0x7060302
; GFX11-NEXT: v_perm_b32 v13, v13, v30, 0x7060302
-; GFX11-NEXT: v_perm_b32 v14, v14, v33, 0x7060302
-; GFX11-NEXT: v_perm_b32 v15, v15, v16, 0x7060302
+; GFX11-NEXT: v_perm_b32 v14, v14, v16, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v87, v96, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v23, v97, v98, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: v_cndmask_b32_e32 v6, v99, v100, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v22, v101, v102, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v103, v112, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v21, v113, v114, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v115, v116, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-NEXT: v_perm_b32 v4, v4, v21, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v20, v117, v118, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-NEXT: v_cndmask_b32_e32 v19, v129, v130, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-NEXT: v_cndmask_b32_e32 v18, v133, v134, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v135, v144, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, v1, v18, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v17, v145, v146, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v147, v33, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_perm_b32 v0, v0, v17, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v131, v132, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v119, v128, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_perm_b32 v3, v3, v20, 0x7060302
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v31
+; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v31
+; GFX11-NEXT: v_perm_b32 v2, v2, v19, 0x7060302
+; GFX11-NEXT: v_mul_f32_e32 v17, v32, v17
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mul_f32_e32 v15, v15, v18
+; GFX11-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11-NEXT: v_and_or_b32 v20, v17, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: v_and_or_b32 v21, v15, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v18, v18, v17, 0x7fff
+; GFX11-NEXT: v_add3_u32 v19, v19, v15, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v17, v18, v20, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-NEXT: v_cndmask_b32_e32 v15, v19, v21, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v15, v15, v17, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fmul <32 x bfloat> %a, %b
ret <32 x bfloat> %op
@@ -12629,6 +17133,8 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fdiv_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, v0
@@ -12648,6 +17154,8 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fdiv_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, v0
@@ -12680,6 +17188,13 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) {
; GFX8-NEXT: v_fma_f32 v2, -v2, v5, v3
; GFX8-NEXT: v_div_fmas_f32 v2, v2, v4, v5
; GFX8-NEXT: v_div_fixup_f32 v0, v2, v1, v0
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -12690,6 +17205,7 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) {
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, v0
; GFX9-NEXT: v_div_scale_f32 v3, vcc, v0, v1, v0
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_rcp_f32_e32 v4, v2
; GFX9-NEXT: v_fma_f32 v5, -v2, v4, 1.0
; GFX9-NEXT: v_fma_f32 v4, v5, v4, v4
@@ -12699,6 +17215,12 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) {
; GFX9-NEXT: v_fma_f32 v2, -v2, v5, v3
; GFX9-NEXT: v_div_fmas_f32 v2, v2, v4, v5
; GFX9-NEXT: v_div_fixup_f32 v0, v2, v1, v0
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -12708,16 +17230,22 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_div_scale_f32 v2, s4, v1, v1, v0
+; GFX10-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v1, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_rcp_f32_e32 v3, v2
; GFX10-NEXT: v_fma_f32 v4, -v2, v3, 1.0
; GFX10-NEXT: v_fmac_f32_e32 v3, v4, v3
-; GFX10-NEXT: v_div_scale_f32 v4, vcc_lo, v0, v1, v0
-; GFX10-NEXT: v_mul_f32_e32 v5, v4, v3
-; GFX10-NEXT: v_fma_f32 v6, -v2, v5, v4
-; GFX10-NEXT: v_fmac_f32_e32 v5, v6, v3
-; GFX10-NEXT: v_fma_f32 v2, -v2, v5, v4
-; GFX10-NEXT: v_div_fmas_f32 v2, v2, v3, v5
+; GFX10-NEXT: v_mul_f32_e32 v4, v5, v3
+; GFX10-NEXT: v_fma_f32 v6, -v2, v4, v5
+; GFX10-NEXT: v_fmac_f32_e32 v4, v6, v3
+; GFX10-NEXT: v_fma_f32 v2, -v2, v4, v5
+; GFX10-NEXT: v_div_fmas_f32 v2, v2, v3, v4
; GFX10-NEXT: v_div_fixup_f32 v0, v2, v1, v0
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -12726,6 +17254,7 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_div_scale_f32 v2, null, v1, v1, v0
; GFX11-NEXT: v_rcp_f32_e32 v3, v2
@@ -12733,16 +17262,23 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) {
; GFX11-NEXT: v_fma_f32 v4, -v2, v3, 1.0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fmac_f32_e32 v3, v4, v3
-; GFX11-NEXT: v_div_scale_f32 v4, vcc_lo, v0, v1, v0
-; GFX11-NEXT: v_mul_f32_e32 v5, v4, v3
+; GFX11-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v1, v0
+; GFX11-NEXT: v_mul_f32_e32 v4, v5, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_fma_f32 v6, -v2, v5, v4
-; GFX11-NEXT: v_fmac_f32_e32 v5, v6, v3
+; GFX11-NEXT: v_fma_f32 v6, -v2, v4, v5
+; GFX11-NEXT: v_fmac_f32_e32 v4, v6, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_fma_f32 v2, -v2, v5, v4
-; GFX11-NEXT: v_div_fmas_f32 v2, v2, v3, v5
+; GFX11-NEXT: v_fma_f32 v2, -v2, v4, v5
+; GFX11-NEXT: v_div_fmas_f32 v2, v2, v3, v4
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_div_fixup_f32 v0, v2, v1, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = fdiv bfloat %a, %b
@@ -12755,16 +17291,18 @@ define bfloat @v_fabs_bf16(bfloat %a) {
; GCN-LABEL: v_fabs_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GCN-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; GCN-NEXT: v_mul_f32_e64 v0, 1.0, |v0|
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_fabs_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; GFX7-NEXT: v_mul_f32_e64 v0, 1.0, |v0|
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -12798,12 +17336,16 @@ define bfloat @v_fabs_bf16(bfloat %a) {
define amdgpu_ps i32 @s_fabs_bf16(bfloat inreg %a) {
; GCN-LABEL: s_fabs_bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_bfe_u32 s0, s0, 0xf0010
+; GCN-NEXT: v_mul_f32_e64 v0, 1.0, s0
+; GCN-NEXT: v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
; GCN-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: s_fabs_bf16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_bfe_u32 s0, s0, 0xf0010
+; GFX7-NEXT: v_mul_f32_e64 v0, 1.0, s0
+; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 15
+; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fabs_bf16:
@@ -12889,14 +17431,18 @@ declare i32 @llvm.amdgcn.readfirstlane(i32)
define amdgpu_ps i32 @s_fneg_bf16(bfloat inreg %a) {
; GCN-LABEL: s_fneg_bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_lshr_b32 s0, s0, 16
-; GCN-NEXT: s_xor_b32 s0, s0, 0x8000
+; GCN-NEXT: v_mul_f32_e64 v0, 1.0, s0
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
; GCN-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: s_fneg_bf16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_lshr_b32 s0, s0, 16
-; GFX7-NEXT: s_xor_b32 s0, s0, 0x8000
+; GFX7-NEXT: v_mul_f32_e64 v0, 1.0, s0
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_xor_b32_e32 v0, 0x8000, v0
+; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fneg_bf16:
@@ -12940,20 +17486,22 @@ define bfloat @v_fneg_fabs_bf16(bfloat %a) {
; GCN-LABEL: v_fneg_fabs_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GCN-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; GCN-NEXT: v_mul_f32_e64 v0, 1.0, |v0|
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-NEXT: v_mul_f32_e32 v0, -1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_fneg_fabs_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
+; GFX7-NEXT: v_mul_f32_e64 v0, 1.0, |v0|
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
+; GFX7-NEXT: v_mul_f32_e32 v0, -1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -12989,14 +17537,18 @@ define bfloat @v_fneg_fabs_bf16(bfloat %a) {
define amdgpu_ps i32 @s_fneg_fabs_bf16(bfloat inreg %a) {
; GCN-LABEL: s_fneg_fabs_bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_lshr_b32 s0, s0, 16
-; GCN-NEXT: s_bitset1_b32 s0, 15
+; GCN-NEXT: v_mul_f32_e64 v0, 1.0, s0
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_or_b32_e32 v0, 0x8000, v0
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
; GCN-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: s_fneg_fabs_bf16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_lshr_b32 s0, s0, 16
-; GFX7-NEXT: s_bitset1_b32 s0, 15
+; GFX7-NEXT: v_mul_f32_e64 v0, 1.0, s0
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_or_b32_e32 v0, 0x8000, v0
+; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fneg_fabs_bf16:
@@ -13049,6 +17601,8 @@ define bfloat @v_minnum_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_minnum_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
@@ -13060,6 +17614,8 @@ define bfloat @v_minnum_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_minnum_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
@@ -13074,6 +17630,13 @@ define bfloat @v_minnum_bf16(bfloat %a, bfloat %b) {
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -13083,6 +17646,13 @@ define bfloat @v_minnum_bf16(bfloat %a, bfloat %b) {
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -13091,7 +17661,13 @@ define bfloat @v_minnum_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -13100,8 +17676,16 @@ define bfloat @v_minnum_bf16(bfloat %a, bfloat %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.minnum.bf16(bfloat %a, bfloat %b)
@@ -13112,6 +17696,10 @@ define <2 x bfloat> @v_minnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_minnum_v2bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -13129,6 +17717,10 @@ define <2 x bfloat> @v_minnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX7-LABEL: v_minnum_v2bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -13148,10 +17740,24 @@ define <2 x bfloat> @v_minnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX8-NEXT: v_min_f32_e32 v2, v3, v2
+; GFX8-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v2
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v2
; GFX8-NEXT: v_min_f32_e32 v0, v0, v1
-; GFX8-NEXT: v_min_f32_e32 v2, v3, v2
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v2, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -13161,10 +17767,23 @@ define <2 x bfloat> @v_minnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX9-NEXT: v_min_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_min_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_add3_u32 v3, v3, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v2, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -13176,9 +17795,20 @@ define <2 x bfloat> @v_minnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_min_f32_e32 v2, v3, v2
; GFX10-NEXT: v_min_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_and_or_b32 v5, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_minnum_v2bf16:
@@ -13188,11 +17818,24 @@ define <2 x bfloat> @v_minnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_min_f32_e32 v0, v0, v1
; GFX11-NEXT: v_min_f32_e32 v2, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v2, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_and_or_b32 v5, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
ret <2 x bfloat> %op
@@ -13202,6 +17845,12 @@ define <3 x bfloat> @v_minnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GCN-LABEL: v_minnum_v3bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
@@ -13225,6 +17874,12 @@ define <3 x bfloat> @v_minnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX7-LABEL: v_minnum_v3bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
@@ -13251,12 +17906,34 @@ define <3 x bfloat> @v_minnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v3
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX8-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
@@ -13268,12 +17945,31 @@ define <3 x bfloat> @v_minnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_add3_u32 v4, v4, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16
@@ -13282,16 +17978,32 @@ define <3 x bfloat> @v_minnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX10-LABEL: v_minnum_v3bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_min_f32_e32 v4, v5, v4
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_min_f32_e32 v0, v0, v2
; GFX10-NEXT: v_min_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
+; GFX10-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v4, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v6, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = call <3 x bfloat> @llvm.minnum.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b)
@@ -13302,6 +18014,14 @@ define <4 x bfloat> @v_minnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GCN-LABEL: v_minnum_v4bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
@@ -13331,6 +18051,14 @@ define <4 x bfloat> @v_minnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX7-LABEL: v_minnum_v4bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
@@ -13362,17 +18090,46 @@ define <4 x bfloat> @v_minnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX8-NEXT: v_min_f32_e32 v4, v5, v4
+; GFX8-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v4
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_min_f32_e32 v4, v5, v4
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX8-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX8-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v3
; GFX8-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_min_f32_e32 v3, v5, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16
@@ -13383,16 +18140,41 @@ define <4 x bfloat> @v_minnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX9-NEXT: v_min_f32_e32 v4, v5, v4
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_min_f32_e32 v4, v5, v4
+; GFX9-NEXT: v_add3_u32 v5, v5, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; GFX9-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX9-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4
@@ -13404,17 +18186,38 @@ define <4 x bfloat> @v_minnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX10-NEXT: v_min_f32_e32 v4, v5, v4
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_min_f32_e32 v4, v5, v4
+; GFX10-NEXT: v_min_f32_e32 v1, v1, v3
; GFX10-NEXT: v_min_f32_e32 v5, v7, v6
+; GFX10-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX10-NEXT: v_min_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
-; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX10-NEXT: v_and_or_b32 v6, v4, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v9, v1, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v6, v5, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc_lo
+; GFX10-NEXT: v_perm_b32 v1, v1, v3, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_minnum_v4bf16:
@@ -13422,19 +18225,45 @@ define <4 x bfloat> @v_minnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_min_f32 v0, v0, v2 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_dual_min_f32 v4, v5, v4 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_min_f32 v0, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-NEXT: v_min_f32_e32 v4, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v3, v4, 16, 1
; GFX11-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX11-NEXT: v_and_or_b32 v6, v4, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_and_or_b32 v9, v1, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v6, v5, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v8, v0, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc_lo
+; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
+; GFX11-NEXT: v_perm_b32 v1, v1, v3, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <4 x bfloat> @llvm.minnum.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b)
ret <4 x bfloat> %op
@@ -13444,6 +18273,22 @@ define <8 x bfloat> @v_minnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GCN-LABEL: v_minnum_v8bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
@@ -13497,6 +18342,22 @@ define <8 x bfloat> @v_minnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX7-LABEL: v_minnum_v8bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
@@ -13552,31 +18413,88 @@ define <8 x bfloat> @v_minnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX8-NEXT: v_min_f32_e32 v8, v9, v8
+; GFX8-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v8
; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_min_f32_e32 v8, v9, v8
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v8
; GFX8-NEXT: v_min_f32_e32 v3, v3, v7
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, s4, v7
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX8-NEXT: v_min_f32_e32 v7, v9, v7
+; GFX8-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v7
; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: v_min_f32_e32 v7, v9, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v7
; GFX8-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, s4, v6
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v6, v9, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v5
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX8-NEXT: v_min_f32_e32 v6, v9, v6
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_min_f32_e32 v6, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v6
; GFX8-NEXT: v_min_f32_e32 v1, v1, v5
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX8-NEXT: v_min_f32_e32 v5, v9, v5
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v5
; GFX8-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v9, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_min_f32_e32 v5, v9, v5
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v5, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v6, 16
@@ -13589,28 +18507,77 @@ define <8 x bfloat> @v_minnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX9-NEXT: v_min_f32_e32 v8, v9, v8
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v8
; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_min_f32_e32 v8, v9, v8
+; GFX9-NEXT: v_add3_u32 v9, v9, v8, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX9-NEXT: v_min_f32_e32 v3, v3, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v7, v7, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX9-NEXT: v_min_f32_e32 v7, v9, v7
+; GFX9-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v7
; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_min_f32_e32 v7, v9, v7
+; GFX9-NEXT: v_add3_u32 v9, v9, v7, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v9, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v5
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX9-NEXT: v_min_f32_e32 v6, v9, v6
+; GFX9-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v6
; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_min_f32_e32 v6, v9, v6
+; GFX9-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX9-NEXT: v_min_f32_e32 v1, v1, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v5, v5, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX9-NEXT: v_min_f32_e32 v5, v9, v5
+; GFX9-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v5
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_min_f32_e32 v5, v9, v5
+; GFX9-NEXT: v_add3_u32 v9, v9, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v9, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v5, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v6, s4
@@ -13623,65 +18590,157 @@ define <8 x bfloat> @v_minnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v6
-; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_min_f32_e32 v8, v9, v8
+; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v6
+; GFX10-NEXT: v_min_f32_e32 v3, v3, v7
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX10-NEXT: v_min_f32_e32 v9, v11, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v8, s4, 0x400000
+; GFX10-NEXT: v_min_f32_e32 v9, v11, v9
+; GFX10-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX10-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX10-NEXT: v_bfe_u32 v8, v9, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v5
+; GFX10-NEXT: v_and_or_b32 v12, v9, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX10-NEXT: v_add3_u32 v10, v11, v3, 0x7fff
; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX10-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX10-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v0
+; GFX10-NEXT: v_min_f32_e32 v6, v11, v6
+; GFX10-NEXT: v_add3_u32 v9, v13, v2, 0x7fff
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v11, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v12, v6, 16, 1
; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_min_f32_e32 v10, v11, v10
-; GFX10-NEXT: v_min_f32_e32 v11, v13, v12
-; GFX10-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX10-NEXT: v_min_f32_e32 v1, v1, v5
-; GFX10-NEXT: v_min_f32_e32 v2, v2, v6
-; GFX10-NEXT: v_min_f32_e32 v3, v3, v7
-; GFX10-NEXT: v_perm_b32 v0, v0, v11, 0x7060302
-; GFX10-NEXT: v_perm_b32 v1, v1, v10, 0x7060302
-; GFX10-NEXT: v_perm_b32 v2, v2, v9, 0x7060302
-; GFX10-NEXT: v_perm_b32 v3, v3, v8, 0x7060302
+; GFX10-NEXT: v_min_f32_e32 v5, v15, v13
+; GFX10-NEXT: v_and_or_b32 v14, v3, s4, 0x400000
+; GFX10-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v9, v11, vcc_lo
+; GFX10-NEXT: v_add3_u32 v4, v12, v6, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v9, v6, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GFX10-NEXT: v_bfe_u32 v12, v5, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_bfe_u32 v13, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v15, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v6, v11, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v11, v5, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc_lo
+; GFX10-NEXT: v_add3_u32 v9, v12, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v13, v0, s4, 0x400000
+; GFX10-NEXT: v_perm_b32 v2, v2, v8, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v6, v15, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v10, v14, vcc_lo
+; GFX10-NEXT: v_perm_b32 v3, v3, v7, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_minnum_v8bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-NEXT: v_dual_min_f32 v8, v9, v8 :: v_dual_min_f32 v9, v11, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_dual_min_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v9, 16, v6
+; GFX11-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_min_f32_e32 v9, v11, v9
+; GFX11-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v12, v9, s0, 0x400000
+; GFX11-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v5
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX11-NEXT: v_min_f32_e32 v3, v3, v7
+; GFX11-NEXT: v_and_or_b32 v7, v8, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v8, v9, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX11-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-NEXT: v_add3_u32 v9, v13, v2, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_and_or_b32 v14, v3, s0, 0x400000
; GFX11-NEXT: v_min_f32_e32 v0, v0, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-NEXT: v_add3_u32 v10, v11, v3, 0x7fff
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_min_f32 v6, v11, v6 :: v_dual_and_b32 v1, 0xffff0000, v1
+; GFX11-NEXT: v_and_or_b32 v11, v2, s0, 0x400000
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_dual_min_f32 v1, v1, v5 :: v_dual_and_b32 v6, 0xffff0000, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_min_f32 v2, v2, v6 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11-NEXT: v_min_f32_e32 v3, v3, v7
-; GFX11-NEXT: v_dual_min_f32 v10, v11, v10 :: v_dual_min_f32 v11, v13, v12
+; GFX11-NEXT: v_bfe_u32 v12, v6, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v9, v11, vcc_lo
+; GFX11-NEXT: v_and_or_b32 v9, v6, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: v_min_f32_e32 v1, v1, v5
+; GFX11-NEXT: v_add3_u32 v4, v12, v6, 0x7fff
+; GFX11-NEXT: v_perm_b32 v2, v2, v8, 0x7060302
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_perm_b32 v2, v2, v9, 0x7060302
-; GFX11-NEXT: v_perm_b32 v3, v3, v8, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v1, v1, v10, 0x7060302
-; GFX11-NEXT: v_perm_b32 v0, v0, v11, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc_lo
+; GFX11-NEXT: v_min_f32_e32 v5, v15, v13
+; GFX11-NEXT: v_bfe_u32 v13, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v15, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v6, v11, v1, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v12, v5, 16, 1
+; GFX11-NEXT: v_and_or_b32 v11, v5, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_add3_u32 v9, v12, v5, 0x7fff
+; GFX11-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v13, v0, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v6, v15, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v10, v14, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v3, v3, v7, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <8 x bfloat> @llvm.minnum.v8bf16(<8 x bfloat> %a, <8 x bfloat> %b)
ret <8 x bfloat> %op
@@ -13691,56 +18750,87 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GCN-LABEL: v_minnum_v16bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GCN-NEXT: v_min_f32_e32 v14, v14, v30
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GCN-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GCN-NEXT: v_min_f32_e32 v13, v13, v29
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GCN-NEXT: v_min_f32_e32 v12, v12, v28
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GCN-NEXT: v_min_f32_e32 v11, v11, v27
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GCN-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GCN-NEXT: v_min_f32_e32 v10, v10, v26
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GCN-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GCN-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GCN-NEXT: v_min_f32_e32 v9, v9, v25
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GCN-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: v_min_f32_e32 v8, v8, v24
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GCN-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: v_min_f32_e32 v7, v7, v23
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GCN-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GCN-NEXT: v_min_f32_e32 v6, v6, v22
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GCN-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_min_f32_e32 v5, v5, v21
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
@@ -13769,6 +18859,8 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GCN-NEXT: v_min_f32_e32 v2, v2, v18
; GCN-NEXT: v_min_f32_e32 v1, v1, v17
; GCN-NEXT: v_min_f32_e32 v0, v0, v16
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v20
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -13783,8 +18875,7 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v20
+; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GCN-NEXT: v_min_f32_e32 v15, v15, v16
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
@@ -13794,12 +18885,43 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-LABEL: v_minnum_v16bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_min_f32_e32 v9, v9, v25
+; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; GFX7-NEXT: v_min_f32_e32 v6, v6, v22
-; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
@@ -13810,13 +18932,13 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
@@ -13839,17 +18961,21 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
-; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
@@ -13863,9 +18989,10 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_min_f32_e32 v12, v12, v28
; GFX7-NEXT: v_min_f32_e32 v11, v11, v27
; GFX7-NEXT: v_min_f32_e32 v10, v10, v26
-; GFX7-NEXT: v_min_f32_e32 v9, v9, v25
+; GFX7-NEXT: v_min_f32_e32 v15, v15, v25
; GFX7-NEXT: v_min_f32_e32 v8, v8, v24
; GFX7-NEXT: v_min_f32_e32 v7, v7, v23
+; GFX7-NEXT: v_min_f32_e32 v6, v6, v22
; GFX7-NEXT: v_min_f32_e32 v5, v5, v21
; GFX7-NEXT: v_min_f32_e32 v4, v4, v20
; GFX7-NEXT: v_min_f32_e32 v3, v3, v19
@@ -13879,10 +19006,6 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; GFX7-NEXT: v_min_f32_e32 v15, v15, v22
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
@@ -13899,51 +19022,165 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v15
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v7
+; GFX8-NEXT: v_min_f32_e32 v16, v17, v16
+; GFX8-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX8-NEXT: v_min_f32_e32 v16, v17, v16
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v16
; GFX8-NEXT: v_min_f32_e32 v7, v7, v15
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
+; GFX8-NEXT: v_bfe_u32 v15, v7, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, v15, v7
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, s4, v15
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v7
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v15, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX8-NEXT: v_min_f32_e32 v15, v17, v15
+; GFX8-NEXT: v_bfe_u32 v17, v15, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v15
; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX8-NEXT: v_min_f32_e32 v15, v17, v15
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v15
; GFX8-NEXT: v_min_f32_e32 v6, v6, v14
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v15, v15
+; GFX8-NEXT: v_bfe_u32 v14, v6, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v14, vcc, v14, v6
+; GFX8-NEXT: v_add_u32_e32 v14, vcc, s4, v14
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v6
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v13
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX8-NEXT: v_min_f32_e32 v14, v17, v14
+; GFX8-NEXT: v_bfe_u32 v17, v14, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v14
; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX8-NEXT: v_min_f32_e32 v14, v17, v14
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v14
; GFX8-NEXT: v_min_f32_e32 v5, v5, v13
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
+; GFX8-NEXT: v_bfe_u32 v13, v5, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v14, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v13, vcc, v13, v5
+; GFX8-NEXT: v_add_u32_e32 v13, vcc, s4, v13
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v5
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v13, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v12
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v4
+; GFX8-NEXT: v_min_f32_e32 v13, v17, v13
+; GFX8-NEXT: v_bfe_u32 v17, v13, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v13
; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v13, v17, v13
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v13
; GFX8-NEXT: v_min_f32_e32 v4, v4, v12
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX8-NEXT: v_bfe_u32 v12, v4, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v13, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v12, vcc, v12, v4
+; GFX8-NEXT: v_add_u32_e32 v12, vcc, s4, v12
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v4
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v12, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v12, 16, v11
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX8-NEXT: v_min_f32_e32 v12, v17, v12
+; GFX8-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v12
; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_min_f32_e32 v12, v17, v12
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v12
; GFX8-NEXT: v_min_f32_e32 v3, v3, v11
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX8-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v11, vcc, v11, v3
+; GFX8-NEXT: v_add_u32_e32 v11, vcc, s4, v11
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v11, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v2
+; GFX8-NEXT: v_min_f32_e32 v11, v17, v11
+; GFX8-NEXT: v_bfe_u32 v17, v11, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v11
; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: v_min_f32_e32 v11, v17, v11
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v11
; GFX8-NEXT: v_min_f32_e32 v2, v2, v10
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
+; GFX8-NEXT: v_bfe_u32 v10, v2, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v11, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, v10, v2
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, s4, v10
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v10, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v9
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; GFX8-NEXT: v_min_f32_e32 v10, v17, v10
+; GFX8-NEXT: v_bfe_u32 v17, v10, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v10
; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_min_f32_e32 v10, v17, v10
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v10
; GFX8-NEXT: v_min_f32_e32 v1, v1, v9
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
+; GFX8-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v10, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v0
+; GFX8-NEXT: v_min_f32_e32 v9, v17, v9
+; GFX8-NEXT: v_bfe_u32 v17, v9, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v9
; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v9
; GFX8-NEXT: v_min_f32_e32 v0, v0, v8
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v9, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, s4, v8
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v17, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
@@ -13951,7 +19188,6 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_min_f32_e32 v9, v17, v9
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v9, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v10, 16
@@ -13968,52 +19204,149 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v15
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v7
+; GFX9-NEXT: v_min_f32_e32 v16, v17, v16
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v16
; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX9-NEXT: v_min_f32_e32 v16, v17, v16
+; GFX9-NEXT: v_add3_u32 v17, v17, v16, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_min_f32_e32 v7, v7, v15
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v15, v7, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v7
+; GFX9-NEXT: v_add3_u32 v15, v15, v7, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v15, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX9-NEXT: v_min_f32_e32 v15, v17, v15
+; GFX9-NEXT: v_bfe_u32 v17, v15, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v15
; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX9-NEXT: v_min_f32_e32 v15, v17, v15
+; GFX9-NEXT: v_add3_u32 v17, v17, v15, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v15, v15
; GFX9-NEXT: v_min_f32_e32 v6, v6, v14
+; GFX9-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v14, v6, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v6
+; GFX9-NEXT: v_add3_u32 v14, v14, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v14, 16, v13
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX9-NEXT: v_min_f32_e32 v14, v17, v14
+; GFX9-NEXT: v_bfe_u32 v17, v14, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v14
; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX9-NEXT: v_min_f32_e32 v14, v17, v14
+; GFX9-NEXT: v_add3_u32 v17, v17, v14, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
; GFX9-NEXT: v_min_f32_e32 v5, v5, v13
+; GFX9-NEXT: v_cndmask_b32_e32 v14, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v13, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v5
+; GFX9-NEXT: v_add3_u32 v13, v13, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v13, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v13, 16, v12
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v4
+; GFX9-NEXT: v_min_f32_e32 v13, v17, v13
+; GFX9-NEXT: v_bfe_u32 v17, v13, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v13
; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX9-NEXT: v_min_f32_e32 v13, v17, v13
+; GFX9-NEXT: v_add3_u32 v17, v17, v13, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
; GFX9-NEXT: v_min_f32_e32 v4, v4, v12
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v12, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v12, v12, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v12, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v11
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX9-NEXT: v_min_f32_e32 v12, v17, v12
+; GFX9-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v12
; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_min_f32_e32 v12, v17, v12
+; GFX9-NEXT: v_add3_u32 v17, v17, v12, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
; GFX9-NEXT: v_min_f32_e32 v3, v3, v11
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v11, v11, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v11, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v2
+; GFX9-NEXT: v_min_f32_e32 v11, v17, v11
+; GFX9-NEXT: v_bfe_u32 v17, v11, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v11
; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_min_f32_e32 v11, v17, v11
+; GFX9-NEXT: v_add3_u32 v17, v17, v11, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
; GFX9-NEXT: v_min_f32_e32 v2, v2, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v11, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v10, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v10, v10, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v9
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; GFX9-NEXT: v_min_f32_e32 v10, v17, v10
+; GFX9-NEXT: v_bfe_u32 v17, v10, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v10
; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_min_f32_e32 v10, v17, v10
+; GFX9-NEXT: v_add3_u32 v17, v17, v10, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
; GFX9-NEXT: v_min_f32_e32 v1, v1, v9
+; GFX9-NEXT: v_cndmask_b32_e32 v10, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v9, v9, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v0
+; GFX9-NEXT: v_min_f32_e32 v9, v17, v9
+; GFX9-NEXT: v_bfe_u32 v17, v9, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v9
; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_min_f32_e32 v9, v17, v9
+; GFX9-NEXT: v_add3_u32 v17, v17, v9, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
; GFX9-NEXT: v_min_f32_e32 v0, v0, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v8, v8, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v17, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v9, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v10, s4
@@ -14032,119 +19365,297 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v7
; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v5
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v6
; GFX10-NEXT: v_min_f32_e32 v16, v17, v16
-; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v14
; GFX10-NEXT: v_min_f32_e32 v7, v7, v15
-; GFX10-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v12
-; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v4
-; GFX10-NEXT: v_min_f32_e32 v15, v17, v15
+; GFX10-NEXT: v_bfe_u32 v15, v16, 16, 1
+; GFX10-NEXT: v_and_or_b32 v20, v16, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v19, v7, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX10-NEXT: v_min_f32_e32 v17, v18, v17
+; GFX10-NEXT: v_add3_u32 v15, v15, v16, 0x7fff
; GFX10-NEXT: v_min_f32_e32 v6, v6, v14
-; GFX10-NEXT: v_min_f32_e32 v14, v19, v18
+; GFX10-NEXT: v_add3_u32 v18, v19, v7, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v19, v7, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v21, v17, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v15, v15, v20, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v5
+; GFX10-NEXT: v_and_or_b32 v16, v17, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v14, v21, v17, 0x7fff
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v7, v18, v19, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX10-NEXT: v_bfe_u32 v18, v6, 16, 1
+; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT: v_perm_b32 v7, v7, v15, 0x7060302
+; GFX10-NEXT: v_min_f32_e32 v17, v20, v19
+; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v4
; GFX10-NEXT: v_min_f32_e32 v5, v5, v13
-; GFX10-NEXT: v_min_f32_e32 v13, v21, v20
-; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v11
-; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v14, v14, v16, vcc_lo
+; GFX10-NEXT: v_add3_u32 v16, v18, v6, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v13, v6, s4, 0x400000
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX10-NEXT: v_bfe_u32 v20, v17, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_bfe_u32 v21, v5, 16, 1
; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v17, v18, v17
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v16, v13, vcc_lo
+; GFX10-NEXT: v_min_f32_e32 v13, v19, v18
+; GFX10-NEXT: v_add3_u32 v16, v20, v17, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v18, v17, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT: v_add3_u32 v19, v21, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v20, v5, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v21, v13, 16, 1
+; GFX10-NEXT: v_min_f32_e32 v4, v4, v12
+; GFX10-NEXT: v_cndmask_b32_e32 v16, v16, v18, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v11
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v3
+; GFX10-NEXT: v_add3_u32 v17, v21, v13, 0x7fff
; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v19, v20, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v19, v13, s4, 0x400000
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT: v_min_f32_e32 v18, v20, v19
+; GFX10-NEXT: v_min_f32_e32 v12, v18, v12
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v10
+; GFX10-NEXT: v_min_f32_e32 v3, v3, v11
+; GFX10-NEXT: v_and_or_b32 v22, v12, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v13, v17, v19, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v2
+; GFX10-NEXT: v_add3_u32 v11, v20, v4, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v20, v3, 16, 1
; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v9
-; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v1
+; GFX10-NEXT: v_add3_u32 v17, v17, v12, 0x7fff
+; GFX10-NEXT: v_min_f32_e32 v18, v19, v18
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX10-NEXT: v_add3_u32 v19, v20, v3, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v20, v3, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v23, v18, 16, 1
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v10
+; GFX10-NEXT: v_cndmask_b32_e32 v12, v17, v22, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_and_or_b32 v17, v18, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v10, v23, v18, 0x7fff
+; GFX10-NEXT: v_lshlrev_b32_e32 v22, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v19, v20, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v19, v2, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v9
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX10-NEXT: v_and_or_b32 v18, v2, s4, 0x400000
+; GFX10-NEXT: v_and_or_b32 v21, v4, s4, 0x400000
+; GFX10-NEXT: v_perm_b32 v3, v3, v12, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v10, v10, v17, vcc_lo
+; GFX10-NEXT: v_add3_u32 v17, v19, v2, 0x7fff
+; GFX10-NEXT: v_min_f32_e32 v19, v22, v20
+; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v8
; GFX10-NEXT: v_lshlrev_b32_e32 v22, 16, v0
; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_min_f32_e32 v19, v20, v19
-; GFX10-NEXT: v_min_f32_e32 v20, v22, v21
-; GFX10-NEXT: v_min_f32_e32 v0, v0, v8
+; GFX10-NEXT: v_bfe_u32 v23, v19, 16, 1
; GFX10-NEXT: v_min_f32_e32 v1, v1, v9
-; GFX10-NEXT: v_min_f32_e32 v2, v2, v10
-; GFX10-NEXT: v_min_f32_e32 v3, v3, v11
-; GFX10-NEXT: v_min_f32_e32 v4, v4, v12
-; GFX10-NEXT: v_perm_b32 v0, v0, v20, 0x7060302
+; GFX10-NEXT: v_min_f32_e32 v9, v22, v20
+; GFX10-NEXT: v_and_or_b32 v22, v19, s4, 0x400000
+; GFX10-NEXT: v_min_f32_e32 v0, v0, v8
+; GFX10-NEXT: v_add3_u32 v20, v23, v19, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX10-NEXT: v_bfe_u32 v23, v9, 16, 1
+; GFX10-NEXT: v_and_or_b32 v24, v9, s4, 0x400000
+; GFX10-NEXT: v_and_or_b32 v25, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v22, v1, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_bfe_u32 v20, v0, 16, 1
+; GFX10-NEXT: v_add3_u32 v23, v23, v9, 0x7fff
+; GFX10-NEXT: v_perm_b32 v5, v5, v16, 0x7060302
+; GFX10-NEXT: v_perm_b32 v6, v6, v14, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v8, v22, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX10-NEXT: v_add3_u32 v20, v20, v0, 0x7fff
; GFX10-NEXT: v_perm_b32 v1, v1, v19, 0x7060302
-; GFX10-NEXT: v_perm_b32 v2, v2, v18, 0x7060302
-; GFX10-NEXT: v_perm_b32 v3, v3, v17, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v8, v23, v24, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v20, v25, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_perm_b32 v0, v0, v8, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_perm_b32 v2, v2, v10, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v11, v21, vcc_lo
; GFX10-NEXT: v_perm_b32 v4, v4, v13, 0x7060302
-; GFX10-NEXT: v_perm_b32 v5, v5, v14, 0x7060302
-; GFX10-NEXT: v_perm_b32 v6, v6, v15, 0x7060302
-; GFX10-NEXT: v_perm_b32 v7, v7, v16, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_minnum_v16bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v0
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_min_f32 v4, v4, v12 :: v_dual_and_b32 v5, 0xffff0000, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v15
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v7
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v13
-; GFX11-NEXT: v_min_f32_e32 v13, v21, v20
-; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v8
-; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-NEXT: v_dual_min_f32 v16, v17, v16 :: v_dual_and_b32 v15, 0xffff0000, v15
-; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_min_f32 v0, v0, v8 :: v_dual_and_b32 v7, 0xffff0000, v7
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-NEXT: v_perm_b32 v4, v4, v13, 0x7060302
-; GFX11-NEXT: v_min_f32_e32 v7, v7, v15
-; GFX11-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_min_f32 v16, v17, v16 :: v_dual_lshlrev_b32 v17, 16, v14
; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v7, v7, v16, 0x7060302
-; GFX11-NEXT: v_min_f32_e32 v15, v17, v15
+; GFX11-NEXT: v_and_or_b32 v20, v16, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_min_f32_e32 v17, v18, v17
+; GFX11-NEXT: v_min_f32_e32 v6, v6, v14
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_bfe_u32 v21, v17, 16, 1
+; GFX11-NEXT: v_add3_u32 v14, v21, v17, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f32_e32 v7, v7, v15
+; GFX11-NEXT: v_bfe_u32 v15, v16, 16, 1
+; GFX11-NEXT: v_add3_u32 v15, v15, v16, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v16, v17, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v15, v15, v20 :: v_dual_lshlrev_b32 v20, 16, v5
+; GFX11-NEXT: v_bfe_u32 v19, v7, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-NEXT: v_add3_u32 v18, v19, v7, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v19, v7, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v18, v19, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v18, v6, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: v_perm_b32 v7, v7, v15, 0x7060302
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_dual_min_f32 v6, v6, v14 :: v_dual_lshlrev_b32 v17, 16, v11
-; GFX11-NEXT: v_min_f32_e32 v14, v19, v18
+; GFX11-NEXT: v_dual_min_f32 v17, v20, v19 :: v_dual_cndmask_b32 v14, v14, v16
+; GFX11-NEXT: v_add3_u32 v16, v18, v6, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; GFX11-NEXT: v_bfe_u32 v20, v17, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f32_e32 v4, v4, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v11
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX11-NEXT: v_min_f32_e32 v5, v5, v13
+; GFX11-NEXT: v_and_or_b32 v13, v6, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_cndmask_b32 v6, v16, v13 :: v_dual_min_f32 v13, v19, v18
+; GFX11-NEXT: v_add3_u32 v16, v20, v17, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v18, v17, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: v_perm_b32 v6, v6, v14, 0x7060302
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v16, v16, v18, vcc_lo
; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v10
+; GFX11-NEXT: v_bfe_u32 v21, v5, 16, 1
+; GFX11-NEXT: v_and_or_b32 v20, v5, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_min_f32_e32 v12, v18, v12
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v19, v21, v5, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v21, v13, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v10
+; GFX11-NEXT: v_and_or_b32 v22, v12, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v19, v20, vcc_lo
+; GFX11-NEXT: v_add3_u32 v17, v21, v13, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v19, v13, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX11-NEXT: v_and_or_b32 v21, v4, s0, 0x400000
+; GFX11-NEXT: v_perm_b32 v5, v5, v16, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v13, v17, v19, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v2
; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-NEXT: v_add3_u32 v17, v17, v12, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_min_f32_e32 v18, v19, v18
+; GFX11-NEXT: v_cndmask_b32_e32 v12, v17, v22, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v1
+; GFX11-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_dual_min_f32 v17, v18, v17 :: v_dual_and_b32 v10, 0xffff0000, v10
-; GFX11-NEXT: v_perm_b32 v5, v5, v14, 0x7060302
-; GFX11-NEXT: v_perm_b32 v6, v6, v15, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v23, v18, 16, 1
+; GFX11-NEXT: v_and_or_b32 v17, v18, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_min_f32 v2, v2, v10 :: v_dual_and_b32 v1, 0xffff0000, v1
; GFX11-NEXT: v_min_f32_e32 v3, v3, v11
-; GFX11-NEXT: v_dual_min_f32 v18, v20, v19 :: v_dual_lshlrev_b32 v19, 16, v9
-; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v1
+; GFX11-NEXT: v_add3_u32 v11, v20, v4, 0x7fff
+; GFX11-NEXT: v_add3_u32 v10, v23, v18, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v20, v3, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v19, v20, v3, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v20, v3, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v19, v20, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v19, v2, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v9
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-NEXT: v_and_or_b32 v18, v2, s0, 0x400000
+; GFX11-NEXT: v_perm_b32 v3, v3, v12, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v10, v10, v17, vcc_lo
+; GFX11-NEXT: v_add3_u32 v17, v19, v2, 0x7fff
+; GFX11-NEXT: v_min_f32_e32 v19, v22, v20
+; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v8
+; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX11-NEXT: v_dual_min_f32 v2, v2, v10 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_perm_b32 v3, v3, v17, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_min_f32 v19, v20, v19 :: v_dual_min_f32 v20, v22, v21
-; GFX11-NEXT: v_min_f32_e32 v1, v1, v9
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_perm_b32 v2, v2, v18, 0x7060302
-; GFX11-NEXT: v_perm_b32 v0, v0, v20, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v23, v19, 16, 1
+; GFX11-NEXT: v_dual_min_f32 v0, v0, v8 :: v_dual_min_f32 v1, v1, v9
+; GFX11-NEXT: v_min_f32_e32 v9, v22, v20
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v20, v23, v19, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v22, v19, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v25, v0, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-NEXT: v_bfe_u32 v23, v9, 16, 1
+; GFX11-NEXT: v_and_or_b32 v24, v9, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX11-NEXT: v_and_or_b32 v22, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_bfe_u32 v20, v0, 16, 1
+; GFX11-NEXT: v_add3_u32 v23, v23, v9, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v8, v22, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-NEXT: v_add3_u32 v20, v20, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_perm_b32 v1, v1, v19, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v8, v23, v24, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v20, v25, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v0, v0, v8, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_perm_b32 v2, v2, v10, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v11, v21, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v4, v4, v13, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <16 x bfloat> @llvm.minnum.v16bf16(<16 x bfloat> %a, <16 x bfloat> %b)
ret <16 x bfloat> %op
@@ -14154,230 +19665,294 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GCN-LABEL: v_minnum_v32bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:128
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32
+; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; GCN-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
-; GCN-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GCN-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:124
-; GCN-NEXT: v_min_f32_e32 v31, v32, v31
+; GCN-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GCN-NEXT: v_min_f32_e32 v31, v31, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124
+; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:120
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v30, v30, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120
+; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GCN-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:116
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v29, v29, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116
+; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v28, v28, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112
+; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:108
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v27, v27, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108
+; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:104
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v26, v26, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104
+; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GCN-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:100
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v25, v25, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100
+; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GCN-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:96
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v24, v24, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96
+; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GCN-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:92
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v23, v23, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GCN-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:88
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v22, v22, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88
+; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GCN-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:84
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v21, v21, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GCN-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:80
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v20, v20, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GCN-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:76
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v19, v19, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GCN-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:72
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v18, v18, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GCN-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:68
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v17, v17, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v16, v16, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v15, v15, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:56
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v14, v14, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:52
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v13, v13, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:48
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v12, v12, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:44
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v11, v11, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GCN-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:40
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v10, v10, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GCN-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:36
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v9, v9, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:32
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v8, v8, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:28
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v7, v7, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:24
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v6, v6, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:20
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v5, v5, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:16
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v4, v4, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v3, v3, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v2, v2, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:4
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v1, v1, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_min_f32_e32 v0, v0, v32
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -14417,260 +19992,324 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX7-LABEL: v_minnum_v32bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:128
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128
+; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30
+; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29
+; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28
+; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27
+; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GFX7-NEXT: v_min_f32_e32 v31, v31, v32
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
-; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GFX7-NEXT: v_min_f32_e32 v31, v32, v31
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v30, v30, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120
; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v29, v29, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v28, v28, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112
; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v27, v27, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108
; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v26, v26, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104
; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v25, v25, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100
; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v24, v24, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96
; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v23, v23, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v22, v22, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88
; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v21, v21, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v20, v20, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80
; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v19, v19, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76
; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v18, v18, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72
; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v17, v17, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68
; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v16, v16, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64
; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v15, v15, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v14, v14, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v13, v13, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52
; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v12, v12, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48
; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v11, v11, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v10, v10, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v9, v9, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v8, v8, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v7, v7, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v6, v6, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v5, v5, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v4, v4, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v3, v3, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v2, v2, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v1, v1, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_min_f32_e32 v0, v0, v32
@@ -14682,114 +20321,329 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v14
+; GFX8-NEXT: v_min_f32_e32 v31, v32, v31
+; GFX8-NEXT: v_bfe_u32 v32, v31, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v32, vcc, v32, v31
; GFX8-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX8-NEXT: v_min_f32_e32 v31, v32, v31
-; GFX8-NEXT: v_min_f32_e32 v30, v14, v30
-; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v29
+; GFX8-NEXT: v_add_u32_e32 v32, vcc, s4, v32
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v31
+; GFX8-NEXT: v_min_f32_e32 v14, v14, v30
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_bfe_u32 v30, v14, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v32, v33, vcc
+; GFX8-NEXT: v_add_u32_e32 v30, vcc, v30, v14
+; GFX8-NEXT: v_add_u32_e32 v30, vcc, s4, v30
+; GFX8-NEXT: v_and_b32_e32 v32, 0x80000000, v14
+; GFX8-NEXT: v_or_b32_e32 v32, 0x400000, v32
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
+; GFX8-NEXT: v_cndmask_b32_e32 v14, v30, v32, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v30, 16, v29
; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v13
+; GFX8-NEXT: v_min_f32_e32 v32, v32, v30
+; GFX8-NEXT: buffer_load_dword v30, off, s[0:3], s32
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX8-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX8-NEXT: v_min_f32_e32 v14, v32, v14
; GFX8-NEXT: v_min_f32_e32 v13, v13, v29
+; GFX8-NEXT: v_bfe_u32 v29, v13, 16, 1
+; GFX8-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX8-NEXT: v_alignbit_b32 v14, v14, v31, 16
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v34, 16, v30
+; GFX8-NEXT: v_min_f32_e32 v33, v33, v34
+; GFX8-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; GFX8-NEXT: v_min_f32_e32 v30, v15, v30
+; GFX8-NEXT: v_bfe_u32 v15, v33, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, v15, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v33
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, s4, v15
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX8-NEXT: v_bfe_u32 v33, v30, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v15, v15, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v30
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v30
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
+; GFX8-NEXT: v_cndmask_b32_e32 v30, v33, v34, vcc
+; GFX8-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v32
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v32
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
+; GFX8-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v29, vcc, v29, v13
+; GFX8-NEXT: v_add_u32_e32 v29, vcc, s4, v29
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v13
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX8-NEXT: v_cndmask_b32_e32 v13, v29, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v29, 16, v28
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v12
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v12
+; GFX8-NEXT: v_min_f32_e32 v29, v33, v29
+; GFX8-NEXT: v_bfe_u32 v33, v29, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v29
; GFX8-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX8-NEXT: v_min_f32_e32 v29, v32, v29
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v29
; GFX8-NEXT: v_min_f32_e32 v12, v12, v28
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v29, v29
+; GFX8-NEXT: v_bfe_u32 v28, v12, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v29, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v28, vcc, v28, v12
+; GFX8-NEXT: v_add_u32_e32 v28, vcc, s4, v28
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v12
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX8-NEXT: v_cndmask_b32_e32 v12, v28, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v28, 16, v27
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v11
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v11
+; GFX8-NEXT: v_min_f32_e32 v28, v33, v28
+; GFX8-NEXT: v_bfe_u32 v33, v28, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v28
; GFX8-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; GFX8-NEXT: v_min_f32_e32 v28, v32, v28
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v28
; GFX8-NEXT: v_min_f32_e32 v11, v11, v27
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v28, v28
+; GFX8-NEXT: v_bfe_u32 v27, v11, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v28, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v27, vcc, v27, v11
+; GFX8-NEXT: v_add_u32_e32 v27, vcc, s4, v27
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v11
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
+; GFX8-NEXT: v_cndmask_b32_e32 v11, v27, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v27, 16, v26
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v10
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v10
+; GFX8-NEXT: v_min_f32_e32 v27, v33, v27
+; GFX8-NEXT: v_bfe_u32 v33, v27, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v27
; GFX8-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX8-NEXT: v_min_f32_e32 v27, v32, v27
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v27
; GFX8-NEXT: v_min_f32_e32 v10, v10, v26
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v27, v27
+; GFX8-NEXT: v_bfe_u32 v26, v10, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v27, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v26, vcc, v26, v10
+; GFX8-NEXT: v_add_u32_e32 v26, vcc, s4, v26
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v10
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
+; GFX8-NEXT: v_cndmask_b32_e32 v10, v26, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v26, 16, v25
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v9
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v9
+; GFX8-NEXT: v_min_f32_e32 v26, v33, v26
+; GFX8-NEXT: v_bfe_u32 v33, v26, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v26
; GFX8-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX8-NEXT: v_min_f32_e32 v26, v32, v26
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v26
; GFX8-NEXT: v_min_f32_e32 v9, v9, v25
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v26, v26
+; GFX8-NEXT: v_bfe_u32 v25, v9, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v26, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v25, vcc, v25, v9
+; GFX8-NEXT: v_add_u32_e32 v25, vcc, s4, v25
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v9
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX8-NEXT: v_cndmask_b32_e32 v9, v25, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v25, 16, v24
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v8
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v8
+; GFX8-NEXT: v_min_f32_e32 v25, v33, v25
+; GFX8-NEXT: v_bfe_u32 v33, v25, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v25
; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v25
; GFX8-NEXT: v_min_f32_e32 v8, v8, v24
-; GFX8-NEXT: buffer_load_dword v24, off, s[0:3], s32
-; GFX8-NEXT: v_min_f32_e32 v25, v32, v25
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v15
-; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; GFX8-NEXT: v_alignbit_b32 v8, v8, v25, 16
-; GFX8-NEXT: v_alignbit_b32 v9, v9, v26, 16
-; GFX8-NEXT: v_alignbit_b32 v10, v10, v27, 16
-; GFX8-NEXT: v_alignbit_b32 v11, v11, v28, 16
-; GFX8-NEXT: v_alignbit_b32 v12, v12, v29, 16
-; GFX8-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v24
-; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; GFX8-NEXT: v_min_f32_e32 v32, v32, v33
-; GFX8-NEXT: v_min_f32_e32 v15, v15, v24
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v25, v25
+; GFX8-NEXT: v_bfe_u32 v24, v8, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v25, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v24, vcc, v24, v8
+; GFX8-NEXT: v_add_u32_e32 v24, vcc, s4, v24
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v8
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v24, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v24, 16, v23
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v7
+; GFX8-NEXT: v_min_f32_e32 v24, v33, v24
+; GFX8-NEXT: v_bfe_u32 v33, v24, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v24
; GFX8-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX8-NEXT: v_min_f32_e32 v24, v33, v24
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v24
; GFX8-NEXT: v_min_f32_e32 v7, v7, v23
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v24, v24
+; GFX8-NEXT: v_bfe_u32 v23, v7, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v24, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v23, vcc, v23, v7
+; GFX8-NEXT: v_add_u32_e32 v23, vcc, s4, v23
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v7
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v23, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v23, 16, v22
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v6
+; GFX8-NEXT: v_min_f32_e32 v23, v33, v23
+; GFX8-NEXT: v_bfe_u32 v33, v23, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v23
; GFX8-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX8-NEXT: v_min_f32_e32 v23, v33, v23
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v23
; GFX8-NEXT: v_min_f32_e32 v6, v6, v22
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v23, v23
+; GFX8-NEXT: v_bfe_u32 v22, v6, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v23, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v22, vcc, v22, v6
+; GFX8-NEXT: v_add_u32_e32 v22, vcc, s4, v22
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v6
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v22, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v22, 16, v21
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v5
+; GFX8-NEXT: v_min_f32_e32 v22, v33, v22
+; GFX8-NEXT: v_bfe_u32 v33, v22, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v22
; GFX8-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX8-NEXT: v_min_f32_e32 v22, v33, v22
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v22
; GFX8-NEXT: v_min_f32_e32 v5, v5, v21
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v22, v22
+; GFX8-NEXT: v_bfe_u32 v21, v5, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v22, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v21, vcc, v21, v5
+; GFX8-NEXT: v_add_u32_e32 v21, vcc, s4, v21
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v5
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v21, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v21, 16, v20
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v4
+; GFX8-NEXT: v_min_f32_e32 v21, v33, v21
+; GFX8-NEXT: v_bfe_u32 v33, v21, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v21
; GFX8-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v21, v33, v21
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v21
; GFX8-NEXT: v_min_f32_e32 v4, v4, v20
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v21, v21
+; GFX8-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v21, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v20, vcc, v20, v4
+; GFX8-NEXT: v_add_u32_e32 v20, vcc, s4, v20
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v4
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v20, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v20, 16, v19
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v3
+; GFX8-NEXT: v_min_f32_e32 v20, v33, v20
+; GFX8-NEXT: v_bfe_u32 v33, v20, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v20
; GFX8-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_min_f32_e32 v20, v33, v20
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v20
; GFX8-NEXT: v_min_f32_e32 v3, v3, v19
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v20, v20
+; GFX8-NEXT: v_bfe_u32 v19, v3, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v20, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v19, vcc, v19, v3
+; GFX8-NEXT: v_add_u32_e32 v19, vcc, s4, v19
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v19, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v19, 16, v18
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v2
+; GFX8-NEXT: v_min_f32_e32 v19, v33, v19
+; GFX8-NEXT: v_bfe_u32 v33, v19, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v19
; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: v_min_f32_e32 v19, v33, v19
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v19
; GFX8-NEXT: v_min_f32_e32 v2, v2, v18
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v19, v19
+; GFX8-NEXT: v_bfe_u32 v18, v2, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v19, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v18, vcc, v18, v2
+; GFX8-NEXT: v_add_u32_e32 v18, vcc, s4, v18
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v18, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v17
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v1
+; GFX8-NEXT: v_min_f32_e32 v18, v33, v18
+; GFX8-NEXT: v_bfe_u32 v33, v18, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v18
; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_min_f32_e32 v18, v33, v18
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v18
; GFX8-NEXT: v_min_f32_e32 v1, v1, v17
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
+; GFX8-NEXT: v_bfe_u32 v17, v1, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v16
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v0
+; GFX8-NEXT: v_min_f32_e32 v17, v33, v17
+; GFX8-NEXT: v_bfe_u32 v33, v17, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v17
; GFX8-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v17
; GFX8-NEXT: v_min_f32_e32 v0, v0, v16
-; GFX8-NEXT: v_min_f32_e32 v17, v33, v17
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
+; GFX8-NEXT: v_bfe_u32 v16, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v16, vcc, v16, v0
+; GFX8-NEXT: v_add_u32_e32 v16, vcc, s4, v16
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v16, v33, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
@@ -14798,8 +20652,13 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v30
+; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11
; GFX8-NEXT: v_alignbit_b32 v0, v0, v17, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v18, 16
; GFX8-NEXT: v_alignbit_b32 v2, v2, v19, 16
@@ -14808,8 +20667,13 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX8-NEXT: v_alignbit_b32 v5, v5, v22, 16
; GFX8-NEXT: v_alignbit_b32 v6, v6, v23, 16
; GFX8-NEXT: v_alignbit_b32 v7, v7, v24, 16
-; GFX8-NEXT: v_alignbit_b32 v14, v16, v31, 16
-; GFX8-NEXT: v_alignbit_b32 v15, v15, v32, 16
+; GFX8-NEXT: v_alignbit_b32 v8, v8, v25, 16
+; GFX8-NEXT: v_alignbit_b32 v9, v9, v26, 16
+; GFX8-NEXT: v_alignbit_b32 v10, v10, v27, 16
+; GFX8-NEXT: v_alignbit_b32 v11, v11, v28, 16
+; GFX8-NEXT: v_alignbit_b32 v12, v12, v29, 16
+; GFX8-NEXT: v_alignbit_b32 v13, v13, v32, 16
+; GFX8-NEXT: v_alignbit_b32 v15, v16, v15, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_minnum_v32bf16:
@@ -14817,110 +20681,296 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v14
+; GFX9-NEXT: v_min_f32_e32 v31, v32, v31
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_bfe_u32 v32, v31, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v31
; GFX9-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX9-NEXT: v_min_f32_e32 v31, v32, v31
+; GFX9-NEXT: v_add3_u32 v32, v32, v31, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
; GFX9-NEXT: v_min_f32_e32 v14, v14, v30
+; GFX9-NEXT: v_cndmask_b32_e32 v31, v32, v33, vcc
+; GFX9-NEXT: v_bfe_u32 v30, v14, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v32, 0x80000000, v14
+; GFX9-NEXT: v_add3_u32 v30, v30, v14, s4
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
+; GFX9-NEXT: v_cndmask_b32_e32 v14, v30, v32, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v29
; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v13
; GFX9-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX9-NEXT: v_min_f32_e32 v30, v32, v30
; GFX9-NEXT: v_min_f32_e32 v13, v13, v29
-; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v28
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v12
+; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32
+; GFX9-NEXT: v_min_f32_e32 v30, v32, v30
+; GFX9-NEXT: v_bfe_u32 v32, v30, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v30
+; GFX9-NEXT: v_add3_u32 v32, v32, v30, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
+; GFX9-NEXT: v_cndmask_b32_e32 v30, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v15
+; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v29
+; GFX9-NEXT: v_min_f32_e32 v32, v32, v33
+; GFX9-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX9-NEXT: v_min_f32_e32 v29, v15, v29
+; GFX9-NEXT: v_bfe_u32 v15, v32, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v32
+; GFX9-NEXT: v_add3_u32 v15, v15, v32, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v15, v15, v33, vcc
+; GFX9-NEXT: v_bfe_u32 v32, v29, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v29
+; GFX9-NEXT: v_add3_u32 v32, v32, v29, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v29, v29
+; GFX9-NEXT: v_cndmask_b32_e32 v29, v32, v33, vcc
+; GFX9-NEXT: v_bfe_u32 v32, v13, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v13
+; GFX9-NEXT: v_add3_u32 v32, v32, v13, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v12
+; GFX9-NEXT: v_min_f32_e32 v32, v33, v32
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v32
; GFX9-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX9-NEXT: v_min_f32_e32 v29, v32, v29
+; GFX9-NEXT: v_add3_u32 v33, v33, v32, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_min_f32_e32 v12, v12, v28
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v28, v12, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v12
+; GFX9-NEXT: v_add3_u32 v28, v28, v12, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v28, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v27
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v11
+; GFX9-NEXT: v_min_f32_e32 v28, v33, v28
+; GFX9-NEXT: v_bfe_u32 v33, v28, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v28
; GFX9-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; GFX9-NEXT: v_min_f32_e32 v28, v32, v28
+; GFX9-NEXT: v_add3_u32 v33, v33, v28, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v28, v28
; GFX9-NEXT: v_min_f32_e32 v11, v11, v27
+; GFX9-NEXT: v_cndmask_b32_e32 v28, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v27, v11, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v11
+; GFX9-NEXT: v_add3_u32 v27, v27, v11, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
+; GFX9-NEXT: v_cndmask_b32_e32 v11, v27, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v26
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v10
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v10
+; GFX9-NEXT: v_min_f32_e32 v27, v33, v27
+; GFX9-NEXT: v_bfe_u32 v33, v27, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v27
; GFX9-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX9-NEXT: v_min_f32_e32 v27, v32, v27
+; GFX9-NEXT: v_add3_u32 v33, v33, v27, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v27, v27
; GFX9-NEXT: v_min_f32_e32 v10, v10, v26
+; GFX9-NEXT: v_cndmask_b32_e32 v27, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v26, v10, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v10
+; GFX9-NEXT: v_add3_u32 v26, v26, v10, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v10, v26, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v25
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v9
+; GFX9-NEXT: v_min_f32_e32 v26, v33, v26
+; GFX9-NEXT: v_bfe_u32 v33, v26, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v26
; GFX9-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX9-NEXT: v_min_f32_e32 v26, v32, v26
+; GFX9-NEXT: v_add3_u32 v33, v33, v26, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v26, v26
; GFX9-NEXT: v_min_f32_e32 v9, v9, v25
+; GFX9-NEXT: v_cndmask_b32_e32 v26, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v25, v9, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v9
+; GFX9-NEXT: v_add3_u32 v25, v25, v9, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v25, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v24
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v8
+; GFX9-NEXT: v_min_f32_e32 v25, v33, v25
+; GFX9-NEXT: v_bfe_u32 v33, v25, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v25
; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; GFX9-NEXT: v_add3_u32 v33, v33, v25, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v25, v25
; GFX9-NEXT: v_min_f32_e32 v8, v8, v24
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32
-; GFX9-NEXT: v_min_f32_e32 v25, v32, v25
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v15
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: v_perm_b32 v8, v8, v25, s4
-; GFX9-NEXT: v_perm_b32 v9, v9, v26, s4
-; GFX9-NEXT: v_perm_b32 v10, v10, v27, s4
-; GFX9-NEXT: v_perm_b32 v11, v11, v28, s4
-; GFX9-NEXT: v_perm_b32 v12, v12, v29, s4
-; GFX9-NEXT: v_perm_b32 v13, v13, v30, s4
-; GFX9-NEXT: v_perm_b32 v14, v14, v31, s4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v24
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; GFX9-NEXT: v_min_f32_e32 v32, v32, v33
-; GFX9-NEXT: v_min_f32_e32 v15, v15, v24
+; GFX9-NEXT: v_cndmask_b32_e32 v25, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v24, v8, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v8
+; GFX9-NEXT: v_add3_u32 v24, v24, v8, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v24, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v23
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v7
+; GFX9-NEXT: v_min_f32_e32 v24, v33, v24
+; GFX9-NEXT: v_bfe_u32 v33, v24, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v24
; GFX9-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX9-NEXT: v_min_f32_e32 v24, v33, v24
+; GFX9-NEXT: v_add3_u32 v33, v33, v24, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v24, v24
; GFX9-NEXT: v_min_f32_e32 v7, v7, v23
+; GFX9-NEXT: v_cndmask_b32_e32 v24, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v23, v7, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v7
+; GFX9-NEXT: v_add3_u32 v23, v23, v7, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v23, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v22
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v6
+; GFX9-NEXT: v_min_f32_e32 v23, v33, v23
+; GFX9-NEXT: v_bfe_u32 v33, v23, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v23
; GFX9-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX9-NEXT: v_min_f32_e32 v23, v33, v23
+; GFX9-NEXT: v_add3_u32 v33, v33, v23, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v23, v23
; GFX9-NEXT: v_min_f32_e32 v6, v6, v22
+; GFX9-NEXT: v_cndmask_b32_e32 v23, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v22, v6, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v6
+; GFX9-NEXT: v_add3_u32 v22, v22, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v22, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v21
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v5
+; GFX9-NEXT: v_min_f32_e32 v22, v33, v22
+; GFX9-NEXT: v_bfe_u32 v33, v22, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v22
; GFX9-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX9-NEXT: v_min_f32_e32 v22, v33, v22
+; GFX9-NEXT: v_add3_u32 v33, v33, v22, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v22, v22
; GFX9-NEXT: v_min_f32_e32 v5, v5, v21
+; GFX9-NEXT: v_cndmask_b32_e32 v22, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v21, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v5
+; GFX9-NEXT: v_add3_u32 v21, v21, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v21, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v20
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v4
+; GFX9-NEXT: v_min_f32_e32 v21, v33, v21
+; GFX9-NEXT: v_bfe_u32 v33, v21, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v21
; GFX9-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX9-NEXT: v_min_f32_e32 v21, v33, v21
+; GFX9-NEXT: v_add3_u32 v33, v33, v21, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v21, v21
; GFX9-NEXT: v_min_f32_e32 v4, v4, v20
+; GFX9-NEXT: v_cndmask_b32_e32 v21, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v20, v20, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v20, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v19
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v3
+; GFX9-NEXT: v_min_f32_e32 v20, v33, v20
+; GFX9-NEXT: v_bfe_u32 v33, v20, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v20
; GFX9-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_min_f32_e32 v20, v33, v20
+; GFX9-NEXT: v_add3_u32 v33, v33, v20, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v20, v20
; GFX9-NEXT: v_min_f32_e32 v3, v3, v19
+; GFX9-NEXT: v_cndmask_b32_e32 v20, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v19, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v19, v19, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v19, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v18
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v2
+; GFX9-NEXT: v_min_f32_e32 v19, v33, v19
+; GFX9-NEXT: v_bfe_u32 v33, v19, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v19
; GFX9-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_min_f32_e32 v19, v33, v19
+; GFX9-NEXT: v_add3_u32 v33, v33, v19, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v19, v19
; GFX9-NEXT: v_min_f32_e32 v2, v2, v18
+; GFX9-NEXT: v_cndmask_b32_e32 v19, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v18, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v18, v18, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v18, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v1
+; GFX9-NEXT: v_min_f32_e32 v18, v33, v18
+; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v18
; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_min_f32_e32 v18, v33, v18
+; GFX9-NEXT: v_add3_u32 v33, v33, v18, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
; GFX9-NEXT: v_min_f32_e32 v1, v1, v17
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v17, v17, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v16
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v0
+; GFX9-NEXT: v_min_f32_e32 v17, v33, v17
+; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v17
; GFX9-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_min_f32_e32 v17, v33, v17
+; GFX9-NEXT: v_add3_u32 v33, v33, v17, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_min_f32_e32 v0, v0, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v16, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v16, v16, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v16, v33, vcc
+; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v17, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v18, s4
; GFX9-NEXT: v_perm_b32 v2, v2, v19, s4
@@ -14929,7 +20979,14 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX9-NEXT: v_perm_b32 v5, v5, v22, s4
; GFX9-NEXT: v_perm_b32 v6, v6, v23, s4
; GFX9-NEXT: v_perm_b32 v7, v7, v24, s4
-; GFX9-NEXT: v_perm_b32 v15, v15, v32, s4
+; GFX9-NEXT: v_perm_b32 v8, v8, v25, s4
+; GFX9-NEXT: v_perm_b32 v9, v9, v26, s4
+; GFX9-NEXT: v_perm_b32 v10, v10, v27, s4
+; GFX9-NEXT: v_perm_b32 v11, v11, v28, s4
+; GFX9-NEXT: v_perm_b32 v12, v12, v32, s4
+; GFX9-NEXT: v_perm_b32 v13, v13, v30, s4
+; GFX9-NEXT: v_perm_b32 v14, v14, v31, s4
+; GFX9-NEXT: v_perm_b32 v15, v29, v15, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_minnum_v32bf16:
@@ -14944,32 +21001,10 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v10
; GFX10-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v30
-; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v14
-; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
-; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v29
-; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v13
-; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v28
; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v12
; GFX10-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX10-NEXT: v_min_f32_e32 v39, v48, v39
-; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v17
-; GFX10-NEXT: v_min_f32_e32 v11, v11, v27
-; GFX10-NEXT: v_lshlrev_b32_e32 v27, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_min_f32_e32 v49, v50, v49
-; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v16
-; GFX10-NEXT: v_min_f32_e32 v10, v10, v26
-; GFX10-NEXT: v_lshlrev_b32_e32 v26, 16, v0
-; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v15
-; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX10-NEXT: v_lshlrev_b32_e32 v51, 16, v25
; GFX10-NEXT: v_lshlrev_b32_e32 v52, 16, v9
; GFX10-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
@@ -14988,6 +21023,68 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX10-NEXT: v_lshlrev_b32_e32 v67, 16, v21
; GFX10-NEXT: v_lshlrev_b32_e32 v68, 16, v5
+; GFX10-NEXT: v_min_f32_e32 v39, v48, v39
+; GFX10-NEXT: v_min_f32_e32 v11, v11, v27
+; GFX10-NEXT: v_min_f32_e32 v49, v50, v49
+; GFX10-NEXT: v_min_f32_e32 v10, v10, v26
+; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v29
+; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v13
+; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; GFX10-NEXT: v_min_f32_e32 v37, v38, v37
+; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v18
+; GFX10-NEXT: v_min_f32_e32 v12, v12, v28
+; GFX10-NEXT: v_lshlrev_b32_e32 v28, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v17
+; GFX10-NEXT: v_lshlrev_b32_e32 v27, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v16
+; GFX10-NEXT: v_lshlrev_b32_e32 v26, 16, v0
+; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: v_min_f32_e32 v9, v9, v25
+; GFX10-NEXT: v_min_f32_e32 v25, v54, v53
+; GFX10-NEXT: v_min_f32_e32 v8, v8, v24
+; GFX10-NEXT: v_min_f32_e32 v24, v64, v55
+; GFX10-NEXT: v_min_f32_e32 v7, v7, v23
+; GFX10-NEXT: v_min_f32_e32 v23, v66, v65
+; GFX10-NEXT: v_min_f32_e32 v6, v6, v22
+; GFX10-NEXT: v_min_f32_e32 v22, v68, v67
+; GFX10-NEXT: v_bfe_u32 v53, v39, 16, 1
+; GFX10-NEXT: v_bfe_u32 v55, v11, 16, 1
+; GFX10-NEXT: v_bfe_u32 v65, v49, 16, 1
+; GFX10-NEXT: v_bfe_u32 v67, v10, 16, 1
+; GFX10-NEXT: s_brev_b32 s23, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v30
+; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v14
+; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; GFX10-NEXT: v_min_f32_e32 v35, v36, v35
+; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v19
+; GFX10-NEXT: v_min_f32_e32 v13, v13, v29
+; GFX10-NEXT: v_lshlrev_b32_e32 v29, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v18
+; GFX10-NEXT: v_min_f32_e32 v18, v27, v48
+; GFX10-NEXT: v_min_f32_e32 v1, v1, v17
+; GFX10-NEXT: v_min_f32_e32 v17, v26, v50
+; GFX10-NEXT: v_min_f32_e32 v0, v0, v16
+; GFX10-NEXT: v_and_or_b32 v54, v39, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v64, v11, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v66, v49, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v68, v10, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s9, v39, v39
+; GFX10-NEXT: v_add3_u32 v39, v53, v39, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s10, v11, v11
+; GFX10-NEXT: v_add3_u32 v11, v55, v11, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s11, v49, v49
+; GFX10-NEXT: v_add3_u32 v49, v65, v49, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s12, v10, v10
+; GFX10-NEXT: v_add3_u32 v10, v67, v10, 0x7fff
; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX10-NEXT: v_min_f32_e32 v33, v34, v33
@@ -14996,59 +21093,180 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v30, 16, v4
; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v35, v36, v35
-; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v19
-; GFX10-NEXT: v_min_f32_e32 v13, v13, v29
-; GFX10-NEXT: v_lshlrev_b32_e32 v29, 16, v3
-; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT: v_min_f32_e32 v37, v38, v37
-; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v18
-; GFX10-NEXT: v_min_f32_e32 v12, v12, v28
-; GFX10-NEXT: v_lshlrev_b32_e32 v28, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_min_f32_e32 v0, v0, v16
-; GFX10-NEXT: v_min_f32_e32 v1, v1, v17
+; GFX10-NEXT: v_min_f32_e32 v3, v3, v19
+; GFX10-NEXT: v_min_f32_e32 v19, v28, v38
+; GFX10-NEXT: v_bfe_u32 v38, v37, 16, 1
+; GFX10-NEXT: v_bfe_u32 v50, v12, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v39, v39, v54, s9
+; GFX10-NEXT: v_bfe_u32 v54, v18, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v11, v11, v64, s10
+; GFX10-NEXT: v_bfe_u32 v64, v1, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v49, v49, v66, s11
+; GFX10-NEXT: v_bfe_u32 v66, v17, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v68, s12
+; GFX10-NEXT: v_bfe_u32 v68, v0, 16, 1
; GFX10-NEXT: v_min_f32_e32 v51, v52, v51
-; GFX10-NEXT: v_min_f32_e32 v9, v9, v25
-; GFX10-NEXT: v_min_f32_e32 v25, v54, v53
-; GFX10-NEXT: v_min_f32_e32 v8, v8, v24
-; GFX10-NEXT: v_min_f32_e32 v24, v64, v55
-; GFX10-NEXT: v_min_f32_e32 v7, v7, v23
-; GFX10-NEXT: v_min_f32_e32 v23, v66, v65
-; GFX10-NEXT: v_min_f32_e32 v6, v6, v22
-; GFX10-NEXT: v_min_f32_e32 v22, v68, v67
; GFX10-NEXT: v_min_f32_e32 v5, v5, v21
; GFX10-NEXT: v_min_f32_e32 v21, v30, v34
-; GFX10-NEXT: v_min_f32_e32 v29, v29, v36
-; GFX10-NEXT: v_min_f32_e32 v28, v28, v38
-; GFX10-NEXT: v_min_f32_e32 v27, v27, v48
-; GFX10-NEXT: v_min_f32_e32 v26, v26, v50
-; GFX10-NEXT: v_min_f32_e32 v2, v2, v18
-; GFX10-NEXT: v_min_f32_e32 v3, v3, v19
; GFX10-NEXT: v_min_f32_e32 v4, v4, v20
-; GFX10-NEXT: v_perm_b32 v1, v1, v27, 0x7060302
-; GFX10-NEXT: v_perm_b32 v0, v0, v26, 0x7060302
-; GFX10-NEXT: v_perm_b32 v2, v2, v28, 0x7060302
-; GFX10-NEXT: v_perm_b32 v3, v3, v29, 0x7060302
+; GFX10-NEXT: v_min_f32_e32 v20, v29, v36
+; GFX10-NEXT: v_bfe_u32 v16, v33, 16, 1
+; GFX10-NEXT: v_bfe_u32 v27, v14, 16, 1
+; GFX10-NEXT: v_bfe_u32 v29, v35, 16, 1
+; GFX10-NEXT: v_bfe_u32 v34, v13, 16, 1
+; GFX10-NEXT: v_and_or_b32 v48, v37, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v52, v12, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s7, v37, v37
+; GFX10-NEXT: v_add3_u32 v37, v38, v37, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s8, v12, v12
+; GFX10-NEXT: v_add3_u32 v12, v50, v12, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s10, v18, v18
+; GFX10-NEXT: v_add3_u32 v54, v54, v18, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v18, v18, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s11, v1, v1
+; GFX10-NEXT: v_add3_u32 v64, v64, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v1, v1, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s12, v17, v17
+; GFX10-NEXT: v_add3_u32 v66, v66, v17, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v17, v17, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s22, v0, v0
+; GFX10-NEXT: v_add3_u32 v68, v68, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v0, v0, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v26, v33, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v28, v14, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v30, v35, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v36, v13, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX10-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v33, v51, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v14, v14
+; GFX10-NEXT: v_add3_u32 v14, v27, v14, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s5, v35, v35
+; GFX10-NEXT: v_add3_u32 v29, v29, v35, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s6, v13, v13
+; GFX10-NEXT: v_add3_u32 v13, v34, v13, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v65, v24, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v37, v37, v48, s7
+; GFX10-NEXT: v_bfe_u32 v48, v19, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v12, v12, v52, s8
+; GFX10-NEXT: v_bfe_u32 v52, v2, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v18, v54, v18, s10
+; GFX10-NEXT: v_cndmask_b32_e64 v17, v66, v17, s12
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v68, v0, s22
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v64, v1, s11
+; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v15
+; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX10-NEXT: v_and_or_b32 v27, v51, s23, 0x400000
+; GFX10-NEXT: v_bfe_u32 v35, v9, 16, 1
+; GFX10-NEXT: v_bfe_u32 v38, v25, 16, 1
+; GFX10-NEXT: v_and_or_b32 v67, v24, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s13, v51, v51
+; GFX10-NEXT: v_add3_u32 v33, v33, v51, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v51, v7, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s17, v24, v24
+; GFX10-NEXT: v_add3_u32 v24, v65, v24, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v65, v6, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v16, v16, v26, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v26, v21, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v14, v14, v28, s4
+; GFX10-NEXT: v_bfe_u32 v28, v4, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v29, v29, v30, s5
+; GFX10-NEXT: v_bfe_u32 v30, v20, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v13, v13, v36, s6
+; GFX10-NEXT: v_bfe_u32 v36, v3, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s8, v19, v19
+; GFX10-NEXT: v_add3_u32 v48, v48, v19, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v19, v19, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s9, v2, v2
+; GFX10-NEXT: v_add3_u32 v52, v52, v2, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v2, v2, s23, 0x400000
+; GFX10-NEXT: v_perm_b32 v0, v0, v17, 0x7060302
+; GFX10-NEXT: v_perm_b32 v1, v1, v18, 0x7060302
+; GFX10-NEXT: v_and_or_b32 v34, v9, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v50, v25, s23, 0x400000
+; GFX10-NEXT: v_bfe_u32 v53, v8, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s14, v9, v9
+; GFX10-NEXT: v_add3_u32 v9, v35, v9, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v35, v7, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s15, v25, v25
+; GFX10-NEXT: v_add3_u32 v25, v38, v25, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v38, v23, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s18, v7, v7
+; GFX10-NEXT: v_add3_u32 v7, v51, v7, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v51, v6, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s20, v6, v6
+; GFX10-NEXT: v_add3_u32 v6, v65, v6, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v65, v5, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v21, v21
+; GFX10-NEXT: v_add3_u32 v26, v26, v21, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v21, v21, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s5, v4, v4
+; GFX10-NEXT: v_add3_u32 v28, v28, v4, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v4, v4, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s6, v20, v20
+; GFX10-NEXT: v_add3_u32 v30, v30, v20, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v20, v20, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s7, v3, v3
+; GFX10-NEXT: v_add3_u32 v36, v36, v3, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v3, v3, s23, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e64 v19, v48, v19, s8
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v52, v2, s9
+; GFX10-NEXT: v_and_or_b32 v55, v8, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s16, v8, v8
+; GFX10-NEXT: v_add3_u32 v8, v53, v8, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v53, v23, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s19, v23, v23
+; GFX10-NEXT: v_add3_u32 v23, v38, v23, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v38, v22, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v65, v65, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v5, v5, s23, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e64 v21, v26, v21, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v28, v4, s5
+; GFX10-NEXT: v_cndmask_b32_e64 v20, v30, v20, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v36, v3, s7
+; GFX10-NEXT: v_perm_b32 v2, v2, v19, 0x7060302
+; GFX10-NEXT: v_cmp_u_f32_e64 s21, v22, v22
+; GFX10-NEXT: v_add3_u32 v38, v38, v22, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v22, v22, s23, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v65, v5, vcc_lo
+; GFX10-NEXT: v_perm_b32 v3, v3, v20, 0x7060302
; GFX10-NEXT: v_perm_b32 v4, v4, v21, 0x7060302
-; GFX10-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
-; GFX10-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
-; GFX10-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e64 v27, v33, v27, s13
+; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v34, s14
+; GFX10-NEXT: v_cndmask_b32_e64 v25, v25, v50, s15
+; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, v55, s16
+; GFX10-NEXT: v_cndmask_b32_e64 v24, v24, v67, s17
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v35, s18
+; GFX10-NEXT: v_cndmask_b32_e64 v23, v23, v53, s19
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v51, s20
+; GFX10-NEXT: v_cndmask_b32_e64 v22, v38, v22, s21
; GFX10-NEXT: v_perm_b32 v8, v8, v25, 0x7060302
-; GFX10-NEXT: v_perm_b32 v9, v9, v51, 0x7060302
+; GFX10-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
+; GFX10-NEXT: v_perm_b32 v9, v9, v27, 0x7060302
+; GFX10-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
+; GFX10-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
; GFX10-NEXT: v_perm_b32 v10, v10, v49, 0x7060302
; GFX10-NEXT: v_perm_b32 v11, v11, v39, 0x7060302
; GFX10-NEXT: v_perm_b32 v12, v12, v37, 0x7060302
-; GFX10-NEXT: v_perm_b32 v13, v13, v35, 0x7060302
-; GFX10-NEXT: v_perm_b32 v14, v14, v33, 0x7060302
+; GFX10-NEXT: v_perm_b32 v13, v13, v29, 0x7060302
+; GFX10-NEXT: v_perm_b32 v14, v14, v16, 0x7060302
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v31
-; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v31
-; GFX10-NEXT: v_min_f32_e32 v16, v32, v16
-; GFX10-NEXT: v_min_f32_e32 v15, v15, v17
-; GFX10-NEXT: v_perm_b32 v15, v15, v16, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v31
+; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v31
+; GFX10-NEXT: v_min_f32_e32 v17, v32, v17
+; GFX10-NEXT: v_min_f32_e32 v15, v15, v18
+; GFX10-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX10-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX10-NEXT: v_and_or_b32 v20, v17, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v21, v15, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v15, v15
+; GFX10-NEXT: v_add3_u32 v17, v18, v17, 0x7fff
+; GFX10-NEXT: v_add3_u32 v15, v19, v15, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v17, v17, v20, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v15, v15, v21, s4
+; GFX10-NEXT: v_perm_b32 v15, v15, v17, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_minnum_v32bf16:
@@ -15059,102 +21277,269 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX11-NEXT: v_lshlrev_b32_e32 v84, 16, v1
; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v24
+; GFX11-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_dual_min_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v64, 16, v7
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v18
; GFX11-NEXT: v_lshlrev_b32_e32 v85, 16, v16
; GFX11-NEXT: v_lshlrev_b32_e32 v86, 16, v0
+; GFX11-NEXT: v_bfe_u32 v135, v1, 16, 1
; GFX11-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v23
+; GFX11-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; GFX11-NEXT: v_and_or_b32 v144, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v54, 16, v8
-; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v7
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: v_dual_min_f32 v17, v86, v85 :: v_dual_and_b32 v8, 0xffff0000, v8
+; GFX11-NEXT: v_dual_min_f32 v7, v7, v23 :: v_dual_lshlrev_b32 v36, 16, v13
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_min_f32 v8, v8, v24 :: v_dual_lshlrev_b32 v39, 16, v27
+; GFX11-NEXT: v_dual_min_f32 v0, v0, v16 :: v_dual_lshlrev_b32 v49, 16, v26
+; GFX11-NEXT: v_min_f32_e32 v24, v64, v55
+; GFX11-NEXT: v_bfe_u32 v87, v7, 16, 1
; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v22
; GFX11-NEXT: v_lshlrev_b32_e32 v66, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v11
-; GFX11-NEXT: v_dual_min_f32 v0, v0, v16 :: v_dual_and_b32 v11, 0xffff0000, v11
; GFX11-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX11-NEXT: v_bfe_u32 v85, v24, 16, 1
; GFX11-NEXT: v_lshlrev_b32_e32 v67, 16, v21
; GFX11-NEXT: v_lshlrev_b32_e32 v68, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v51, 16, v25
; GFX11-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v20
; GFX11-NEXT: v_lshlrev_b32_e32 v70, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v23
-; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v19
-; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v3
-; GFX11-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v9
-; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v18
-; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v2
+; GFX11-NEXT: v_and_or_b32 v86, v24, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v96, v7, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v85, v85, v24, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v20
+; GFX11-NEXT: v_add3_u32 v87, v87, v7, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX11-NEXT: v_dual_min_f32 v23, v66, v65 :: v_dual_lshlrev_b32 v48, 16, v11
+; GFX11-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; GFX11-NEXT: v_dual_min_f32 v5, v5, v21 :: v_dual_lshlrev_b32 v50, 16, v10
+; GFX11-NEXT: v_dual_min_f32 v21, v70, v69 :: v_dual_and_b32 v26, 0xffff0000, v26
; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v24
-; GFX11-NEXT: v_dual_min_f32 v1, v1, v17 :: v_dual_and_b32 v24, 0xffff0000, v24
-; GFX11-NEXT: v_dual_min_f32 v5, v5, v21 :: v_dual_lshlrev_b32 v50, 16, v10
-; GFX11-NEXT: v_dual_min_f32 v21, v70, v69 :: v_dual_and_b32 v10, 0xffff0000, v10
-; GFX11-NEXT: v_dual_min_f32 v2, v2, v18 :: v_dual_min_f32 v3, v3, v19
-; GFX11-NEXT: v_dual_min_f32 v4, v4, v20 :: v_dual_lshlrev_b32 v49, 16, v26
-; GFX11-NEXT: v_dual_min_f32 v9, v9, v25 :: v_dual_and_b32 v26, 0xffff0000, v26
; GFX11-NEXT: v_min_f32_e32 v6, v6, v22
-; GFX11-NEXT: v_dual_min_f32 v22, v68, v67 :: v_dual_lshlrev_b32 v37, 16, v28
-; GFX11-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_min_f32_e32 v10, v10, v26
-; GFX11-NEXT: v_min_f32_e32 v26, v52, v51
-; GFX11-NEXT: v_perm_b32 v4, v4, v21, 0x7060302
-; GFX11-NEXT: v_min_f32_e32 v25, v54, v53
-; GFX11-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
-; GFX11-NEXT: v_perm_b32 v9, v9, v26, 0x7060302
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v31
-; GFX11-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v31
-; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-NEXT: v_lshlrev_b32_e32 v36, 16, v13
-; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX11-NEXT: v_lshlrev_b32_e32 v39, 16, v27
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_dual_min_f32 v8, v8, v24 :: v_dual_and_b32 v27, 0xffff0000, v27
-; GFX11-NEXT: v_min_f32_e32 v24, v64, v55
+; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v9
+; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; GFX11-NEXT: v_bfe_u32 v97, v23, 16, 1
+; GFX11-NEXT: v_min_f32_e32 v2, v2, v18
+; GFX11-NEXT: v_min_f32_e32 v18, v84, v83
+; GFX11-NEXT: v_bfe_u32 v83, v8, 16, 1
+; GFX11-NEXT: v_bfe_u32 v99, v6, 16, 1
+; GFX11-NEXT: v_bfe_u32 v103, v5, 16, 1
+; GFX11-NEXT: v_bfe_u32 v113, v21, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v19
+; GFX11-NEXT: v_and_or_b32 v84, v8, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v98, v23, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v100, v6, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v112, v5, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v114, v21, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v83, v83, v8, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; GFX11-NEXT: v_add3_u32 v97, v97, v23, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; GFX11-NEXT: v_add3_u32 v99, v99, v6, 0x7fff
+; GFX11-NEXT: v_add3_u32 v103, v103, v5, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-NEXT: v_add3_u32 v113, v113, v21, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v12
-; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX11-NEXT: v_dual_min_f32 v3, v3, v19 :: v_dual_and_b32 v10, 0xffff0000, v10
+; GFX11-NEXT: v_dual_min_f32 v22, v68, v67 :: v_dual_lshlrev_b32 v51, 16, v25
+; GFX11-NEXT: v_lshlrev_b32_e32 v37, 16, v28
+; GFX11-NEXT: v_dual_min_f32 v4, v4, v20 :: v_dual_and_b32 v25, 0xffff0000, v25
+; GFX11-NEXT: v_min_f32_e32 v20, v80, v71
+; GFX11-NEXT: v_dual_min_f32 v19, v82, v81 :: v_dual_and_b32 v28, 0xffff0000, v28
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_dual_min_f32 v9, v9, v25 :: v_dual_and_b32 v12, 0xffff0000, v12
+; GFX11-NEXT: v_min_f32_e32 v25, v54, v53
; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v29
-; GFX11-NEXT: v_min_f32_e32 v7, v7, v23
-; GFX11-NEXT: v_min_f32_e32 v23, v66, v65
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_min_f32 v12, v12, v28 :: v_dual_and_b32 v29, 0xffff0000, v29
-; GFX11-NEXT: v_dual_min_f32 v28, v48, v39 :: v_dual_lshlrev_b32 v33, 16, v30
+; GFX11-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX11-NEXT: v_dual_min_f32 v10, v10, v26 :: v_dual_and_b32 v13, 0xffff0000, v13
+; GFX11-NEXT: v_dual_min_f32 v12, v12, v28 :: v_dual_lshlrev_b32 v33, 16, v30
+; GFX11-NEXT: v_min_f32_e32 v28, v48, v39
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_dual_min_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v34, 16, v14
-; GFX11-NEXT: v_lshlrev_b32_e32 v32, 16, v15
; GFX11-NEXT: v_dual_min_f32 v11, v11, v27 :: v_dual_and_b32 v14, 0xffff0000, v14
-; GFX11-NEXT: v_dual_min_f32 v27, v50, v49 :: v_dual_and_b32 v30, 0xffff0000, v30
-; GFX11-NEXT: v_min_f32_e32 v29, v38, v37
+; GFX11-NEXT: v_dual_min_f32 v27, v50, v49 :: v_dual_min_f32 v26, v52, v51
+; GFX11-NEXT: v_dual_min_f32 v29, v38, v37 :: v_dual_and_b32 v30, 0xffff0000, v30
+; GFX11-NEXT: v_lshlrev_b32_e32 v32, 16, v15
; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX11-NEXT: v_min_f32_e32 v37, v86, v85
-; GFX11-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v39, v13, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_min_f32_e32 v14, v14, v30
; GFX11-NEXT: v_dual_min_f32 v30, v36, v35 :: v_dual_min_f32 v33, v34, v33
-; GFX11-NEXT: v_dual_min_f32 v34, v80, v71 :: v_dual_min_f32 v35, v82, v81
-; GFX11-NEXT: v_min_f32_e32 v36, v84, v83
-; GFX11-NEXT: v_dual_min_f32 v16, v32, v16 :: v_dual_min_f32 v15, v15, v17
-; GFX11-NEXT: v_perm_b32 v0, v0, v37, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v2, v2, v35, 0x7060302
-; GFX11-NEXT: v_perm_b32 v1, v1, v36, 0x7060302
-; GFX11-NEXT: v_perm_b32 v3, v3, v34, 0x7060302
-; GFX11-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
-; GFX11-NEXT: v_perm_b32 v8, v8, v25, 0x7060302
+; GFX11-NEXT: v_and_or_b32 v48, v13, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v49, v29, 16, 1
+; GFX11-NEXT: v_bfe_u32 v35, v14, 16, 1
+; GFX11-NEXT: v_and_or_b32 v36, v14, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v16, v33, 16, 1
+; GFX11-NEXT: v_and_or_b32 v34, v33, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX11-NEXT: v_bfe_u32 v37, v30, 16, 1
+; GFX11-NEXT: v_add3_u32 v35, v35, v14, 0x7fff
+; GFX11-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v38, v30, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v39, v39, v13, 0x7fff
+; GFX11-NEXT: v_add3_u32 v37, v37, v30, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v50, v29, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v16, v16, v34, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-NEXT: v_bfe_u32 v51, v12, 16, 1
+; GFX11-NEXT: v_add3_u32 v49, v49, v29, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v52, v12, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v53, v28, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v14, v35, v36, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
+; GFX11-NEXT: v_add3_u32 v51, v51, v12, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v54, v28, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v55, v11, 16, 1
+; GFX11-NEXT: v_add3_u32 v53, v53, v28, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v30, v37, v38, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-NEXT: v_and_or_b32 v64, v11, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v65, v27, 16, 1
+; GFX11-NEXT: v_add3_u32 v55, v55, v11, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v66, v27, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v13, v39, v48, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-NEXT: v_bfe_u32 v67, v10, 16, 1
+; GFX11-NEXT: v_add3_u32 v65, v65, v27, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v68, v10, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v69, v26, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v29, v49, v50, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-NEXT: v_add3_u32 v67, v67, v10, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v70, v26, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v71, v9, 16, 1
+; GFX11-NEXT: v_add3_u32 v69, v69, v26, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v12, v51, v52, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX11-NEXT: v_and_or_b32 v80, v9, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v81, v25, 16, 1
+; GFX11-NEXT: v_add3_u32 v71, v71, v9, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v82, v25, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v28, v53, v54, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-NEXT: v_add3_u32 v81, v81, v25, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v101, v22, 16, 1
+; GFX11-NEXT: v_and_or_b32 v102, v22, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v115, v4, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v11, v55, v64, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX11-NEXT: v_add3_u32 v101, v101, v22, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v116, v4, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v117, v20, 16, 1
+; GFX11-NEXT: v_add3_u32 v115, v115, v4, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v27, v65, v66, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-NEXT: v_and_or_b32 v118, v20, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v129, v19, 16, 1
+; GFX11-NEXT: v_add3_u32 v117, v117, v20, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v130, v19, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v10, v67, v68, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-NEXT: v_bfe_u32 v133, v18, 16, 1
+; GFX11-NEXT: v_add3_u32 v129, v129, v19, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v134, v18, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v145, v17, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v26, v69, v70, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-NEXT: v_add3_u32 v133, v133, v18, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v146, v17, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v147, v0, 16, 1
+; GFX11-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v9, v71, v80, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-NEXT: v_bfe_u32 v131, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v33, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v147, v147, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v132, v2, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v25, v81, v82, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-NEXT: v_add3_u32 v131, v131, v2, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v119, v3, 16, 1
+; GFX11-NEXT: v_and_or_b32 v128, v3, s0, 0x400000
+; GFX11-NEXT: v_perm_b32 v9, v9, v26, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v8, v83, v84, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-NEXT: v_add3_u32 v119, v119, v3, 0x7fff
; GFX11-NEXT: v_perm_b32 v10, v10, v27, 0x7060302
; GFX11-NEXT: v_perm_b32 v11, v11, v28, 0x7060302
+; GFX11-NEXT: v_perm_b32 v8, v8, v25, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v24, v85, v86, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11-NEXT: v_perm_b32 v12, v12, v29, 0x7060302
; GFX11-NEXT: v_perm_b32 v13, v13, v30, 0x7060302
-; GFX11-NEXT: v_perm_b32 v14, v14, v33, 0x7060302
-; GFX11-NEXT: v_perm_b32 v15, v15, v16, 0x7060302
+; GFX11-NEXT: v_perm_b32 v14, v14, v16, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v87, v96, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v23, v97, v98, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: v_cndmask_b32_e32 v6, v99, v100, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v22, v101, v102, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v103, v112, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v21, v113, v114, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v115, v116, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-NEXT: v_perm_b32 v4, v4, v21, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v20, v117, v118, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-NEXT: v_cndmask_b32_e32 v19, v129, v130, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-NEXT: v_cndmask_b32_e32 v18, v133, v134, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v135, v144, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, v1, v18, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v17, v145, v146, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v147, v33, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_perm_b32 v0, v0, v17, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v131, v132, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v119, v128, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_perm_b32 v3, v3, v20, 0x7060302
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v31
+; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v31
+; GFX11-NEXT: v_perm_b32 v2, v2, v19, 0x7060302
+; GFX11-NEXT: v_min_f32_e32 v17, v32, v17
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_min_f32_e32 v15, v15, v18
+; GFX11-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11-NEXT: v_and_or_b32 v20, v17, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: v_and_or_b32 v21, v15, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v18, v18, v17, 0x7fff
+; GFX11-NEXT: v_add3_u32 v19, v19, v15, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v17, v18, v20, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-NEXT: v_cndmask_b32_e32 v15, v19, v21, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v15, v15, v17, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <32 x bfloat> @llvm.minnum.v32bf16(<32 x bfloat> %a, <32 x bfloat> %b)
ret <32 x bfloat> %op
@@ -15173,6 +21558,8 @@ define bfloat @v_maxnum_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_maxnum_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
@@ -15184,6 +21571,8 @@ define bfloat @v_maxnum_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_maxnum_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
@@ -15198,6 +21587,13 @@ define bfloat @v_maxnum_bf16(bfloat %a, bfloat %b) {
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -15207,6 +21603,13 @@ define bfloat @v_maxnum_bf16(bfloat %a, bfloat %b) {
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -15215,7 +21618,13 @@ define bfloat @v_maxnum_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -15224,8 +21633,16 @@ define bfloat @v_maxnum_bf16(bfloat %a, bfloat %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
@@ -15236,6 +21653,10 @@ define <2 x bfloat> @v_maxnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_maxnum_v2bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -15253,6 +21674,10 @@ define <2 x bfloat> @v_maxnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX7-LABEL: v_maxnum_v2bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -15272,10 +21697,24 @@ define <2 x bfloat> @v_maxnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX8-NEXT: v_max_f32_e32 v2, v3, v2
+; GFX8-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v2
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v2
; GFX8-NEXT: v_max_f32_e32 v0, v0, v1
-; GFX8-NEXT: v_max_f32_e32 v2, v3, v2
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v2, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -15285,10 +21724,23 @@ define <2 x bfloat> @v_maxnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_max_f32_e32 v2, v3, v2
+; GFX9-NEXT: v_add3_u32 v3, v3, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v2, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -15300,9 +21752,20 @@ define <2 x bfloat> @v_maxnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_max_f32_e32 v2, v3, v2
; GFX10-NEXT: v_max_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_and_or_b32 v5, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_maxnum_v2bf16:
@@ -15312,11 +21775,24 @@ define <2 x bfloat> @v_maxnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_max_f32_e32 v0, v0, v1
; GFX11-NEXT: v_max_f32_e32 v2, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v2, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_and_or_b32 v5, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
ret <2 x bfloat> %op
@@ -15326,6 +21802,12 @@ define <3 x bfloat> @v_maxnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GCN-LABEL: v_maxnum_v3bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
@@ -15349,6 +21831,12 @@ define <3 x bfloat> @v_maxnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX7-LABEL: v_maxnum_v3bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
@@ -15375,12 +21863,34 @@ define <3 x bfloat> @v_maxnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v3
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX8-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
@@ -15392,12 +21902,31 @@ define <3 x bfloat> @v_maxnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX9-NEXT: v_add3_u32 v4, v4, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16
@@ -15406,16 +21935,32 @@ define <3 x bfloat> @v_maxnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX10-LABEL: v_maxnum_v3bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_max_f32_e32 v4, v5, v4
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_max_f32_e32 v0, v0, v2
; GFX10-NEXT: v_max_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
+; GFX10-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v4, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v6, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = call <3 x bfloat> @llvm.maxnum.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b)
@@ -15426,6 +21971,14 @@ define <4 x bfloat> @v_maxnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GCN-LABEL: v_maxnum_v4bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
@@ -15455,6 +22008,14 @@ define <4 x bfloat> @v_maxnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX7-LABEL: v_maxnum_v4bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
@@ -15486,17 +22047,46 @@ define <4 x bfloat> @v_maxnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX8-NEXT: v_max_f32_e32 v4, v5, v4
+; GFX8-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v4
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_max_f32_e32 v4, v5, v4
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX8-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX8-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v3
; GFX8-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_max_f32_e32 v3, v5, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16
@@ -15507,16 +22097,41 @@ define <4 x bfloat> @v_maxnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX9-NEXT: v_max_f32_e32 v4, v5, v4
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_max_f32_e32 v4, v5, v4
+; GFX9-NEXT: v_add3_u32 v5, v5, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; GFX9-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4
@@ -15528,17 +22143,38 @@ define <4 x bfloat> @v_maxnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX10-NEXT: v_max_f32_e32 v4, v5, v4
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_max_f32_e32 v4, v5, v4
+; GFX10-NEXT: v_max_f32_e32 v1, v1, v3
; GFX10-NEXT: v_max_f32_e32 v5, v7, v6
+; GFX10-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX10-NEXT: v_max_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
-; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX10-NEXT: v_and_or_b32 v6, v4, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v9, v1, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v6, v5, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc_lo
+; GFX10-NEXT: v_perm_b32 v1, v1, v3, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_maxnum_v4bf16:
@@ -15546,19 +22182,45 @@ define <4 x bfloat> @v_maxnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_max_f32 v0, v0, v2 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_dual_max_f32 v4, v5, v4 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_max_f32 v0, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-NEXT: v_max_f32_e32 v4, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v3, v4, 16, 1
; GFX11-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX11-NEXT: v_and_or_b32 v6, v4, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_and_or_b32 v9, v1, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v6, v5, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v8, v0, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v9, vcc_lo
+; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
+; GFX11-NEXT: v_perm_b32 v1, v1, v3, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <4 x bfloat> @llvm.maxnum.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b)
ret <4 x bfloat> %op
@@ -15568,6 +22230,22 @@ define <8 x bfloat> @v_maxnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GCN-LABEL: v_maxnum_v8bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
@@ -15621,6 +22299,22 @@ define <8 x bfloat> @v_maxnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX7-LABEL: v_maxnum_v8bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
@@ -15676,31 +22370,88 @@ define <8 x bfloat> @v_maxnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX8-NEXT: v_max_f32_e32 v8, v9, v8
+; GFX8-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v8
; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_max_f32_e32 v8, v9, v8
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v8
; GFX8-NEXT: v_max_f32_e32 v3, v3, v7
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, s4, v7
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX8-NEXT: v_max_f32_e32 v7, v9, v7
+; GFX8-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v7
; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: v_max_f32_e32 v7, v9, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v7
; GFX8-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, s4, v6
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v6, v9, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v5
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX8-NEXT: v_max_f32_e32 v6, v9, v6
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_max_f32_e32 v6, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v6
; GFX8-NEXT: v_max_f32_e32 v1, v1, v5
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX8-NEXT: v_max_f32_e32 v5, v9, v5
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v5
; GFX8-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_and_b32_e32 v9, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v9, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_max_f32_e32 v5, v9, v5
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v5, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v6, 16
@@ -15713,28 +22464,77 @@ define <8 x bfloat> @v_maxnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX9-NEXT: v_max_f32_e32 v8, v9, v8
+; GFX9-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v8
; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_max_f32_e32 v8, v9, v8
+; GFX9-NEXT: v_add3_u32 v9, v9, v8, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX9-NEXT: v_max_f32_e32 v3, v3, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v7, v7, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX9-NEXT: v_max_f32_e32 v7, v9, v7
+; GFX9-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v7
; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_max_f32_e32 v7, v9, v7
+; GFX9-NEXT: v_add3_u32 v9, v9, v7, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX9-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v6, v6, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v9, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v5
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX9-NEXT: v_max_f32_e32 v6, v9, v6
+; GFX9-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v6
; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_max_f32_e32 v6, v9, v6
+; GFX9-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX9-NEXT: v_max_f32_e32 v1, v1, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v5, v5, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v5, v9, v5
+; GFX9-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v5
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_max_f32_e32 v5, v9, v5
+; GFX9-NEXT: v_add3_u32 v9, v9, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX9-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v9, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v5, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v6, s4
@@ -15747,65 +22547,157 @@ define <8 x bfloat> @v_maxnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v6
-; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_max_f32_e32 v8, v9, v8
+; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v6
+; GFX10-NEXT: v_max_f32_e32 v3, v3, v7
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX10-NEXT: v_max_f32_e32 v9, v11, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v8, s4, 0x400000
+; GFX10-NEXT: v_max_f32_e32 v9, v11, v9
+; GFX10-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX10-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX10-NEXT: v_bfe_u32 v8, v9, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v5
+; GFX10-NEXT: v_and_or_b32 v12, v9, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX10-NEXT: v_add3_u32 v10, v11, v3, 0x7fff
; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX10-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX10-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v0
+; GFX10-NEXT: v_max_f32_e32 v6, v11, v6
+; GFX10-NEXT: v_add3_u32 v9, v13, v2, 0x7fff
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v11, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v12, v6, 16, 1
; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_max_f32_e32 v10, v11, v10
-; GFX10-NEXT: v_max_f32_e32 v11, v13, v12
-; GFX10-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX10-NEXT: v_max_f32_e32 v1, v1, v5
-; GFX10-NEXT: v_max_f32_e32 v2, v2, v6
-; GFX10-NEXT: v_max_f32_e32 v3, v3, v7
-; GFX10-NEXT: v_perm_b32 v0, v0, v11, 0x7060302
-; GFX10-NEXT: v_perm_b32 v1, v1, v10, 0x7060302
-; GFX10-NEXT: v_perm_b32 v2, v2, v9, 0x7060302
-; GFX10-NEXT: v_perm_b32 v3, v3, v8, 0x7060302
+; GFX10-NEXT: v_max_f32_e32 v5, v15, v13
+; GFX10-NEXT: v_and_or_b32 v14, v3, s4, 0x400000
+; GFX10-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v9, v11, vcc_lo
+; GFX10-NEXT: v_add3_u32 v4, v12, v6, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v9, v6, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GFX10-NEXT: v_bfe_u32 v12, v5, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_bfe_u32 v13, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v15, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v6, v11, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v11, v5, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc_lo
+; GFX10-NEXT: v_add3_u32 v9, v12, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v13, v0, s4, 0x400000
+; GFX10-NEXT: v_perm_b32 v2, v2, v8, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v6, v15, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v10, v14, vcc_lo
+; GFX10-NEXT: v_perm_b32 v3, v3, v7, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_maxnum_v8bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-NEXT: v_dual_max_f32 v8, v9, v8 :: v_dual_max_f32 v9, v11, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_dual_max_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v9, 16, v6
+; GFX11-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_max_f32_e32 v9, v11, v9
+; GFX11-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v12, v9, s0, 0x400000
+; GFX11-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v5
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX11-NEXT: v_max_f32_e32 v3, v3, v7
+; GFX11-NEXT: v_and_or_b32 v7, v8, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v8, v9, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc_lo
+; GFX11-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-NEXT: v_add3_u32 v9, v13, v2, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_and_or_b32 v14, v3, s0, 0x400000
; GFX11-NEXT: v_max_f32_e32 v0, v0, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-NEXT: v_add3_u32 v10, v11, v3, 0x7fff
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_max_f32 v6, v11, v6 :: v_dual_and_b32 v1, 0xffff0000, v1
+; GFX11-NEXT: v_and_or_b32 v11, v2, s0, 0x400000
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_dual_max_f32 v1, v1, v5 :: v_dual_and_b32 v6, 0xffff0000, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_max_f32 v2, v2, v6 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11-NEXT: v_max_f32_e32 v3, v3, v7
-; GFX11-NEXT: v_dual_max_f32 v10, v11, v10 :: v_dual_max_f32 v11, v13, v12
+; GFX11-NEXT: v_bfe_u32 v12, v6, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v9, v11, vcc_lo
+; GFX11-NEXT: v_and_or_b32 v9, v6, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: v_max_f32_e32 v1, v1, v5
+; GFX11-NEXT: v_add3_u32 v4, v12, v6, 0x7fff
+; GFX11-NEXT: v_perm_b32 v2, v2, v8, 0x7060302
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_perm_b32 v2, v2, v9, 0x7060302
-; GFX11-NEXT: v_perm_b32 v3, v3, v8, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v1, v1, v10, 0x7060302
-; GFX11-NEXT: v_perm_b32 v0, v0, v11, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc_lo
+; GFX11-NEXT: v_max_f32_e32 v5, v15, v13
+; GFX11-NEXT: v_bfe_u32 v13, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v15, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v6, v11, v1, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v12, v5, 16, 1
+; GFX11-NEXT: v_and_or_b32 v11, v5, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_add3_u32 v9, v12, v5, 0x7fff
+; GFX11-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v13, v0, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v6, v15, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v10, v14, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v3, v3, v7, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <8 x bfloat> @llvm.maxnum.v8bf16(<8 x bfloat> %a, <8 x bfloat> %b)
ret <8 x bfloat> %op
@@ -15815,56 +22707,87 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GCN-LABEL: v_maxnum_v16bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GCN-NEXT: v_max_f32_e32 v14, v14, v30
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GCN-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GCN-NEXT: v_max_f32_e32 v13, v13, v29
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GCN-NEXT: v_max_f32_e32 v12, v12, v28
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GCN-NEXT: v_max_f32_e32 v11, v11, v27
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GCN-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GCN-NEXT: v_max_f32_e32 v10, v10, v26
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GCN-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GCN-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GCN-NEXT: v_max_f32_e32 v9, v9, v25
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GCN-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: v_max_f32_e32 v8, v8, v24
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GCN-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: v_max_f32_e32 v7, v7, v23
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GCN-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GCN-NEXT: v_max_f32_e32 v6, v6, v22
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GCN-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_max_f32_e32 v5, v5, v21
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
@@ -15893,6 +22816,8 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GCN-NEXT: v_max_f32_e32 v2, v2, v18
; GCN-NEXT: v_max_f32_e32 v1, v1, v17
; GCN-NEXT: v_max_f32_e32 v0, v0, v16
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v20
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -15907,8 +22832,7 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v20
+; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GCN-NEXT: v_max_f32_e32 v15, v15, v16
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
@@ -15918,12 +22842,43 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-LABEL: v_maxnum_v16bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_max_f32_e32 v9, v9, v25
+; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
-; GFX7-NEXT: v_max_f32_e32 v6, v6, v22
-; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
@@ -15934,13 +22889,13 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
@@ -15963,17 +22918,21 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
-; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
-; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
-; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
@@ -15987,9 +22946,10 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_max_f32_e32 v12, v12, v28
; GFX7-NEXT: v_max_f32_e32 v11, v11, v27
; GFX7-NEXT: v_max_f32_e32 v10, v10, v26
-; GFX7-NEXT: v_max_f32_e32 v9, v9, v25
+; GFX7-NEXT: v_max_f32_e32 v15, v15, v25
; GFX7-NEXT: v_max_f32_e32 v8, v8, v24
; GFX7-NEXT: v_max_f32_e32 v7, v7, v23
+; GFX7-NEXT: v_max_f32_e32 v6, v6, v22
; GFX7-NEXT: v_max_f32_e32 v5, v5, v21
; GFX7-NEXT: v_max_f32_e32 v4, v4, v20
; GFX7-NEXT: v_max_f32_e32 v3, v3, v19
@@ -16003,10 +22963,6 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
-; GFX7-NEXT: v_max_f32_e32 v15, v15, v22
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
@@ -16023,51 +22979,165 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v15
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v7
+; GFX8-NEXT: v_max_f32_e32 v16, v17, v16
+; GFX8-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX8-NEXT: v_max_f32_e32 v16, v17, v16
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v16
; GFX8-NEXT: v_max_f32_e32 v7, v7, v15
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
+; GFX8-NEXT: v_bfe_u32 v15, v7, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, v15, v7
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, s4, v15
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v7
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v15, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX8-NEXT: v_max_f32_e32 v15, v17, v15
+; GFX8-NEXT: v_bfe_u32 v17, v15, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v15
; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX8-NEXT: v_max_f32_e32 v15, v17, v15
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v15
; GFX8-NEXT: v_max_f32_e32 v6, v6, v14
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v15, v15
+; GFX8-NEXT: v_bfe_u32 v14, v6, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v14, vcc, v14, v6
+; GFX8-NEXT: v_add_u32_e32 v14, vcc, s4, v14
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v6
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v13
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX8-NEXT: v_max_f32_e32 v14, v17, v14
+; GFX8-NEXT: v_bfe_u32 v17, v14, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v14
; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX8-NEXT: v_max_f32_e32 v14, v17, v14
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v14
; GFX8-NEXT: v_max_f32_e32 v5, v5, v13
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
+; GFX8-NEXT: v_bfe_u32 v13, v5, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v14, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v13, vcc, v13, v5
+; GFX8-NEXT: v_add_u32_e32 v13, vcc, s4, v13
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v5
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v13, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v12
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v4
+; GFX8-NEXT: v_max_f32_e32 v13, v17, v13
+; GFX8-NEXT: v_bfe_u32 v17, v13, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v13
; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v13, v17, v13
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v13
; GFX8-NEXT: v_max_f32_e32 v4, v4, v12
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX8-NEXT: v_bfe_u32 v12, v4, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v13, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v12, vcc, v12, v4
+; GFX8-NEXT: v_add_u32_e32 v12, vcc, s4, v12
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v4
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v12, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v12, 16, v11
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX8-NEXT: v_max_f32_e32 v12, v17, v12
+; GFX8-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v12
; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_max_f32_e32 v12, v17, v12
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v12
; GFX8-NEXT: v_max_f32_e32 v3, v3, v11
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX8-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v11, vcc, v11, v3
+; GFX8-NEXT: v_add_u32_e32 v11, vcc, s4, v11
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v11, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v2
+; GFX8-NEXT: v_max_f32_e32 v11, v17, v11
+; GFX8-NEXT: v_bfe_u32 v17, v11, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v11
; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: v_max_f32_e32 v11, v17, v11
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v11
; GFX8-NEXT: v_max_f32_e32 v2, v2, v10
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
+; GFX8-NEXT: v_bfe_u32 v10, v2, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v11, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, v10, v2
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, s4, v10
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v10, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v9
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; GFX8-NEXT: v_max_f32_e32 v10, v17, v10
+; GFX8-NEXT: v_bfe_u32 v17, v10, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v10
; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_max_f32_e32 v10, v17, v10
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v10
; GFX8-NEXT: v_max_f32_e32 v1, v1, v9
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
+; GFX8-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v10, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v1
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v9
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v17, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v0
+; GFX8-NEXT: v_max_f32_e32 v9, v17, v9
+; GFX8-NEXT: v_bfe_u32 v17, v9, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v9
; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v18, 0x80000000, v9
; GFX8-NEXT: v_max_f32_e32 v0, v0, v8
+; GFX8-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v9, v17, v18, vcc
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, s4, v8
+; GFX8-NEXT: v_and_b32_e32 v17, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v17, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
@@ -16075,7 +23145,6 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_max_f32_e32 v9, v17, v9
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v9, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v10, 16
@@ -16092,52 +23161,149 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v15
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v7
+; GFX9-NEXT: v_max_f32_e32 v16, v17, v16
+; GFX9-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v16
; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX9-NEXT: v_max_f32_e32 v16, v17, v16
+; GFX9-NEXT: v_add3_u32 v17, v17, v16, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v16, v16
; GFX9-NEXT: v_max_f32_e32 v7, v7, v15
+; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v15, v7, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v7
+; GFX9-NEXT: v_add3_u32 v15, v15, v7, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v15, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX9-NEXT: v_max_f32_e32 v15, v17, v15
+; GFX9-NEXT: v_bfe_u32 v17, v15, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v15
; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX9-NEXT: v_max_f32_e32 v15, v17, v15
+; GFX9-NEXT: v_add3_u32 v17, v17, v15, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v15, v15
; GFX9-NEXT: v_max_f32_e32 v6, v6, v14
+; GFX9-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v14, v6, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v6
+; GFX9-NEXT: v_add3_u32 v14, v14, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v14, 16, v13
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX9-NEXT: v_max_f32_e32 v14, v17, v14
+; GFX9-NEXT: v_bfe_u32 v17, v14, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v14
; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX9-NEXT: v_max_f32_e32 v14, v17, v14
+; GFX9-NEXT: v_add3_u32 v17, v17, v14, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
; GFX9-NEXT: v_max_f32_e32 v5, v5, v13
+; GFX9-NEXT: v_cndmask_b32_e32 v14, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v13, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v5
+; GFX9-NEXT: v_add3_u32 v13, v13, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v13, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v13, 16, v12
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v4
+; GFX9-NEXT: v_max_f32_e32 v13, v17, v13
+; GFX9-NEXT: v_bfe_u32 v17, v13, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v13
; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX9-NEXT: v_max_f32_e32 v13, v17, v13
+; GFX9-NEXT: v_add3_u32 v17, v17, v13, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
; GFX9-NEXT: v_max_f32_e32 v4, v4, v12
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v12, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v12, v12, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v12, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v11
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX9-NEXT: v_max_f32_e32 v12, v17, v12
+; GFX9-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v12
; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_max_f32_e32 v12, v17, v12
+; GFX9-NEXT: v_add3_u32 v17, v17, v12, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
; GFX9-NEXT: v_max_f32_e32 v3, v3, v11
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v11, v11, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v11, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v2
+; GFX9-NEXT: v_max_f32_e32 v11, v17, v11
+; GFX9-NEXT: v_bfe_u32 v17, v11, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v11
; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_max_f32_e32 v11, v17, v11
+; GFX9-NEXT: v_add3_u32 v17, v17, v11, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
; GFX9-NEXT: v_max_f32_e32 v2, v2, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v11, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v10, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v10, v10, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v10, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v9
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; GFX9-NEXT: v_max_f32_e32 v10, v17, v10
+; GFX9-NEXT: v_bfe_u32 v17, v10, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v10
; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_max_f32_e32 v10, v17, v10
+; GFX9-NEXT: v_add3_u32 v17, v17, v10, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
; GFX9-NEXT: v_max_f32_e32 v1, v1, v9
+; GFX9-NEXT: v_cndmask_b32_e32 v10, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v9, v9, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v17, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v9, v17, v9
+; GFX9-NEXT: v_bfe_u32 v17, v9, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v18, 0x80000000, v9
; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_max_f32_e32 v9, v17, v9
+; GFX9-NEXT: v_add3_u32 v17, v17, v9, s4
+; GFX9-NEXT: v_or_b32_e32 v18, 0x400000, v18
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
; GFX9-NEXT: v_max_f32_e32 v0, v0, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v17, v18, vcc
+; GFX9-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v17, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v8, v8, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v17, 0x400000, v17
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v8, v17, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v9, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v10, s4
@@ -16156,119 +23322,297 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v7
; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v5
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v6
; GFX10-NEXT: v_max_f32_e32 v16, v17, v16
-; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v14
; GFX10-NEXT: v_max_f32_e32 v7, v7, v15
-; GFX10-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v12
-; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v4
-; GFX10-NEXT: v_max_f32_e32 v15, v17, v15
+; GFX10-NEXT: v_bfe_u32 v15, v16, 16, 1
+; GFX10-NEXT: v_and_or_b32 v20, v16, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v19, v7, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX10-NEXT: v_max_f32_e32 v17, v18, v17
+; GFX10-NEXT: v_add3_u32 v15, v15, v16, 0x7fff
; GFX10-NEXT: v_max_f32_e32 v6, v6, v14
-; GFX10-NEXT: v_max_f32_e32 v14, v19, v18
+; GFX10-NEXT: v_add3_u32 v18, v19, v7, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v19, v7, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v21, v17, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v15, v15, v20, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v5
+; GFX10-NEXT: v_and_or_b32 v16, v17, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v14, v21, v17, 0x7fff
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v7, v18, v19, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX10-NEXT: v_bfe_u32 v18, v6, 16, 1
+; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT: v_perm_b32 v7, v7, v15, 0x7060302
+; GFX10-NEXT: v_max_f32_e32 v17, v20, v19
+; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v4
; GFX10-NEXT: v_max_f32_e32 v5, v5, v13
-; GFX10-NEXT: v_max_f32_e32 v13, v21, v20
-; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v11
-; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v14, v14, v16, vcc_lo
+; GFX10-NEXT: v_add3_u32 v16, v18, v6, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v13, v6, s4, 0x400000
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX10-NEXT: v_bfe_u32 v20, v17, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_bfe_u32 v21, v5, 16, 1
; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v17, v18, v17
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v16, v13, vcc_lo
+; GFX10-NEXT: v_max_f32_e32 v13, v19, v18
+; GFX10-NEXT: v_add3_u32 v16, v20, v17, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v18, v17, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT: v_add3_u32 v19, v21, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v20, v5, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v21, v13, 16, 1
+; GFX10-NEXT: v_max_f32_e32 v4, v4, v12
+; GFX10-NEXT: v_cndmask_b32_e32 v16, v16, v18, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v11
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v3
+; GFX10-NEXT: v_add3_u32 v17, v21, v13, 0x7fff
; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v19, v20, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v19, v13, s4, 0x400000
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT: v_max_f32_e32 v18, v20, v19
+; GFX10-NEXT: v_max_f32_e32 v12, v18, v12
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX10-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v10
+; GFX10-NEXT: v_max_f32_e32 v3, v3, v11
+; GFX10-NEXT: v_and_or_b32 v22, v12, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v13, v17, v19, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v2
+; GFX10-NEXT: v_add3_u32 v11, v20, v4, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v20, v3, 16, 1
; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v9
-; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v1
+; GFX10-NEXT: v_add3_u32 v17, v17, v12, 0x7fff
+; GFX10-NEXT: v_max_f32_e32 v18, v19, v18
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX10-NEXT: v_add3_u32 v19, v20, v3, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v20, v3, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v23, v18, 16, 1
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v10
+; GFX10-NEXT: v_cndmask_b32_e32 v12, v17, v22, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_and_or_b32 v17, v18, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v10, v23, v18, 0x7fff
+; GFX10-NEXT: v_lshlrev_b32_e32 v22, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v19, v20, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v19, v2, 16, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v9
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX10-NEXT: v_and_or_b32 v18, v2, s4, 0x400000
+; GFX10-NEXT: v_and_or_b32 v21, v4, s4, 0x400000
+; GFX10-NEXT: v_perm_b32 v3, v3, v12, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v10, v10, v17, vcc_lo
+; GFX10-NEXT: v_add3_u32 v17, v19, v2, 0x7fff
+; GFX10-NEXT: v_max_f32_e32 v19, v22, v20
+; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v8
; GFX10-NEXT: v_lshlrev_b32_e32 v22, 16, v0
; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_max_f32_e32 v19, v20, v19
-; GFX10-NEXT: v_max_f32_e32 v20, v22, v21
-; GFX10-NEXT: v_max_f32_e32 v0, v0, v8
+; GFX10-NEXT: v_bfe_u32 v23, v19, 16, 1
; GFX10-NEXT: v_max_f32_e32 v1, v1, v9
-; GFX10-NEXT: v_max_f32_e32 v2, v2, v10
-; GFX10-NEXT: v_max_f32_e32 v3, v3, v11
-; GFX10-NEXT: v_max_f32_e32 v4, v4, v12
-; GFX10-NEXT: v_perm_b32 v0, v0, v20, 0x7060302
+; GFX10-NEXT: v_max_f32_e32 v9, v22, v20
+; GFX10-NEXT: v_and_or_b32 v22, v19, s4, 0x400000
+; GFX10-NEXT: v_max_f32_e32 v0, v0, v8
+; GFX10-NEXT: v_add3_u32 v20, v23, v19, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX10-NEXT: v_bfe_u32 v23, v9, 16, 1
+; GFX10-NEXT: v_and_or_b32 v24, v9, s4, 0x400000
+; GFX10-NEXT: v_and_or_b32 v25, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v22, v1, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_bfe_u32 v20, v0, 16, 1
+; GFX10-NEXT: v_add3_u32 v23, v23, v9, 0x7fff
+; GFX10-NEXT: v_perm_b32 v5, v5, v16, 0x7060302
+; GFX10-NEXT: v_perm_b32 v6, v6, v14, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v8, v22, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX10-NEXT: v_add3_u32 v20, v20, v0, 0x7fff
; GFX10-NEXT: v_perm_b32 v1, v1, v19, 0x7060302
-; GFX10-NEXT: v_perm_b32 v2, v2, v18, 0x7060302
-; GFX10-NEXT: v_perm_b32 v3, v3, v17, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v8, v23, v24, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v20, v25, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_perm_b32 v0, v0, v8, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_perm_b32 v2, v2, v10, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v11, v21, vcc_lo
; GFX10-NEXT: v_perm_b32 v4, v4, v13, 0x7060302
-; GFX10-NEXT: v_perm_b32 v5, v5, v14, 0x7060302
-; GFX10-NEXT: v_perm_b32 v6, v6, v15, 0x7060302
-; GFX10-NEXT: v_perm_b32 v7, v7, v16, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_maxnum_v16bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v12
-; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v0
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_max_f32 v4, v4, v12 :: v_dual_and_b32 v5, 0xffff0000, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v15
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v7
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v13
-; GFX11-NEXT: v_max_f32_e32 v13, v21, v20
-; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v8
-; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-NEXT: v_dual_max_f32 v16, v17, v16 :: v_dual_and_b32 v15, 0xffff0000, v15
-; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_max_f32 v0, v0, v8 :: v_dual_and_b32 v7, 0xffff0000, v7
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-NEXT: v_perm_b32 v4, v4, v13, 0x7060302
-; GFX11-NEXT: v_max_f32_e32 v7, v7, v15
-; GFX11-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_max_f32 v16, v17, v16 :: v_dual_lshlrev_b32 v17, 16, v14
; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v7, v7, v16, 0x7060302
-; GFX11-NEXT: v_max_f32_e32 v15, v17, v15
+; GFX11-NEXT: v_and_or_b32 v20, v16, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_max_f32_e32 v17, v18, v17
+; GFX11-NEXT: v_max_f32_e32 v6, v6, v14
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_bfe_u32 v21, v17, 16, 1
+; GFX11-NEXT: v_add3_u32 v14, v21, v17, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v7, v7, v15
+; GFX11-NEXT: v_bfe_u32 v15, v16, 16, 1
+; GFX11-NEXT: v_add3_u32 v15, v15, v16, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v16, v17, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v15, v15, v20 :: v_dual_lshlrev_b32 v20, 16, v5
+; GFX11-NEXT: v_bfe_u32 v19, v7, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-NEXT: v_add3_u32 v18, v19, v7, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v19, v7, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v18, v19, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v18, v6, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: v_perm_b32 v7, v7, v15, 0x7060302
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_dual_max_f32 v6, v6, v14 :: v_dual_lshlrev_b32 v17, 16, v11
-; GFX11-NEXT: v_max_f32_e32 v14, v19, v18
+; GFX11-NEXT: v_dual_max_f32 v17, v20, v19 :: v_dual_cndmask_b32 v14, v14, v16
+; GFX11-NEXT: v_add3_u32 v16, v18, v6, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; GFX11-NEXT: v_bfe_u32 v20, v17, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v4, v4, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v11
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX11-NEXT: v_max_f32_e32 v5, v5, v13
+; GFX11-NEXT: v_and_or_b32 v13, v6, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_cndmask_b32 v6, v16, v13 :: v_dual_max_f32 v13, v19, v18
+; GFX11-NEXT: v_add3_u32 v16, v20, v17, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v18, v17, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: v_perm_b32 v6, v6, v14, 0x7060302
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v16, v16, v18, vcc_lo
; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v10
+; GFX11-NEXT: v_bfe_u32 v21, v5, 16, 1
+; GFX11-NEXT: v_and_or_b32 v20, v5, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_max_f32_e32 v12, v18, v12
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v19, v21, v5, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v21, v13, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v10
+; GFX11-NEXT: v_and_or_b32 v22, v12, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v19, v20, vcc_lo
+; GFX11-NEXT: v_add3_u32 v17, v21, v13, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v19, v13, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX11-NEXT: v_and_or_b32 v21, v4, s0, 0x400000
+; GFX11-NEXT: v_perm_b32 v5, v5, v16, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v13, v17, v19, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v17, v12, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v2
; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-NEXT: v_add3_u32 v17, v17, v12, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_max_f32_e32 v18, v19, v18
+; GFX11-NEXT: v_cndmask_b32_e32 v12, v17, v22, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v1
+; GFX11-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_dual_max_f32 v17, v18, v17 :: v_dual_and_b32 v10, 0xffff0000, v10
-; GFX11-NEXT: v_perm_b32 v5, v5, v14, 0x7060302
-; GFX11-NEXT: v_perm_b32 v6, v6, v15, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v23, v18, 16, 1
+; GFX11-NEXT: v_and_or_b32 v17, v18, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_max_f32 v2, v2, v10 :: v_dual_and_b32 v1, 0xffff0000, v1
; GFX11-NEXT: v_max_f32_e32 v3, v3, v11
-; GFX11-NEXT: v_dual_max_f32 v18, v20, v19 :: v_dual_lshlrev_b32 v19, 16, v9
-; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v1
+; GFX11-NEXT: v_add3_u32 v11, v20, v4, 0x7fff
+; GFX11-NEXT: v_add3_u32 v10, v23, v18, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v20, v3, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v19, v20, v3, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v20, v3, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v19, v20, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v19, v2, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v9
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-NEXT: v_and_or_b32 v18, v2, s0, 0x400000
+; GFX11-NEXT: v_perm_b32 v3, v3, v12, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v10, v10, v17, vcc_lo
+; GFX11-NEXT: v_add3_u32 v17, v19, v2, 0x7fff
+; GFX11-NEXT: v_max_f32_e32 v19, v22, v20
+; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v8
+; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX11-NEXT: v_dual_max_f32 v2, v2, v10 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_perm_b32 v3, v3, v17, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_max_f32 v19, v20, v19 :: v_dual_max_f32 v20, v22, v21
-; GFX11-NEXT: v_max_f32_e32 v1, v1, v9
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_perm_b32 v2, v2, v18, 0x7060302
-; GFX11-NEXT: v_perm_b32 v0, v0, v20, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v23, v19, 16, 1
+; GFX11-NEXT: v_dual_max_f32 v0, v0, v8 :: v_dual_max_f32 v1, v1, v9
+; GFX11-NEXT: v_max_f32_e32 v9, v22, v20
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v20, v23, v19, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v22, v19, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v25, v0, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-NEXT: v_bfe_u32 v23, v9, 16, 1
+; GFX11-NEXT: v_and_or_b32 v24, v9, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v19, v20, v22, vcc_lo
+; GFX11-NEXT: v_and_or_b32 v22, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_bfe_u32 v20, v0, 16, 1
+; GFX11-NEXT: v_add3_u32 v23, v23, v9, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v8, v22, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-NEXT: v_add3_u32 v20, v20, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_perm_b32 v1, v1, v19, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v8, v23, v24, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v20, v25, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v0, v0, v8, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v17, v18, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_perm_b32 v2, v2, v10, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v11, v21, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v4, v4, v13, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <16 x bfloat> @llvm.maxnum.v16bf16(<16 x bfloat> %a, <16 x bfloat> %b)
ret <16 x bfloat> %op
@@ -16278,230 +23622,294 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GCN-LABEL: v_maxnum_v32bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:128
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32
+; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; GCN-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
-; GCN-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GCN-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:124
-; GCN-NEXT: v_max_f32_e32 v31, v32, v31
+; GCN-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GCN-NEXT: v_max_f32_e32 v31, v31, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124
+; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:120
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v30, v30, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120
+; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GCN-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:116
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v29, v29, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116
+; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v28, v28, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112
+; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:108
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v27, v27, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108
+; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:104
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v26, v26, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104
+; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GCN-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:100
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v25, v25, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100
+; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GCN-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:96
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v24, v24, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96
+; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GCN-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:92
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v23, v23, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GCN-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:88
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v22, v22, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88
+; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GCN-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:84
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v21, v21, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GCN-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:80
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v20, v20, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GCN-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:76
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v19, v19, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GCN-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:72
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v18, v18, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GCN-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:68
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v17, v17, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v16, v16, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v15, v15, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:56
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v14, v14, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:52
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v13, v13, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:48
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v12, v12, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:44
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v11, v11, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GCN-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:40
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v10, v10, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GCN-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:36
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v9, v9, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:32
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v8, v8, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:28
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v7, v7, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:24
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v6, v6, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:20
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v5, v5, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:16
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v4, v4, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v3, v3, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v2, v2, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:4
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v1, v1, v32
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v33
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GCN-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GCN-NEXT: v_max_f32_e32 v0, v0, v32
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -16541,260 +23949,324 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX7-LABEL: v_maxnum_v32bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:128
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128
+; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30
+; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29
+; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28
+; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27
; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27
+; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GFX7-NEXT: v_max_f32_e32 v31, v31, v32
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
-; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
-; GFX7-NEXT: v_max_f32_e32 v31, v32, v31
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v30, v30, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:120
; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v29, v29, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:116
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v28, v28, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:112
; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v27, v27, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:108
; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v26, v26, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104
; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v25, v25, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:100
; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v24, v24, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:96
; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v23, v23, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v22, v22, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88
; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v21, v21, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:84
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v20, v20, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80
; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v19, v19, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:76
; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v18, v18, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72
; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v17, v17, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68
; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v16, v16, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64
; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v15, v15, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v14, v14, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v13, v13, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:52
; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v12, v12, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48
; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v11, v11, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:44
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v10, v10, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v9, v9, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:36
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v8, v8, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v7, v7, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:28
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v6, v6, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v5, v5, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:20
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v4, v4, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v3, v3, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:12
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v2, v2, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v1, v1, v32
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_max_f32_e32 v0, v0, v32
@@ -16806,114 +24278,329 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v14
+; GFX8-NEXT: v_max_f32_e32 v31, v32, v31
+; GFX8-NEXT: v_bfe_u32 v32, v31, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v32, vcc, v32, v31
; GFX8-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX8-NEXT: v_max_f32_e32 v31, v32, v31
-; GFX8-NEXT: v_max_f32_e32 v30, v14, v30
-; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v29
+; GFX8-NEXT: v_add_u32_e32 v32, vcc, s4, v32
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v31
+; GFX8-NEXT: v_max_f32_e32 v14, v14, v30
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
+; GFX8-NEXT: v_bfe_u32 v30, v14, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v31, v32, v33, vcc
+; GFX8-NEXT: v_add_u32_e32 v30, vcc, v30, v14
+; GFX8-NEXT: v_add_u32_e32 v30, vcc, s4, v30
+; GFX8-NEXT: v_and_b32_e32 v32, 0x80000000, v14
+; GFX8-NEXT: v_or_b32_e32 v32, 0x400000, v32
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
+; GFX8-NEXT: v_cndmask_b32_e32 v14, v30, v32, vcc
+; GFX8-NEXT: v_lshlrev_b32_e32 v30, 16, v29
; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v13
+; GFX8-NEXT: v_max_f32_e32 v32, v32, v30
+; GFX8-NEXT: buffer_load_dword v30, off, s[0:3], s32
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX8-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX8-NEXT: v_max_f32_e32 v14, v32, v14
; GFX8-NEXT: v_max_f32_e32 v13, v13, v29
+; GFX8-NEXT: v_bfe_u32 v29, v13, 16, 1
+; GFX8-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX8-NEXT: v_alignbit_b32 v14, v14, v31, 16
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v34, 16, v30
+; GFX8-NEXT: v_max_f32_e32 v33, v33, v34
+; GFX8-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; GFX8-NEXT: v_max_f32_e32 v30, v15, v30
+; GFX8-NEXT: v_bfe_u32 v15, v33, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, v15, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v33
+; GFX8-NEXT: v_add_u32_e32 v15, vcc, s4, v15
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v33, v33
+; GFX8-NEXT: v_bfe_u32 v33, v30, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v15, v15, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v30
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v30
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
+; GFX8-NEXT: v_cndmask_b32_e32 v30, v33, v34, vcc
+; GFX8-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v32
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v32
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
+; GFX8-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v29, vcc, v29, v13
+; GFX8-NEXT: v_add_u32_e32 v29, vcc, s4, v29
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v13
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX8-NEXT: v_cndmask_b32_e32 v13, v29, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v29, 16, v28
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v12
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v12
+; GFX8-NEXT: v_max_f32_e32 v29, v33, v29
+; GFX8-NEXT: v_bfe_u32 v33, v29, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v29
; GFX8-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX8-NEXT: v_max_f32_e32 v29, v32, v29
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v29
; GFX8-NEXT: v_max_f32_e32 v12, v12, v28
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v29, v29
+; GFX8-NEXT: v_bfe_u32 v28, v12, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v29, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v28, vcc, v28, v12
+; GFX8-NEXT: v_add_u32_e32 v28, vcc, s4, v28
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v12
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX8-NEXT: v_cndmask_b32_e32 v12, v28, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v28, 16, v27
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v11
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v11
+; GFX8-NEXT: v_max_f32_e32 v28, v33, v28
+; GFX8-NEXT: v_bfe_u32 v33, v28, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v28
; GFX8-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; GFX8-NEXT: v_max_f32_e32 v28, v32, v28
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v28
; GFX8-NEXT: v_max_f32_e32 v11, v11, v27
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v28, v28
+; GFX8-NEXT: v_bfe_u32 v27, v11, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v28, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v27, vcc, v27, v11
+; GFX8-NEXT: v_add_u32_e32 v27, vcc, s4, v27
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v11
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
+; GFX8-NEXT: v_cndmask_b32_e32 v11, v27, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v27, 16, v26
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v10
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v10
+; GFX8-NEXT: v_max_f32_e32 v27, v33, v27
+; GFX8-NEXT: v_bfe_u32 v33, v27, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v27
; GFX8-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX8-NEXT: v_max_f32_e32 v27, v32, v27
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v27
; GFX8-NEXT: v_max_f32_e32 v10, v10, v26
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v27, v27
+; GFX8-NEXT: v_bfe_u32 v26, v10, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v27, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v26, vcc, v26, v10
+; GFX8-NEXT: v_add_u32_e32 v26, vcc, s4, v26
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v10
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
+; GFX8-NEXT: v_cndmask_b32_e32 v10, v26, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v26, 16, v25
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v9
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v9
+; GFX8-NEXT: v_max_f32_e32 v26, v33, v26
+; GFX8-NEXT: v_bfe_u32 v33, v26, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v26
; GFX8-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX8-NEXT: v_max_f32_e32 v26, v32, v26
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v26
; GFX8-NEXT: v_max_f32_e32 v9, v9, v25
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v26, v26
+; GFX8-NEXT: v_bfe_u32 v25, v9, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v26, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v25, vcc, v25, v9
+; GFX8-NEXT: v_add_u32_e32 v25, vcc, s4, v25
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v9
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX8-NEXT: v_cndmask_b32_e32 v9, v25, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v25, 16, v24
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v8
+; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v8
+; GFX8-NEXT: v_max_f32_e32 v25, v33, v25
+; GFX8-NEXT: v_bfe_u32 v33, v25, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v25
; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v25
; GFX8-NEXT: v_max_f32_e32 v8, v8, v24
-; GFX8-NEXT: buffer_load_dword v24, off, s[0:3], s32
-; GFX8-NEXT: v_max_f32_e32 v25, v32, v25
-; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v15
-; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; GFX8-NEXT: v_alignbit_b32 v8, v8, v25, 16
-; GFX8-NEXT: v_alignbit_b32 v9, v9, v26, 16
-; GFX8-NEXT: v_alignbit_b32 v10, v10, v27, 16
-; GFX8-NEXT: v_alignbit_b32 v11, v11, v28, 16
-; GFX8-NEXT: v_alignbit_b32 v12, v12, v29, 16
-; GFX8-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v24
-; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; GFX8-NEXT: v_max_f32_e32 v32, v32, v33
-; GFX8-NEXT: v_max_f32_e32 v15, v15, v24
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v25, v25
+; GFX8-NEXT: v_bfe_u32 v24, v8, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v25, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v24, vcc, v24, v8
+; GFX8-NEXT: v_add_u32_e32 v24, vcc, s4, v24
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v8
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v24, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v24, 16, v23
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v7
+; GFX8-NEXT: v_max_f32_e32 v24, v33, v24
+; GFX8-NEXT: v_bfe_u32 v33, v24, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v24
; GFX8-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX8-NEXT: v_max_f32_e32 v24, v33, v24
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v24
; GFX8-NEXT: v_max_f32_e32 v7, v7, v23
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v24, v24
+; GFX8-NEXT: v_bfe_u32 v23, v7, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v24, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v23, vcc, v23, v7
+; GFX8-NEXT: v_add_u32_e32 v23, vcc, s4, v23
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v7
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v23, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v23, 16, v22
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v6
+; GFX8-NEXT: v_max_f32_e32 v23, v33, v23
+; GFX8-NEXT: v_bfe_u32 v33, v23, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v23
; GFX8-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX8-NEXT: v_max_f32_e32 v23, v33, v23
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v23
; GFX8-NEXT: v_max_f32_e32 v6, v6, v22
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v23, v23
+; GFX8-NEXT: v_bfe_u32 v22, v6, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v23, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v22, vcc, v22, v6
+; GFX8-NEXT: v_add_u32_e32 v22, vcc, s4, v22
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v6
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v22, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v22, 16, v21
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v5
+; GFX8-NEXT: v_max_f32_e32 v22, v33, v22
+; GFX8-NEXT: v_bfe_u32 v33, v22, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v22
; GFX8-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX8-NEXT: v_max_f32_e32 v22, v33, v22
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v22
; GFX8-NEXT: v_max_f32_e32 v5, v5, v21
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v22, v22
+; GFX8-NEXT: v_bfe_u32 v21, v5, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v22, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v21, vcc, v21, v5
+; GFX8-NEXT: v_add_u32_e32 v21, vcc, s4, v21
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v5
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v21, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v21, 16, v20
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v4
+; GFX8-NEXT: v_max_f32_e32 v21, v33, v21
+; GFX8-NEXT: v_bfe_u32 v33, v21, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v21
; GFX8-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v21, v33, v21
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v21
; GFX8-NEXT: v_max_f32_e32 v4, v4, v20
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v21, v21
+; GFX8-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v21, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v20, vcc, v20, v4
+; GFX8-NEXT: v_add_u32_e32 v20, vcc, s4, v20
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v4
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v20, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v20, 16, v19
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v3
+; GFX8-NEXT: v_max_f32_e32 v20, v33, v20
+; GFX8-NEXT: v_bfe_u32 v33, v20, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v20
; GFX8-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_max_f32_e32 v20, v33, v20
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v20
; GFX8-NEXT: v_max_f32_e32 v3, v3, v19
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v20, v20
+; GFX8-NEXT: v_bfe_u32 v19, v3, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v20, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v19, vcc, v19, v3
+; GFX8-NEXT: v_add_u32_e32 v19, vcc, s4, v19
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v19, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v19, 16, v18
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v2
+; GFX8-NEXT: v_max_f32_e32 v19, v33, v19
+; GFX8-NEXT: v_bfe_u32 v33, v19, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v19
; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX8-NEXT: v_max_f32_e32 v19, v33, v19
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v19
; GFX8-NEXT: v_max_f32_e32 v2, v2, v18
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v19, v19
+; GFX8-NEXT: v_bfe_u32 v18, v2, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v19, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v18, vcc, v18, v2
+; GFX8-NEXT: v_add_u32_e32 v18, vcc, s4, v18
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v18, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v17
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v1
+; GFX8-NEXT: v_max_f32_e32 v18, v33, v18
+; GFX8-NEXT: v_bfe_u32 v33, v18, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v18
; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_max_f32_e32 v18, v33, v18
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v18
; GFX8-NEXT: v_max_f32_e32 v1, v1, v17
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
+; GFX8-NEXT: v_bfe_u32 v17, v1, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, v17, v1
+; GFX8-NEXT: v_add_u32_e32 v17, vcc, s4, v17
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v33, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v16
; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v0
+; GFX8-NEXT: v_max_f32_e32 v17, v33, v17
+; GFX8-NEXT: v_bfe_u32 v33, v17, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, v33, v17
; GFX8-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v33, vcc, s4, v33
+; GFX8-NEXT: v_and_b32_e32 v34, 0x80000000, v17
; GFX8-NEXT: v_max_f32_e32 v0, v0, v16
-; GFX8-NEXT: v_max_f32_e32 v17, v33, v17
+; GFX8-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
+; GFX8-NEXT: v_bfe_u32 v16, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
+; GFX8-NEXT: v_add_u32_e32 v16, vcc, v16, v0
+; GFX8-NEXT: v_add_u32_e32 v16, vcc, s4, v16
+; GFX8-NEXT: v_and_b32_e32 v33, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v16, v33, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
@@ -16922,8 +24609,13 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v30
+; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11
; GFX8-NEXT: v_alignbit_b32 v0, v0, v17, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v18, 16
; GFX8-NEXT: v_alignbit_b32 v2, v2, v19, 16
@@ -16932,8 +24624,13 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX8-NEXT: v_alignbit_b32 v5, v5, v22, 16
; GFX8-NEXT: v_alignbit_b32 v6, v6, v23, 16
; GFX8-NEXT: v_alignbit_b32 v7, v7, v24, 16
-; GFX8-NEXT: v_alignbit_b32 v14, v16, v31, 16
-; GFX8-NEXT: v_alignbit_b32 v15, v15, v32, 16
+; GFX8-NEXT: v_alignbit_b32 v8, v8, v25, 16
+; GFX8-NEXT: v_alignbit_b32 v9, v9, v26, 16
+; GFX8-NEXT: v_alignbit_b32 v10, v10, v27, 16
+; GFX8-NEXT: v_alignbit_b32 v11, v11, v28, 16
+; GFX8-NEXT: v_alignbit_b32 v12, v12, v29, 16
+; GFX8-NEXT: v_alignbit_b32 v13, v13, v32, 16
+; GFX8-NEXT: v_alignbit_b32 v15, v16, v15, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_maxnum_v32bf16:
@@ -16941,110 +24638,296 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v30
; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v14
+; GFX9-NEXT: v_max_f32_e32 v31, v32, v31
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_bfe_u32 v32, v31, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v31
; GFX9-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX9-NEXT: v_max_f32_e32 v31, v32, v31
+; GFX9-NEXT: v_add3_u32 v32, v32, v31, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
; GFX9-NEXT: v_max_f32_e32 v14, v14, v30
+; GFX9-NEXT: v_cndmask_b32_e32 v31, v32, v33, vcc
+; GFX9-NEXT: v_bfe_u32 v30, v14, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v32, 0x80000000, v14
+; GFX9-NEXT: v_add3_u32 v30, v30, v14, s4
+; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v32
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v14, v14
+; GFX9-NEXT: v_cndmask_b32_e32 v14, v30, v32, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v29
; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v13
; GFX9-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX9-NEXT: v_max_f32_e32 v30, v32, v30
; GFX9-NEXT: v_max_f32_e32 v13, v13, v29
-; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v28
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v12
+; GFX9-NEXT: buffer_load_dword v29, off, s[0:3], s32
+; GFX9-NEXT: v_max_f32_e32 v30, v32, v30
+; GFX9-NEXT: v_bfe_u32 v32, v30, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v30
+; GFX9-NEXT: v_add3_u32 v32, v32, v30, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v30, v30
+; GFX9-NEXT: v_cndmask_b32_e32 v30, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v15
+; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v29
+; GFX9-NEXT: v_max_f32_e32 v32, v32, v33
+; GFX9-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX9-NEXT: v_max_f32_e32 v29, v15, v29
+; GFX9-NEXT: v_bfe_u32 v15, v32, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v32
+; GFX9-NEXT: v_add3_u32 v15, v15, v32, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
+; GFX9-NEXT: v_cndmask_b32_e32 v15, v15, v33, vcc
+; GFX9-NEXT: v_bfe_u32 v32, v29, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v29
+; GFX9-NEXT: v_add3_u32 v32, v32, v29, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v29, v29
+; GFX9-NEXT: v_cndmask_b32_e32 v29, v32, v33, vcc
+; GFX9-NEXT: v_bfe_u32 v32, v13, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v13
+; GFX9-NEXT: v_add3_u32 v32, v32, v13, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v32, v33, vcc
+; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v28
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v12
+; GFX9-NEXT: v_max_f32_e32 v32, v33, v32
+; GFX9-NEXT: v_bfe_u32 v33, v32, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v32
; GFX9-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX9-NEXT: v_max_f32_e32 v29, v32, v29
+; GFX9-NEXT: v_add3_u32 v33, v33, v32, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v32, v32
; GFX9-NEXT: v_max_f32_e32 v12, v12, v28
+; GFX9-NEXT: v_cndmask_b32_e32 v32, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v28, v12, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v12
+; GFX9-NEXT: v_add3_u32 v28, v28, v12, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
+; GFX9-NEXT: v_cndmask_b32_e32 v12, v28, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v27
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v11
+; GFX9-NEXT: v_max_f32_e32 v28, v33, v28
+; GFX9-NEXT: v_bfe_u32 v33, v28, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v28
; GFX9-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; GFX9-NEXT: v_max_f32_e32 v28, v32, v28
+; GFX9-NEXT: v_add3_u32 v33, v33, v28, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v28, v28
; GFX9-NEXT: v_max_f32_e32 v11, v11, v27
+; GFX9-NEXT: v_cndmask_b32_e32 v28, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v27, v11, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v11
+; GFX9-NEXT: v_add3_u32 v27, v27, v11, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
+; GFX9-NEXT: v_cndmask_b32_e32 v11, v27, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v26
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v10
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v10
+; GFX9-NEXT: v_max_f32_e32 v27, v33, v27
+; GFX9-NEXT: v_bfe_u32 v33, v27, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v27
; GFX9-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX9-NEXT: v_max_f32_e32 v27, v32, v27
+; GFX9-NEXT: v_add3_u32 v33, v33, v27, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v27, v27
; GFX9-NEXT: v_max_f32_e32 v10, v10, v26
+; GFX9-NEXT: v_cndmask_b32_e32 v27, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v26, v10, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v10
+; GFX9-NEXT: v_add3_u32 v26, v26, v10, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v10, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v10, v26, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v25
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v9
+; GFX9-NEXT: v_max_f32_e32 v26, v33, v26
+; GFX9-NEXT: v_bfe_u32 v33, v26, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v26
; GFX9-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX9-NEXT: v_max_f32_e32 v26, v32, v26
+; GFX9-NEXT: v_add3_u32 v33, v33, v26, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v26, v26
; GFX9-NEXT: v_max_f32_e32 v9, v9, v25
+; GFX9-NEXT: v_cndmask_b32_e32 v26, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v25, v9, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v9
+; GFX9-NEXT: v_add3_u32 v25, v25, v9, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v9, v9
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v25, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v24
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v8
+; GFX9-NEXT: v_max_f32_e32 v25, v33, v25
+; GFX9-NEXT: v_bfe_u32 v33, v25, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v25
; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; GFX9-NEXT: v_add3_u32 v33, v33, v25, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v25, v25
; GFX9-NEXT: v_max_f32_e32 v8, v8, v24
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32
-; GFX9-NEXT: v_max_f32_e32 v25, v32, v25
-; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v15
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: v_perm_b32 v8, v8, v25, s4
-; GFX9-NEXT: v_perm_b32 v9, v9, v26, s4
-; GFX9-NEXT: v_perm_b32 v10, v10, v27, s4
-; GFX9-NEXT: v_perm_b32 v11, v11, v28, s4
-; GFX9-NEXT: v_perm_b32 v12, v12, v29, s4
-; GFX9-NEXT: v_perm_b32 v13, v13, v30, s4
-; GFX9-NEXT: v_perm_b32 v14, v14, v31, s4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v24
-; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; GFX9-NEXT: v_max_f32_e32 v32, v32, v33
-; GFX9-NEXT: v_max_f32_e32 v15, v15, v24
+; GFX9-NEXT: v_cndmask_b32_e32 v25, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v24, v8, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v8
+; GFX9-NEXT: v_add3_u32 v24, v24, v8, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v24, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v23
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v7
+; GFX9-NEXT: v_max_f32_e32 v24, v33, v24
+; GFX9-NEXT: v_bfe_u32 v33, v24, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v24
; GFX9-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX9-NEXT: v_max_f32_e32 v24, v33, v24
+; GFX9-NEXT: v_add3_u32 v33, v33, v24, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v24, v24
; GFX9-NEXT: v_max_f32_e32 v7, v7, v23
+; GFX9-NEXT: v_cndmask_b32_e32 v24, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v23, v7, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v7
+; GFX9-NEXT: v_add3_u32 v23, v23, v7, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v23, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v22
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v6
+; GFX9-NEXT: v_max_f32_e32 v23, v33, v23
+; GFX9-NEXT: v_bfe_u32 v33, v23, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v23
; GFX9-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX9-NEXT: v_max_f32_e32 v23, v33, v23
+; GFX9-NEXT: v_add3_u32 v33, v33, v23, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v23, v23
; GFX9-NEXT: v_max_f32_e32 v6, v6, v22
+; GFX9-NEXT: v_cndmask_b32_e32 v23, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v22, v6, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v6
+; GFX9-NEXT: v_add3_u32 v22, v22, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v22, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v21
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v5
+; GFX9-NEXT: v_max_f32_e32 v22, v33, v22
+; GFX9-NEXT: v_bfe_u32 v33, v22, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v22
; GFX9-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX9-NEXT: v_max_f32_e32 v22, v33, v22
+; GFX9-NEXT: v_add3_u32 v33, v33, v22, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v22, v22
; GFX9-NEXT: v_max_f32_e32 v5, v5, v21
+; GFX9-NEXT: v_cndmask_b32_e32 v22, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v21, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v5
+; GFX9-NEXT: v_add3_u32 v21, v21, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v21, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v20
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v4
+; GFX9-NEXT: v_max_f32_e32 v21, v33, v21
+; GFX9-NEXT: v_bfe_u32 v33, v21, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v21
; GFX9-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX9-NEXT: v_max_f32_e32 v21, v33, v21
+; GFX9-NEXT: v_add3_u32 v33, v33, v21, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v21, v21
; GFX9-NEXT: v_max_f32_e32 v4, v4, v20
+; GFX9-NEXT: v_cndmask_b32_e32 v21, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v20, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v20, v20, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v20, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v19
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v3
+; GFX9-NEXT: v_max_f32_e32 v20, v33, v20
+; GFX9-NEXT: v_bfe_u32 v33, v20, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v20
; GFX9-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_max_f32_e32 v20, v33, v20
+; GFX9-NEXT: v_add3_u32 v33, v33, v20, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v20, v20
; GFX9-NEXT: v_max_f32_e32 v3, v3, v19
+; GFX9-NEXT: v_cndmask_b32_e32 v20, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v19, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v19, v19, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v19, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v18
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v2
+; GFX9-NEXT: v_max_f32_e32 v19, v33, v19
+; GFX9-NEXT: v_bfe_u32 v33, v19, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v19
; GFX9-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_max_f32_e32 v19, v33, v19
+; GFX9-NEXT: v_add3_u32 v33, v33, v19, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v19, v19
; GFX9-NEXT: v_max_f32_e32 v2, v2, v18
+; GFX9-NEXT: v_cndmask_b32_e32 v19, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v18, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v18, v18, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v18, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v1
+; GFX9-NEXT: v_max_f32_e32 v18, v33, v18
+; GFX9-NEXT: v_bfe_u32 v33, v18, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v18
; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_max_f32_e32 v18, v33, v18
+; GFX9-NEXT: v_add3_u32 v33, v33, v18, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
; GFX9-NEXT: v_max_f32_e32 v1, v1, v17
+; GFX9-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v17, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v17, v17, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v33, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v16
; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v0
+; GFX9-NEXT: v_max_f32_e32 v17, v33, v17
+; GFX9-NEXT: v_bfe_u32 v33, v17, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v34, 0x80000000, v17
; GFX9-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_max_f32_e32 v17, v33, v17
+; GFX9-NEXT: v_add3_u32 v33, v33, v17, s4
+; GFX9-NEXT: v_or_b32_e32 v34, 0x400000, v34
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_max_f32_e32 v0, v0, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v17, v33, v34, vcc
+; GFX9-NEXT: v_bfe_u32 v16, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v33, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v16, v16, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v33
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v16, v33, vcc
+; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v17, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v18, s4
; GFX9-NEXT: v_perm_b32 v2, v2, v19, s4
@@ -17053,7 +24936,14 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX9-NEXT: v_perm_b32 v5, v5, v22, s4
; GFX9-NEXT: v_perm_b32 v6, v6, v23, s4
; GFX9-NEXT: v_perm_b32 v7, v7, v24, s4
-; GFX9-NEXT: v_perm_b32 v15, v15, v32, s4
+; GFX9-NEXT: v_perm_b32 v8, v8, v25, s4
+; GFX9-NEXT: v_perm_b32 v9, v9, v26, s4
+; GFX9-NEXT: v_perm_b32 v10, v10, v27, s4
+; GFX9-NEXT: v_perm_b32 v11, v11, v28, s4
+; GFX9-NEXT: v_perm_b32 v12, v12, v32, s4
+; GFX9-NEXT: v_perm_b32 v13, v13, v30, s4
+; GFX9-NEXT: v_perm_b32 v14, v14, v31, s4
+; GFX9-NEXT: v_perm_b32 v15, v29, v15, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_maxnum_v32bf16:
@@ -17068,32 +24958,10 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v10
; GFX10-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v30
-; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v14
-; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
-; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v29
-; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v13
-; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v28
; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v12
; GFX10-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX10-NEXT: v_max_f32_e32 v39, v48, v39
-; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v17
-; GFX10-NEXT: v_max_f32_e32 v11, v11, v27
-; GFX10-NEXT: v_lshlrev_b32_e32 v27, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_max_f32_e32 v49, v50, v49
-; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v16
-; GFX10-NEXT: v_max_f32_e32 v10, v10, v26
-; GFX10-NEXT: v_lshlrev_b32_e32 v26, 16, v0
-; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v15
-; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX10-NEXT: v_lshlrev_b32_e32 v51, 16, v25
; GFX10-NEXT: v_lshlrev_b32_e32 v52, 16, v9
; GFX10-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
@@ -17112,29 +24980,28 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX10-NEXT: v_lshlrev_b32_e32 v67, 16, v21
; GFX10-NEXT: v_lshlrev_b32_e32 v68, 16, v5
-; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT: v_max_f32_e32 v33, v34, v33
-; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v20
-; GFX10-NEXT: v_max_f32_e32 v14, v14, v30
-; GFX10-NEXT: v_lshlrev_b32_e32 v30, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v35, v36, v35
-; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v19
-; GFX10-NEXT: v_max_f32_e32 v13, v13, v29
-; GFX10-NEXT: v_lshlrev_b32_e32 v29, 16, v3
-; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_max_f32_e32 v39, v48, v39
+; GFX10-NEXT: v_max_f32_e32 v11, v11, v27
+; GFX10-NEXT: v_max_f32_e32 v49, v50, v49
+; GFX10-NEXT: v_max_f32_e32 v10, v10, v26
+; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v29
+; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v13
+; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX10-NEXT: v_max_f32_e32 v37, v38, v37
; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v18
; GFX10-NEXT: v_max_f32_e32 v12, v12, v28
; GFX10-NEXT: v_lshlrev_b32_e32 v28, 16, v2
; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_max_f32_e32 v0, v0, v16
-; GFX10-NEXT: v_max_f32_e32 v1, v1, v17
-; GFX10-NEXT: v_max_f32_e32 v51, v52, v51
+; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v17
+; GFX10-NEXT: v_lshlrev_b32_e32 v27, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v16
+; GFX10-NEXT: v_lshlrev_b32_e32 v26, 16, v0
+; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX10-NEXT: v_max_f32_e32 v9, v9, v25
; GFX10-NEXT: v_max_f32_e32 v25, v54, v53
; GFX10-NEXT: v_max_f32_e32 v8, v8, v24
@@ -17143,36 +25010,220 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX10-NEXT: v_max_f32_e32 v23, v66, v65
; GFX10-NEXT: v_max_f32_e32 v6, v6, v22
; GFX10-NEXT: v_max_f32_e32 v22, v68, v67
-; GFX10-NEXT: v_max_f32_e32 v5, v5, v21
-; GFX10-NEXT: v_max_f32_e32 v21, v30, v34
-; GFX10-NEXT: v_max_f32_e32 v29, v29, v36
-; GFX10-NEXT: v_max_f32_e32 v28, v28, v38
-; GFX10-NEXT: v_max_f32_e32 v27, v27, v48
-; GFX10-NEXT: v_max_f32_e32 v26, v26, v50
+; GFX10-NEXT: v_bfe_u32 v53, v39, 16, 1
+; GFX10-NEXT: v_bfe_u32 v55, v11, 16, 1
+; GFX10-NEXT: v_bfe_u32 v65, v49, 16, 1
+; GFX10-NEXT: v_bfe_u32 v67, v10, 16, 1
+; GFX10-NEXT: s_brev_b32 s23, 1
+; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v30
+; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v14
+; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; GFX10-NEXT: v_max_f32_e32 v35, v36, v35
+; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v19
+; GFX10-NEXT: v_max_f32_e32 v13, v13, v29
+; GFX10-NEXT: v_lshlrev_b32_e32 v29, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX10-NEXT: v_max_f32_e32 v2, v2, v18
+; GFX10-NEXT: v_max_f32_e32 v18, v27, v48
+; GFX10-NEXT: v_max_f32_e32 v1, v1, v17
+; GFX10-NEXT: v_max_f32_e32 v17, v26, v50
+; GFX10-NEXT: v_max_f32_e32 v0, v0, v16
+; GFX10-NEXT: v_and_or_b32 v54, v39, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v64, v11, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v66, v49, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v68, v10, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s9, v39, v39
+; GFX10-NEXT: v_add3_u32 v39, v53, v39, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s10, v11, v11
+; GFX10-NEXT: v_add3_u32 v11, v55, v11, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s11, v49, v49
+; GFX10-NEXT: v_add3_u32 v49, v65, v49, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s12, v10, v10
+; GFX10-NEXT: v_add3_u32 v10, v67, v10, 0x7fff
+; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX10-NEXT: v_max_f32_e32 v33, v34, v33
+; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v20
+; GFX10-NEXT: v_max_f32_e32 v14, v14, v30
+; GFX10-NEXT: v_lshlrev_b32_e32 v30, 16, v4
+; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX10-NEXT: v_max_f32_e32 v3, v3, v19
+; GFX10-NEXT: v_max_f32_e32 v19, v28, v38
+; GFX10-NEXT: v_bfe_u32 v38, v37, 16, 1
+; GFX10-NEXT: v_bfe_u32 v50, v12, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v39, v39, v54, s9
+; GFX10-NEXT: v_bfe_u32 v54, v18, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v11, v11, v64, s10
+; GFX10-NEXT: v_bfe_u32 v64, v1, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v49, v49, v66, s11
+; GFX10-NEXT: v_bfe_u32 v66, v17, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v68, s12
+; GFX10-NEXT: v_bfe_u32 v68, v0, 16, 1
+; GFX10-NEXT: v_max_f32_e32 v51, v52, v51
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v21
+; GFX10-NEXT: v_max_f32_e32 v21, v30, v34
; GFX10-NEXT: v_max_f32_e32 v4, v4, v20
-; GFX10-NEXT: v_perm_b32 v1, v1, v27, 0x7060302
-; GFX10-NEXT: v_perm_b32 v0, v0, v26, 0x7060302
-; GFX10-NEXT: v_perm_b32 v2, v2, v28, 0x7060302
-; GFX10-NEXT: v_perm_b32 v3, v3, v29, 0x7060302
+; GFX10-NEXT: v_max_f32_e32 v20, v29, v36
+; GFX10-NEXT: v_bfe_u32 v16, v33, 16, 1
+; GFX10-NEXT: v_bfe_u32 v27, v14, 16, 1
+; GFX10-NEXT: v_bfe_u32 v29, v35, 16, 1
+; GFX10-NEXT: v_bfe_u32 v34, v13, 16, 1
+; GFX10-NEXT: v_and_or_b32 v48, v37, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v52, v12, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s7, v37, v37
+; GFX10-NEXT: v_add3_u32 v37, v38, v37, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s8, v12, v12
+; GFX10-NEXT: v_add3_u32 v12, v50, v12, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s10, v18, v18
+; GFX10-NEXT: v_add3_u32 v54, v54, v18, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v18, v18, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s11, v1, v1
+; GFX10-NEXT: v_add3_u32 v64, v64, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v1, v1, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s12, v17, v17
+; GFX10-NEXT: v_add3_u32 v66, v66, v17, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v17, v17, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s22, v0, v0
+; GFX10-NEXT: v_add3_u32 v68, v68, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v0, v0, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v26, v33, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v28, v14, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v30, v35, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v36, v13, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX10-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v33, v51, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v14, v14
+; GFX10-NEXT: v_add3_u32 v14, v27, v14, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s5, v35, v35
+; GFX10-NEXT: v_add3_u32 v29, v29, v35, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s6, v13, v13
+; GFX10-NEXT: v_add3_u32 v13, v34, v13, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v65, v24, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v37, v37, v48, s7
+; GFX10-NEXT: v_bfe_u32 v48, v19, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v12, v12, v52, s8
+; GFX10-NEXT: v_bfe_u32 v52, v2, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v18, v54, v18, s10
+; GFX10-NEXT: v_cndmask_b32_e64 v17, v66, v17, s12
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v68, v0, s22
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v64, v1, s11
+; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v15
+; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX10-NEXT: v_and_or_b32 v27, v51, s23, 0x400000
+; GFX10-NEXT: v_bfe_u32 v35, v9, 16, 1
+; GFX10-NEXT: v_bfe_u32 v38, v25, 16, 1
+; GFX10-NEXT: v_and_or_b32 v67, v24, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s13, v51, v51
+; GFX10-NEXT: v_add3_u32 v33, v33, v51, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v51, v7, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s17, v24, v24
+; GFX10-NEXT: v_add3_u32 v24, v65, v24, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v65, v6, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v16, v16, v26, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v26, v21, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v14, v14, v28, s4
+; GFX10-NEXT: v_bfe_u32 v28, v4, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v29, v29, v30, s5
+; GFX10-NEXT: v_bfe_u32 v30, v20, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e64 v13, v13, v36, s6
+; GFX10-NEXT: v_bfe_u32 v36, v3, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s8, v19, v19
+; GFX10-NEXT: v_add3_u32 v48, v48, v19, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v19, v19, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s9, v2, v2
+; GFX10-NEXT: v_add3_u32 v52, v52, v2, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v2, v2, s23, 0x400000
+; GFX10-NEXT: v_perm_b32 v0, v0, v17, 0x7060302
+; GFX10-NEXT: v_perm_b32 v1, v1, v18, 0x7060302
+; GFX10-NEXT: v_and_or_b32 v34, v9, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v50, v25, s23, 0x400000
+; GFX10-NEXT: v_bfe_u32 v53, v8, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s14, v9, v9
+; GFX10-NEXT: v_add3_u32 v9, v35, v9, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v35, v7, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s15, v25, v25
+; GFX10-NEXT: v_add3_u32 v25, v38, v25, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v38, v23, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s18, v7, v7
+; GFX10-NEXT: v_add3_u32 v7, v51, v7, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v51, v6, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s20, v6, v6
+; GFX10-NEXT: v_add3_u32 v6, v65, v6, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v65, v5, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v21, v21
+; GFX10-NEXT: v_add3_u32 v26, v26, v21, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v21, v21, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s5, v4, v4
+; GFX10-NEXT: v_add3_u32 v28, v28, v4, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v4, v4, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s6, v20, v20
+; GFX10-NEXT: v_add3_u32 v30, v30, v20, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v20, v20, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s7, v3, v3
+; GFX10-NEXT: v_add3_u32 v36, v36, v3, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v3, v3, s23, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e64 v19, v48, v19, s8
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v52, v2, s9
+; GFX10-NEXT: v_and_or_b32 v55, v8, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s16, v8, v8
+; GFX10-NEXT: v_add3_u32 v8, v53, v8, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v53, v23, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e64 s19, v23, v23
+; GFX10-NEXT: v_add3_u32 v23, v38, v23, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v38, v22, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v65, v65, v5, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v5, v5, s23, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e64 v21, v26, v21, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v28, v4, s5
+; GFX10-NEXT: v_cndmask_b32_e64 v20, v30, v20, s6
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v36, v3, s7
+; GFX10-NEXT: v_perm_b32 v2, v2, v19, 0x7060302
+; GFX10-NEXT: v_cmp_u_f32_e64 s21, v22, v22
+; GFX10-NEXT: v_add3_u32 v38, v38, v22, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v22, v22, s23, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v65, v5, vcc_lo
+; GFX10-NEXT: v_perm_b32 v3, v3, v20, 0x7060302
; GFX10-NEXT: v_perm_b32 v4, v4, v21, 0x7060302
-; GFX10-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
-; GFX10-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
-; GFX10-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e64 v27, v33, v27, s13
+; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v34, s14
+; GFX10-NEXT: v_cndmask_b32_e64 v25, v25, v50, s15
+; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, v55, s16
+; GFX10-NEXT: v_cndmask_b32_e64 v24, v24, v67, s17
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v35, s18
+; GFX10-NEXT: v_cndmask_b32_e64 v23, v23, v53, s19
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v51, s20
+; GFX10-NEXT: v_cndmask_b32_e64 v22, v38, v22, s21
; GFX10-NEXT: v_perm_b32 v8, v8, v25, 0x7060302
-; GFX10-NEXT: v_perm_b32 v9, v9, v51, 0x7060302
+; GFX10-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
+; GFX10-NEXT: v_perm_b32 v9, v9, v27, 0x7060302
+; GFX10-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
+; GFX10-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
; GFX10-NEXT: v_perm_b32 v10, v10, v49, 0x7060302
; GFX10-NEXT: v_perm_b32 v11, v11, v39, 0x7060302
; GFX10-NEXT: v_perm_b32 v12, v12, v37, 0x7060302
-; GFX10-NEXT: v_perm_b32 v13, v13, v35, 0x7060302
-; GFX10-NEXT: v_perm_b32 v14, v14, v33, 0x7060302
+; GFX10-NEXT: v_perm_b32 v13, v13, v29, 0x7060302
+; GFX10-NEXT: v_perm_b32 v14, v14, v16, 0x7060302
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v31
-; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v31
-; GFX10-NEXT: v_max_f32_e32 v16, v32, v16
-; GFX10-NEXT: v_max_f32_e32 v15, v15, v17
-; GFX10-NEXT: v_perm_b32 v15, v15, v16, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v31
+; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v31
+; GFX10-NEXT: v_max_f32_e32 v17, v32, v17
+; GFX10-NEXT: v_max_f32_e32 v15, v15, v18
+; GFX10-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX10-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX10-NEXT: v_and_or_b32 v20, v17, s23, 0x400000
+; GFX10-NEXT: v_and_or_b32 v21, v15, s23, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v15, v15
+; GFX10-NEXT: v_add3_u32 v17, v18, v17, 0x7fff
+; GFX10-NEXT: v_add3_u32 v15, v19, v15, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v17, v17, v20, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v15, v15, v21, s4
+; GFX10-NEXT: v_perm_b32 v15, v15, v17, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_maxnum_v32bf16:
@@ -17183,102 +25234,269 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX11-NEXT: v_lshlrev_b32_e32 v84, 16, v1
; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v24
+; GFX11-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_dual_max_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v64, 16, v7
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v18
; GFX11-NEXT: v_lshlrev_b32_e32 v85, 16, v16
; GFX11-NEXT: v_lshlrev_b32_e32 v86, 16, v0
+; GFX11-NEXT: v_bfe_u32 v135, v1, 16, 1
; GFX11-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v23
+; GFX11-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; GFX11-NEXT: v_and_or_b32 v144, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v54, 16, v8
-; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v7
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-NEXT: v_dual_max_f32 v17, v86, v85 :: v_dual_and_b32 v8, 0xffff0000, v8
+; GFX11-NEXT: v_dual_max_f32 v7, v7, v23 :: v_dual_lshlrev_b32 v36, 16, v13
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_dual_max_f32 v8, v8, v24 :: v_dual_lshlrev_b32 v39, 16, v27
+; GFX11-NEXT: v_dual_max_f32 v0, v0, v16 :: v_dual_lshlrev_b32 v49, 16, v26
+; GFX11-NEXT: v_max_f32_e32 v24, v64, v55
+; GFX11-NEXT: v_bfe_u32 v87, v7, 16, 1
; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v22
; GFX11-NEXT: v_lshlrev_b32_e32 v66, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v11
-; GFX11-NEXT: v_dual_max_f32 v0, v0, v16 :: v_dual_and_b32 v11, 0xffff0000, v11
; GFX11-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX11-NEXT: v_bfe_u32 v85, v24, 16, 1
; GFX11-NEXT: v_lshlrev_b32_e32 v67, 16, v21
; GFX11-NEXT: v_lshlrev_b32_e32 v68, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v51, 16, v25
; GFX11-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v20
; GFX11-NEXT: v_lshlrev_b32_e32 v70, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v23
-; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v19
-; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v3
-; GFX11-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v9
-; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v18
-; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v2
+; GFX11-NEXT: v_and_or_b32 v86, v24, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v96, v7, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v85, v85, v24, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v20
+; GFX11-NEXT: v_add3_u32 v87, v87, v7, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX11-NEXT: v_dual_max_f32 v23, v66, v65 :: v_dual_lshlrev_b32 v48, 16, v11
+; GFX11-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; GFX11-NEXT: v_dual_max_f32 v5, v5, v21 :: v_dual_lshlrev_b32 v50, 16, v10
+; GFX11-NEXT: v_dual_max_f32 v21, v70, v69 :: v_dual_and_b32 v26, 0xffff0000, v26
; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v24
-; GFX11-NEXT: v_dual_max_f32 v1, v1, v17 :: v_dual_and_b32 v24, 0xffff0000, v24
-; GFX11-NEXT: v_dual_max_f32 v5, v5, v21 :: v_dual_lshlrev_b32 v50, 16, v10
-; GFX11-NEXT: v_dual_max_f32 v21, v70, v69 :: v_dual_and_b32 v10, 0xffff0000, v10
-; GFX11-NEXT: v_dual_max_f32 v2, v2, v18 :: v_dual_max_f32 v3, v3, v19
-; GFX11-NEXT: v_dual_max_f32 v4, v4, v20 :: v_dual_lshlrev_b32 v49, 16, v26
-; GFX11-NEXT: v_dual_max_f32 v9, v9, v25 :: v_dual_and_b32 v26, 0xffff0000, v26
; GFX11-NEXT: v_max_f32_e32 v6, v6, v22
-; GFX11-NEXT: v_dual_max_f32 v22, v68, v67 :: v_dual_lshlrev_b32 v37, 16, v28
-; GFX11-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_max_f32_e32 v10, v10, v26
-; GFX11-NEXT: v_max_f32_e32 v26, v52, v51
-; GFX11-NEXT: v_perm_b32 v4, v4, v21, 0x7060302
-; GFX11-NEXT: v_max_f32_e32 v25, v54, v53
-; GFX11-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
-; GFX11-NEXT: v_perm_b32 v9, v9, v26, 0x7060302
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v31
-; GFX11-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v31
-; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11-NEXT: v_lshlrev_b32_e32 v36, 16, v13
-; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX11-NEXT: v_lshlrev_b32_e32 v39, 16, v27
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_dual_max_f32 v8, v8, v24 :: v_dual_and_b32 v27, 0xffff0000, v27
-; GFX11-NEXT: v_max_f32_e32 v24, v64, v55
+; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v9
+; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; GFX11-NEXT: v_bfe_u32 v97, v23, 16, 1
+; GFX11-NEXT: v_max_f32_e32 v2, v2, v18
+; GFX11-NEXT: v_max_f32_e32 v18, v84, v83
+; GFX11-NEXT: v_bfe_u32 v83, v8, 16, 1
+; GFX11-NEXT: v_bfe_u32 v99, v6, 16, 1
+; GFX11-NEXT: v_bfe_u32 v103, v5, 16, 1
+; GFX11-NEXT: v_bfe_u32 v113, v21, 16, 1
+; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v19
+; GFX11-NEXT: v_and_or_b32 v84, v8, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v98, v23, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v100, v6, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v112, v5, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v114, v21, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v83, v83, v8, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; GFX11-NEXT: v_add3_u32 v97, v97, v23, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; GFX11-NEXT: v_add3_u32 v99, v99, v6, 0x7fff
+; GFX11-NEXT: v_add3_u32 v103, v103, v5, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-NEXT: v_add3_u32 v113, v113, v21, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v12
-; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX11-NEXT: v_dual_max_f32 v3, v3, v19 :: v_dual_and_b32 v10, 0xffff0000, v10
+; GFX11-NEXT: v_dual_max_f32 v22, v68, v67 :: v_dual_lshlrev_b32 v51, 16, v25
+; GFX11-NEXT: v_lshlrev_b32_e32 v37, 16, v28
+; GFX11-NEXT: v_dual_max_f32 v4, v4, v20 :: v_dual_and_b32 v25, 0xffff0000, v25
+; GFX11-NEXT: v_max_f32_e32 v20, v80, v71
+; GFX11-NEXT: v_dual_max_f32 v19, v82, v81 :: v_dual_and_b32 v28, 0xffff0000, v28
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_dual_max_f32 v9, v9, v25 :: v_dual_and_b32 v12, 0xffff0000, v12
+; GFX11-NEXT: v_max_f32_e32 v25, v54, v53
; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v29
-; GFX11-NEXT: v_max_f32_e32 v7, v7, v23
-; GFX11-NEXT: v_max_f32_e32 v23, v66, v65
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_max_f32 v12, v12, v28 :: v_dual_and_b32 v29, 0xffff0000, v29
-; GFX11-NEXT: v_dual_max_f32 v28, v48, v39 :: v_dual_lshlrev_b32 v33, 16, v30
+; GFX11-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; GFX11-NEXT: v_dual_max_f32 v10, v10, v26 :: v_dual_and_b32 v13, 0xffff0000, v13
+; GFX11-NEXT: v_dual_max_f32 v12, v12, v28 :: v_dual_lshlrev_b32 v33, 16, v30
+; GFX11-NEXT: v_max_f32_e32 v28, v48, v39
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_dual_max_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v34, 16, v14
-; GFX11-NEXT: v_lshlrev_b32_e32 v32, 16, v15
; GFX11-NEXT: v_dual_max_f32 v11, v11, v27 :: v_dual_and_b32 v14, 0xffff0000, v14
-; GFX11-NEXT: v_dual_max_f32 v27, v50, v49 :: v_dual_and_b32 v30, 0xffff0000, v30
-; GFX11-NEXT: v_max_f32_e32 v29, v38, v37
+; GFX11-NEXT: v_dual_max_f32 v27, v50, v49 :: v_dual_max_f32 v26, v52, v51
+; GFX11-NEXT: v_dual_max_f32 v29, v38, v37 :: v_dual_and_b32 v30, 0xffff0000, v30
+; GFX11-NEXT: v_lshlrev_b32_e32 v32, 16, v15
; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX11-NEXT: v_max_f32_e32 v37, v86, v85
-; GFX11-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v39, v13, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_max_f32_e32 v14, v14, v30
; GFX11-NEXT: v_dual_max_f32 v30, v36, v35 :: v_dual_max_f32 v33, v34, v33
-; GFX11-NEXT: v_dual_max_f32 v34, v80, v71 :: v_dual_max_f32 v35, v82, v81
-; GFX11-NEXT: v_max_f32_e32 v36, v84, v83
-; GFX11-NEXT: v_dual_max_f32 v16, v32, v16 :: v_dual_max_f32 v15, v15, v17
-; GFX11-NEXT: v_perm_b32 v0, v0, v37, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v2, v2, v35, 0x7060302
-; GFX11-NEXT: v_perm_b32 v1, v1, v36, 0x7060302
-; GFX11-NEXT: v_perm_b32 v3, v3, v34, 0x7060302
-; GFX11-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
-; GFX11-NEXT: v_perm_b32 v8, v8, v25, 0x7060302
+; GFX11-NEXT: v_and_or_b32 v48, v13, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v49, v29, 16, 1
+; GFX11-NEXT: v_bfe_u32 v35, v14, 16, 1
+; GFX11-NEXT: v_and_or_b32 v36, v14, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v16, v33, 16, 1
+; GFX11-NEXT: v_and_or_b32 v34, v33, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX11-NEXT: v_bfe_u32 v37, v30, 16, 1
+; GFX11-NEXT: v_add3_u32 v35, v35, v14, 0x7fff
+; GFX11-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v38, v30, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v39, v39, v13, 0x7fff
+; GFX11-NEXT: v_add3_u32 v37, v37, v30, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v50, v29, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v16, v16, v34, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-NEXT: v_bfe_u32 v51, v12, 16, 1
+; GFX11-NEXT: v_add3_u32 v49, v49, v29, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v52, v12, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v53, v28, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v14, v35, v36, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
+; GFX11-NEXT: v_add3_u32 v51, v51, v12, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v54, v28, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v55, v11, 16, 1
+; GFX11-NEXT: v_add3_u32 v53, v53, v28, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v30, v37, v38, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-NEXT: v_and_or_b32 v64, v11, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v65, v27, 16, 1
+; GFX11-NEXT: v_add3_u32 v55, v55, v11, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v66, v27, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v13, v39, v48, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-NEXT: v_bfe_u32 v67, v10, 16, 1
+; GFX11-NEXT: v_add3_u32 v65, v65, v27, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v68, v10, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v69, v26, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v29, v49, v50, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-NEXT: v_add3_u32 v67, v67, v10, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v70, v26, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v71, v9, 16, 1
+; GFX11-NEXT: v_add3_u32 v69, v69, v26, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v12, v51, v52, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX11-NEXT: v_and_or_b32 v80, v9, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v81, v25, 16, 1
+; GFX11-NEXT: v_add3_u32 v71, v71, v9, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v82, v25, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v28, v53, v54, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-NEXT: v_add3_u32 v81, v81, v25, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v101, v22, 16, 1
+; GFX11-NEXT: v_and_or_b32 v102, v22, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v115, v4, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v11, v55, v64, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX11-NEXT: v_add3_u32 v101, v101, v22, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v116, v4, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v117, v20, 16, 1
+; GFX11-NEXT: v_add3_u32 v115, v115, v4, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v27, v65, v66, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-NEXT: v_and_or_b32 v118, v20, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v129, v19, 16, 1
+; GFX11-NEXT: v_add3_u32 v117, v117, v20, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v130, v19, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v10, v67, v68, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-NEXT: v_bfe_u32 v133, v18, 16, 1
+; GFX11-NEXT: v_add3_u32 v129, v129, v19, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v134, v18, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v145, v17, 16, 1
+; GFX11-NEXT: v_cndmask_b32_e32 v26, v69, v70, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-NEXT: v_add3_u32 v133, v133, v18, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v146, v17, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v147, v0, 16, 1
+; GFX11-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v9, v71, v80, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-NEXT: v_bfe_u32 v131, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v33, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v147, v147, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v132, v2, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v25, v81, v82, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-NEXT: v_add3_u32 v131, v131, v2, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v119, v3, 16, 1
+; GFX11-NEXT: v_and_or_b32 v128, v3, s0, 0x400000
+; GFX11-NEXT: v_perm_b32 v9, v9, v26, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v8, v83, v84, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-NEXT: v_add3_u32 v119, v119, v3, 0x7fff
; GFX11-NEXT: v_perm_b32 v10, v10, v27, 0x7060302
; GFX11-NEXT: v_perm_b32 v11, v11, v28, 0x7060302
+; GFX11-NEXT: v_perm_b32 v8, v8, v25, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v24, v85, v86, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11-NEXT: v_perm_b32 v12, v12, v29, 0x7060302
; GFX11-NEXT: v_perm_b32 v13, v13, v30, 0x7060302
-; GFX11-NEXT: v_perm_b32 v14, v14, v33, 0x7060302
-; GFX11-NEXT: v_perm_b32 v15, v15, v16, 0x7060302
+; GFX11-NEXT: v_perm_b32 v14, v14, v16, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v87, v96, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v7, v7, v24, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v23, v97, v98, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: v_cndmask_b32_e32 v6, v99, v100, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-NEXT: v_perm_b32 v6, v6, v23, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v22, v101, v102, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v103, v112, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v5, v5, v22, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v21, v113, v114, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v115, v116, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-NEXT: v_perm_b32 v4, v4, v21, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v20, v117, v118, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-NEXT: v_cndmask_b32_e32 v19, v129, v130, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-NEXT: v_cndmask_b32_e32 v18, v133, v134, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v135, v144, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, v1, v18, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v17, v145, v146, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v147, v33, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_perm_b32 v0, v0, v17, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v131, v132, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v119, v128, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_perm_b32 v3, v3, v20, 0x7060302
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v31
+; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v31
+; GFX11-NEXT: v_perm_b32 v2, v2, v19, 0x7060302
+; GFX11-NEXT: v_max_f32_e32 v17, v32, v17
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_max_f32_e32 v15, v15, v18
+; GFX11-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11-NEXT: v_and_or_b32 v20, v17, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-NEXT: v_and_or_b32 v21, v15, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v18, v18, v17, 0x7fff
+; GFX11-NEXT: v_add3_u32 v19, v19, v15, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v17, v18, v20, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-NEXT: v_cndmask_b32_e32 v15, v19, v21, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v15, v15, v17, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <32 x bfloat> @llvm.maxnum.v32bf16(<32 x bfloat> %a, <32 x bfloat> %b)
ret <32 x bfloat> %op
@@ -17290,9 +25508,10 @@ define bfloat @v_sqrt_bf16(bfloat %a) {
; GCN-LABEL: v_sqrt_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, 0xf800000
; GCN-NEXT: v_mov_b32_e32 v1, 0x260
+; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_mul_f32_e32 v2, 0x4f800000, v0
; GCN-NEXT: v_cmp_gt_f32_e32 vcc, s4, v0
; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
@@ -17315,6 +25534,7 @@ define bfloat @v_sqrt_bf16(bfloat %a) {
; GFX7-LABEL: v_sqrt_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: s_mov_b32 s4, 0xf800000
; GFX7-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0
@@ -17359,6 +25579,13 @@ define bfloat @v_sqrt_bf16(bfloat %a) {
; GFX8-NEXT: v_mov_b32_e32 v2, 0x260
; GFX8-NEXT: v_cmp_class_f32_e32 vcc, v0, v2
; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -17384,6 +25611,13 @@ define bfloat @v_sqrt_bf16(bfloat %a) {
; GFX9-NEXT: v_mov_b32_e32 v2, 0x260
; GFX9-NEXT: v_cmp_class_f32_e32 vcc, v0, v2
; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -17403,10 +25637,16 @@ define bfloat @v_sqrt_bf16(bfloat %a) {
; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v2, s4
; GFX10-NEXT: v_cmp_lt_f32_e64 s4, 0, v5
; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v3, s4
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_mul_f32_e32 v2, 0x37800000, v1
; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
; GFX10-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 0x260
; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -17430,14 +25670,21 @@ define bfloat @v_sqrt_bf16(bfloat %a) {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v2, s0
; GFX11-NEXT: v_cmp_lt_f32_e64 s0, 0, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v3, s0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: v_mul_f32_e32 v2, 0x37800000, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 0x260
; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.sqrt.bf16(bfloat %a)
@@ -17450,6 +25697,7 @@ define bfloat @v_ldexp_bf16_i32(bfloat %a, i32 %b) {
; GCN-LABEL: v_ldexp_bf16_i32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_ldexp_f32_e32 v0, v0, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -17458,6 +25706,7 @@ define bfloat @v_ldexp_bf16_i32(bfloat %a, i32 %b) {
; GFX7-LABEL: v_ldexp_bf16_i32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_ldexp_f32_e32 v0, v0, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -17468,6 +25717,13 @@ define bfloat @v_ldexp_bf16_i32(bfloat %a, i32 %b) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -17476,6 +25732,13 @@ define bfloat @v_ldexp_bf16_i32(bfloat %a, i32 %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -17483,7 +25746,13 @@ define bfloat @v_ldexp_bf16_i32(bfloat %a, i32 %b) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -17491,8 +25760,16 @@ define bfloat @v_ldexp_bf16_i32(bfloat %a, i32 %b) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.ldexp.bf16.i32(bfloat %a, i32 %b)
@@ -17505,12 +25782,14 @@ define { bfloat, i16 } @v_frexp_bf16_i16(bfloat %a) {
; GCN-LABEL: v_frexp_bf16_i16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, 0x7f800000
+; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_frexp_mant_f32_e32 v1, v0
; GCN-NEXT: v_frexp_exp_i32_f32_e32 v2, v0
; GCN-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s4
; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -17518,6 +25797,7 @@ define { bfloat, i16 } @v_frexp_bf16_i16(bfloat %a) {
; GFX7-LABEL: v_frexp_bf16_i16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
; GFX7-NEXT: v_frexp_mant_f32_e32 v0, v0
@@ -17527,10 +25807,17 @@ define { bfloat, i16 } @v_frexp_bf16_i16(bfloat %a) {
; GFX8-LABEL: v_frexp_bf16_i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
-; GFX8-NEXT: v_frexp_mant_f32_e32 v0, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX8-NEXT: v_frexp_mant_f32_e32 v0, v1
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_frexp_bf16_i16:
@@ -17538,6 +25825,13 @@ define { bfloat, i16 } @v_frexp_bf16_i16(bfloat %a) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v0
; GFX9-NEXT: v_frexp_mant_f32_e32 v0, v1
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -17546,8 +25840,14 @@ define { bfloat, i16 } @v_frexp_bf16_i16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_frexp_mant_f32_e32 v0, v1
; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX10-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v3, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = call { bfloat, i16 } @llvm.frexp.bf16.i16(bfloat %a)
@@ -17563,11 +25863,12 @@ define bfloat @v_log_bf16(bfloat %a) {
; GCN-LABEL: v_log_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, 0x800000
; GCN-NEXT: v_mov_b32_e32 v1, 0x4f800000
; GCN-NEXT: s_mov_b32 s5, 0x7f800000
; GCN-NEXT: v_mov_b32_e32 v2, 0x41b17218
+; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_gt_f32_e32 vcc, s4, v0
; GCN-NEXT: v_cndmask_b32_e32 v1, 1.0, v1, vcc
; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
@@ -17591,6 +25892,7 @@ define bfloat @v_log_bf16(bfloat %a) {
; GFX7-LABEL: v_log_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: s_mov_b32 s4, 0x800000
; GFX7-NEXT: v_mov_b32_e32 v1, 0x4f800000
@@ -17638,6 +25940,13 @@ define bfloat @v_log_bf16(bfloat %a) {
; GFX8-NEXT: v_mov_b32_e32 v1, 0x41b17218
; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -17663,6 +25972,13 @@ define bfloat @v_log_bf16(bfloat %a) {
; GFX9-NEXT: v_mov_b32_e32 v1, 0x41b17218
; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -17670,18 +25986,24 @@ define bfloat @v_log_bf16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
; GFX10-NEXT: v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX10-NEXT: v_log_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX10-NEXT: v_cmp_gt_f32_e64 s4, 0x7f800000, |v0|
; GFX10-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
; GFX10-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
; GFX10-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v1, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX10-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 0x41b17218, vcc_lo
+; GFX10-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX10-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -17689,24 +26011,31 @@ define bfloat @v_log_bf16(bfloat %a) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
; GFX11-NEXT: v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
; GFX11-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_log_f32_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr 0xfff
; GFX11-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX11-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
; GFX11-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
; GFX11-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 0x41b17218, vcc_lo
+; GFX11-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.log.bf16(bfloat %a)
@@ -17717,10 +26046,11 @@ define bfloat @v_log2_bf16(bfloat %a) {
; GCN-LABEL: v_log2_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, 0x800000
; GCN-NEXT: v_mov_b32_e32 v1, 0x4f800000
; GCN-NEXT: v_mov_b32_e32 v2, 0x42000000
+; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_gt_f32_e32 vcc, s4, v0
; GCN-NEXT: v_cndmask_b32_e32 v1, 1.0, v1, vcc
; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
@@ -17733,6 +26063,7 @@ define bfloat @v_log2_bf16(bfloat %a) {
; GFX7-LABEL: v_log2_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: s_mov_b32 s4, 0x800000
; GFX7-NEXT: v_mov_b32_e32 v1, 0x4f800000
@@ -17759,6 +26090,13 @@ define bfloat @v_log2_bf16(bfloat %a) {
; GFX8-NEXT: v_mov_b32_e32 v1, 0x42000000
; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -17774,7 +26112,14 @@ define bfloat @v_log2_bf16(bfloat %a) {
; GFX9-NEXT: v_log_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0x42000000
; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -17782,12 +26127,18 @@ define bfloat @v_log2_bf16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
; GFX10-NEXT: v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v2
; GFX10-NEXT: v_log_f32_e32 v0, v0
; GFX10-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -17795,6 +26146,7 @@ define bfloat @v_log2_bf16(bfloat %a) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
; GFX11-NEXT: v_cndmask_b32_e64 v2, 1.0, 0x4f800000, vcc_lo
@@ -17804,6 +26156,13 @@ define bfloat @v_log2_bf16(bfloat %a) {
; GFX11-NEXT: v_log_f32_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr 0xfff
; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.log2.bf16(bfloat %a)
@@ -17814,11 +26173,12 @@ define bfloat @v_log10_bf16(bfloat %a) {
; GCN-LABEL: v_log10_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, 0x800000
; GCN-NEXT: v_mov_b32_e32 v1, 0x4f800000
; GCN-NEXT: s_mov_b32 s5, 0x7f800000
; GCN-NEXT: v_mov_b32_e32 v2, 0x411a209b
+; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_gt_f32_e32 vcc, s4, v0
; GCN-NEXT: v_cndmask_b32_e32 v1, 1.0, v1, vcc
; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
@@ -17842,6 +26202,7 @@ define bfloat @v_log10_bf16(bfloat %a) {
; GFX7-LABEL: v_log10_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: s_mov_b32 s4, 0x800000
; GFX7-NEXT: v_mov_b32_e32 v1, 0x4f800000
@@ -17889,6 +26250,13 @@ define bfloat @v_log10_bf16(bfloat %a) {
; GFX8-NEXT: v_mov_b32_e32 v1, 0x411a209b
; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -17914,6 +26282,13 @@ define bfloat @v_log10_bf16(bfloat %a) {
; GFX9-NEXT: v_mov_b32_e32 v1, 0x411a209b
; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -17921,18 +26296,24 @@ define bfloat @v_log10_bf16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
; GFX10-NEXT: v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX10-NEXT: v_log_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX10-NEXT: v_cmp_gt_f32_e64 s4, 0x7f800000, |v0|
; GFX10-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
; GFX10-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
; GFX10-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v1, s4
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX10-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 0x411a209b, vcc_lo
+; GFX10-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX10-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -17940,24 +26321,31 @@ define bfloat @v_log10_bf16(bfloat %a) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
; GFX11-NEXT: v_cndmask_b32_e64 v1, 1.0, 0x4f800000, vcc_lo
; GFX11-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_log_f32_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr 0xfff
; GFX11-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX11-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
; GFX11-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
; GFX11-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 0x411a209b, vcc_lo
+; GFX11-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.log10.bf16(bfloat %a)
@@ -17972,10 +26360,11 @@ define bfloat @v_exp_bf16(bfloat %a) {
; GCN-LABEL: v_exp_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, 0xc2ce8ed0
; GCN-NEXT: s_mov_b32 s5, 0x42b17218
; GCN-NEXT: v_mov_b32_e32 v1, 0x7f800000
+; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_mul_f32_e32 v2, 0x3fb8a000, v0
; GCN-NEXT: v_sub_f32_e32 v3, v0, v0
; GCN-NEXT: v_mul_f32_e32 v4, 0x39a3b295, v0
@@ -17999,6 +26388,7 @@ define bfloat @v_exp_bf16(bfloat %a) {
; GFX7-LABEL: v_exp_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: s_mov_b32 s4, 0x3fb8aa3b
; GFX7-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
@@ -18045,6 +26435,13 @@ define bfloat @v_exp_bf16(bfloat %a) {
; GFX8-NEXT: v_mov_b32_e32 v2, 0x7f800000
; GFX8-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -18070,6 +26467,13 @@ define bfloat @v_exp_bf16(bfloat %a) {
; GFX9-NEXT: v_mov_b32_e32 v2, 0x7f800000
; GFX9-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -18077,6 +26481,7 @@ define bfloat @v_exp_bf16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
; GFX10-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
; GFX10-NEXT: v_rndne_f32_e32 v2, v1
@@ -18090,6 +26495,11 @@ define bfloat @v_exp_bf16(bfloat %a) {
; GFX10-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
; GFX10-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0
; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -18097,6 +26507,7 @@ define bfloat @v_exp_bf16(bfloat %a) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
; GFX11-NEXT: v_rndne_f32_e32 v2, v1
@@ -18115,7 +26526,13 @@ define bfloat @v_exp_bf16(bfloat %a) {
; GFX11-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
; GFX11-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0
; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.exp.bf16(bfloat %a)
@@ -18126,10 +26543,11 @@ define bfloat @v_exp2_bf16(bfloat %a) {
; GCN-LABEL: v_exp2_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, 0xc2fc0000
; GCN-NEXT: v_mov_b32_e32 v1, 0x42800000
; GCN-NEXT: v_mov_b32_e32 v2, 0x1f800000
+; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_gt_f32_e32 vcc, s4, v0
; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GCN-NEXT: v_add_f32_e32 v0, v0, v1
@@ -18142,6 +26560,7 @@ define bfloat @v_exp2_bf16(bfloat %a) {
; GFX7-LABEL: v_exp2_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: s_mov_b32 s4, 0xc2fc0000
; GFX7-NEXT: v_mov_b32_e32 v1, 0x42800000
@@ -18168,6 +26587,13 @@ define bfloat @v_exp2_bf16(bfloat %a) {
; GFX8-NEXT: v_mov_b32_e32 v1, 0x1f800000
; GFX8-NEXT: v_cndmask_b32_e32 v1, 1.0, v1, vcc
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -18183,7 +26609,14 @@ define bfloat @v_exp2_bf16(bfloat %a) {
; GFX9-NEXT: v_exp_f32_e32 v0, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0x1f800000
; GFX9-NEXT: v_cndmask_b32_e32 v1, 1.0, v1, vcc
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -18191,12 +26624,18 @@ define bfloat @v_exp2_bf16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v1, 1.0, 0x1f800000, vcc_lo
; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
; GFX10-NEXT: v_exp_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -18204,6 +26643,7 @@ define bfloat @v_exp2_bf16(bfloat %a) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
@@ -18213,6 +26653,13 @@ define bfloat @v_exp2_bf16(bfloat %a) {
; GFX11-NEXT: v_exp_f32_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr 0xfff
; GFX11-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.exp2.bf16(bfloat %a)
@@ -18223,10 +26670,11 @@ define bfloat @v_exp10_bf16(bfloat %a) {
; GCN-LABEL: v_exp10_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, 0xc23369f4
; GCN-NEXT: s_mov_b32 s5, 0x421a209b
; GCN-NEXT: v_mov_b32_e32 v1, 0x7f800000
+; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_mul_f32_e32 v2, 0x40549000, v0
; GCN-NEXT: v_sub_f32_e32 v3, v0, v0
; GCN-NEXT: v_mul_f32_e32 v4, 0x3a2784bc, v0
@@ -18250,6 +26698,7 @@ define bfloat @v_exp10_bf16(bfloat %a) {
; GFX7-LABEL: v_exp10_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: s_mov_b32 s4, 0x40549a78
; GFX7-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
@@ -18296,6 +26745,13 @@ define bfloat @v_exp10_bf16(bfloat %a) {
; GFX8-NEXT: v_mov_b32_e32 v2, 0x7f800000
; GFX8-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -18321,6 +26777,13 @@ define bfloat @v_exp10_bf16(bfloat %a) {
; GFX9-NEXT: v_mov_b32_e32 v2, 0x7f800000
; GFX9-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -18328,6 +26791,7 @@ define bfloat @v_exp10_bf16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
; GFX10-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
; GFX10-NEXT: v_rndne_f32_e32 v2, v1
@@ -18341,6 +26805,11 @@ define bfloat @v_exp10_bf16(bfloat %a) {
; GFX10-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
; GFX10-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0
; GFX10-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -18348,6 +26817,7 @@ define bfloat @v_exp10_bf16(bfloat %a) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
; GFX11-NEXT: v_rndne_f32_e32 v2, v1
@@ -18366,7 +26836,13 @@ define bfloat @v_exp10_bf16(bfloat %a) {
; GFX11-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
; GFX11-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0
; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.exp10.bf16(bfloat %a)
@@ -18379,6 +26855,7 @@ define bfloat @v_ceil_bf16(bfloat %a) {
; GCN-LABEL: v_ceil_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_ceil_f32_e32 v0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -18387,6 +26864,7 @@ define bfloat @v_ceil_bf16(bfloat %a) {
; GFX7-LABEL: v_ceil_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_ceil_f32_e32 v0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -18397,6 +26875,13 @@ define bfloat @v_ceil_bf16(bfloat %a) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_ceil_f32_e32 v0, v0
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -18405,6 +26890,13 @@ define bfloat @v_ceil_bf16(bfloat %a) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_ceil_f32_e32 v0, v0
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -18412,7 +26904,13 @@ define bfloat @v_ceil_bf16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_ceil_f32_e32 v0, v0
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -18420,8 +26918,16 @@ define bfloat @v_ceil_bf16(bfloat %a) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_ceil_f32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.ceil.bf16(bfloat %a)
@@ -18434,6 +26940,7 @@ define bfloat @v_trunc_bf16(bfloat %a) {
; GCN-LABEL: v_trunc_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_trunc_f32_e32 v0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -18442,6 +26949,7 @@ define bfloat @v_trunc_bf16(bfloat %a) {
; GFX7-LABEL: v_trunc_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_trunc_f32_e32 v0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -18452,6 +26960,13 @@ define bfloat @v_trunc_bf16(bfloat %a) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_trunc_f32_e32 v0, v0
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -18460,6 +26975,13 @@ define bfloat @v_trunc_bf16(bfloat %a) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_trunc_f32_e32 v0, v0
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -18467,7 +26989,13 @@ define bfloat @v_trunc_bf16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_trunc_f32_e32 v0, v0
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -18475,8 +27003,16 @@ define bfloat @v_trunc_bf16(bfloat %a) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_trunc_f32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.trunc.bf16(bfloat %a)
@@ -18489,6 +27025,7 @@ define bfloat @v_rint_bf16(bfloat %a) {
; GCN-LABEL: v_rint_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_rndne_f32_e32 v0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -18497,6 +27034,7 @@ define bfloat @v_rint_bf16(bfloat %a) {
; GFX7-LABEL: v_rint_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_rndne_f32_e32 v0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -18507,6 +27045,13 @@ define bfloat @v_rint_bf16(bfloat %a) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_rndne_f32_e32 v0, v0
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -18515,6 +27060,13 @@ define bfloat @v_rint_bf16(bfloat %a) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_rndne_f32_e32 v0, v0
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -18522,7 +27074,13 @@ define bfloat @v_rint_bf16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_rndne_f32_e32 v0, v0
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -18530,8 +27088,16 @@ define bfloat @v_rint_bf16(bfloat %a) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rndne_f32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.rint.bf16(bfloat %a)
@@ -18544,6 +27110,7 @@ define bfloat @v_nearbyint_bf16(bfloat %a) {
; GCN-LABEL: v_nearbyint_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_rndne_f32_e32 v0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -18552,6 +27119,7 @@ define bfloat @v_nearbyint_bf16(bfloat %a) {
; GFX7-LABEL: v_nearbyint_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_rndne_f32_e32 v0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -18562,6 +27130,13 @@ define bfloat @v_nearbyint_bf16(bfloat %a) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_rndne_f32_e32 v0, v0
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -18570,6 +27145,13 @@ define bfloat @v_nearbyint_bf16(bfloat %a) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_rndne_f32_e32 v0, v0
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -18577,7 +27159,13 @@ define bfloat @v_nearbyint_bf16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_rndne_f32_e32 v0, v0
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -18585,8 +27173,16 @@ define bfloat @v_nearbyint_bf16(bfloat %a) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rndne_f32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.nearbyint.bf16(bfloat %a)
@@ -18599,6 +27195,7 @@ define bfloat @v_round_bf16(bfloat %a) {
; GCN-LABEL: v_round_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_trunc_f32_e32 v1, v0
; GCN-NEXT: v_sub_f32_e32 v2, v0, v1
@@ -18613,6 +27210,7 @@ define bfloat @v_round_bf16(bfloat %a) {
; GFX7-LABEL: v_round_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_trunc_f32_e32 v1, v0
; GFX7-NEXT: v_sub_f32_e32 v2, v0, v1
@@ -18635,6 +27233,13 @@ define bfloat @v_round_bf16(bfloat %a) {
; GFX8-NEXT: s_brev_b32 s4, -2
; GFX8-NEXT: v_bfi_b32 v0, s4, v2, v0
; GFX8-NEXT: v_add_f32_e32 v0, v1, v0
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -18649,6 +27254,13 @@ define bfloat @v_round_bf16(bfloat %a) {
; GFX9-NEXT: s_brev_b32 s4, -2
; GFX9-NEXT: v_bfi_b32 v0, s4, v2, v0
; GFX9-NEXT: v_add_f32_e32 v0, v1, v0
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -18660,8 +27272,14 @@ define bfloat @v_round_bf16(bfloat %a) {
; GFX10-NEXT: v_sub_f32_e32 v2, v0, v1
; GFX10-NEXT: v_cmp_ge_f32_e64 s4, |v2|, 0.5
; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s4
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0
; GFX10-NEXT: v_add_f32_e32 v0, v1, v0
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -18675,10 +27293,17 @@ define bfloat @v_round_bf16(bfloat %a) {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cmp_ge_f32_e64 s0, |v2|, 0.5
; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0
; GFX11-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.round.bf16(bfloat %a)
@@ -18691,6 +27316,7 @@ define bfloat @v_roundeven_bf16(bfloat %a) {
; GCN-LABEL: v_roundeven_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_rndne_f32_e32 v0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -18699,6 +27325,7 @@ define bfloat @v_roundeven_bf16(bfloat %a) {
; GFX7-LABEL: v_roundeven_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_rndne_f32_e32 v0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -18709,6 +27336,13 @@ define bfloat @v_roundeven_bf16(bfloat %a) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_rndne_f32_e32 v0, v0
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -18717,6 +27351,13 @@ define bfloat @v_roundeven_bf16(bfloat %a) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_rndne_f32_e32 v0, v0
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -18724,7 +27365,13 @@ define bfloat @v_roundeven_bf16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_rndne_f32_e32 v0, v0
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -18732,8 +27379,16 @@ define bfloat @v_roundeven_bf16(bfloat %a) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rndne_f32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.roundeven.bf16(bfloat %a)
@@ -18746,6 +27401,7 @@ define bfloat @v_floor_bf16(bfloat %a) {
; GCN-LABEL: v_floor_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_floor_f32_e32 v0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -18754,6 +27410,7 @@ define bfloat @v_floor_bf16(bfloat %a) {
; GFX7-LABEL: v_floor_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_floor_f32_e32 v0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -18764,6 +27421,13 @@ define bfloat @v_floor_bf16(bfloat %a) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_floor_f32_e32 v0, v0
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -18772,6 +27436,13 @@ define bfloat @v_floor_bf16(bfloat %a) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_floor_f32_e32 v0, v0
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -18779,7 +27450,13 @@ define bfloat @v_floor_bf16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_floor_f32_e32 v0, v0
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -18787,8 +27464,16 @@ define bfloat @v_floor_bf16(bfloat %a) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_floor_f32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.floor.bf16(bfloat %a)
@@ -18813,6 +27498,13 @@ define bfloat @v_canonicalize_bf16(bfloat %a) {
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -18821,6 +27513,13 @@ define bfloat @v_canonicalize_bf16(bfloat %a) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_max_f32_e32 v0, v0, v0
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -18828,7 +27527,13 @@ define bfloat @v_canonicalize_bf16(bfloat %a) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_max_f32_e32 v0, v0, v0
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -18836,8 +27541,16 @@ define bfloat @v_canonicalize_bf16(bfloat %a) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f32_e32 v0, v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.canonicalize.bf16(bfloat %a)
@@ -18896,6 +27609,8 @@ define i1 @v_fcmp_oeq_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fcmp_oeq_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_eq_f32_e32 vcc, v0, v1
@@ -18905,6 +27620,8 @@ define i1 @v_fcmp_oeq_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fcmp_oeq_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cmp_eq_f32_e32 vcc, v0, v1
@@ -18955,6 +27672,8 @@ define i1 @v_fcmp_ogt_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fcmp_ogt_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_gt_f32_e32 vcc, v0, v1
@@ -18964,6 +27683,8 @@ define i1 @v_fcmp_ogt_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fcmp_ogt_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cmp_gt_f32_e32 vcc, v0, v1
@@ -19014,6 +27735,8 @@ define i1 @v_fcmp_oge_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fcmp_oge_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_ge_f32_e32 vcc, v0, v1
@@ -19023,6 +27746,8 @@ define i1 @v_fcmp_oge_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fcmp_oge_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cmp_ge_f32_e32 vcc, v0, v1
@@ -19073,6 +27798,8 @@ define i1 @v_fcmp_olt_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fcmp_olt_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_lt_f32_e32 vcc, v0, v1
@@ -19082,6 +27809,8 @@ define i1 @v_fcmp_olt_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fcmp_olt_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cmp_lt_f32_e32 vcc, v0, v1
@@ -19132,6 +27861,8 @@ define i1 @v_fcmp_ole_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fcmp_ole_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_le_f32_e32 vcc, v0, v1
@@ -19141,6 +27872,8 @@ define i1 @v_fcmp_ole_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fcmp_ole_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cmp_le_f32_e32 vcc, v0, v1
@@ -19191,6 +27924,8 @@ define i1 @v_fcmp_one_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fcmp_one_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_lg_f32_e32 vcc, v0, v1
@@ -19200,6 +27935,8 @@ define i1 @v_fcmp_one_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fcmp_one_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cmp_lg_f32_e32 vcc, v0, v1
@@ -19250,6 +27987,8 @@ define i1 @v_fcmp_uno_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fcmp_uno_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_u_f32_e32 vcc, v0, v1
@@ -19259,6 +27998,8 @@ define i1 @v_fcmp_uno_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fcmp_uno_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cmp_u_f32_e32 vcc, v0, v1
@@ -19309,6 +28050,8 @@ define i1 @v_fcmp_ueq_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fcmp_ueq_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_nlg_f32_e32 vcc, v0, v1
@@ -19318,6 +28061,8 @@ define i1 @v_fcmp_ueq_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fcmp_ueq_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cmp_nlg_f32_e32 vcc, v0, v1
@@ -19368,6 +28113,8 @@ define i1 @v_fcmp_ugt_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fcmp_ugt_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_nle_f32_e32 vcc, v0, v1
@@ -19377,6 +28124,8 @@ define i1 @v_fcmp_ugt_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fcmp_ugt_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cmp_nle_f32_e32 vcc, v0, v1
@@ -19427,6 +28176,8 @@ define i1 @v_fcmp_uge_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fcmp_uge_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_nlt_f32_e32 vcc, v0, v1
@@ -19436,6 +28187,8 @@ define i1 @v_fcmp_uge_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fcmp_uge_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cmp_nlt_f32_e32 vcc, v0, v1
@@ -19486,6 +28239,8 @@ define i1 @v_fcmp_ult_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fcmp_ult_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_nge_f32_e32 vcc, v0, v1
@@ -19495,6 +28250,8 @@ define i1 @v_fcmp_ult_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fcmp_ult_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cmp_nge_f32_e32 vcc, v0, v1
@@ -19545,6 +28302,8 @@ define i1 @v_fcmp_ule_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fcmp_ule_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_ngt_f32_e32 vcc, v0, v1
@@ -19554,6 +28313,8 @@ define i1 @v_fcmp_ule_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fcmp_ule_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cmp_ngt_f32_e32 vcc, v0, v1
@@ -19604,6 +28365,8 @@ define i1 @v_fcmp_une_bf16(bfloat %a, bfloat %b) {
; GCN-LABEL: v_fcmp_une_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cmp_neq_f32_e32 vcc, v0, v1
@@ -19613,6 +28376,8 @@ define i1 @v_fcmp_une_bf16(bfloat %a, bfloat %b) {
; GFX7-LABEL: v_fcmp_une_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cmp_neq_f32_e32 vcc, v0, v1
@@ -19705,6 +28470,7 @@ define bfloat @v_copysign_bf16_bf16(bfloat %mag, bfloat %sign) {
; GCN-LABEL: v_copysign_bf16_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v1, 0x80000000, v1
; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: v_bfe_u32 v0, v0, 16, 15
@@ -19715,6 +28481,7 @@ define bfloat @v_copysign_bf16_bf16(bfloat %mag, bfloat %sign) {
; GFX7-LABEL: v_copysign_bf16_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0x80000000, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 15
@@ -19762,6 +28529,7 @@ define bfloat @v_copysign_bf16_s_bf16(bfloat %mag, bfloat inreg %sign) {
; GCN-LABEL: v_copysign_bf16_s_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_and_b32 s4, s4, 0x80000000
; GCN-NEXT: s_lshr_b32 s4, s4, 16
; GCN-NEXT: v_bfe_u32 v0, v0, 16, 15
@@ -19772,6 +28540,7 @@ define bfloat @v_copysign_bf16_s_bf16(bfloat %mag, bfloat inreg %sign) {
; GFX7-LABEL: v_copysign_bf16_s_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: s_and_b32 s4, s4, 0x80000000
; GFX7-NEXT: s_lshr_b32 s4, s4, 16
; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 15
@@ -19821,20 +28590,22 @@ define bfloat @v_copysign_s_bf16_bf16(bfloat inreg %mag, bfloat %sign) {
; GCN-LABEL: v_copysign_s_bf16_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e64 v1, 1.0, s4
; GCN-NEXT: v_and_b32_e32 v0, 0x80000000, v0
; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GCN-NEXT: s_bfe_u32 s4, s4, 0xf0010
-; GCN-NEXT: v_or_b32_e32 v0, s4, v0
+; GCN-NEXT: v_bfe_u32 v1, v1, 16, 15
+; GCN-NEXT: v_or_b32_e32 v0, v1, v0
; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_copysign_s_bf16_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e64 v1, 1.0, s4
; GFX7-NEXT: v_and_b32_e32 v0, 0x80000000, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_bfe_u32 s4, s4, 0xf0010
-; GFX7-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX7-NEXT: v_bfe_u32 v1, v1, 16, 15
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -19880,6 +28651,7 @@ define bfloat @v_copysign_bf16_f32(bfloat %mag, float %sign.f32) {
; GCN-LABEL: v_copysign_bf16_f32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v1, 0x80000000, v1
; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: v_bfe_u32 v0, v0, 16, 15
@@ -19890,6 +28662,7 @@ define bfloat @v_copysign_bf16_f32(bfloat %mag, float %sign.f32) {
; GFX7-LABEL: v_copysign_bf16_f32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0x80000000, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 15
@@ -19939,6 +28712,7 @@ define bfloat @v_copysign_bf16_f64(bfloat %mag, double %sign.f64) {
; GCN-LABEL: v_copysign_bf16_f64:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v1, 0x80000000, v2
; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: v_bfe_u32 v0, v0, 16, 15
@@ -19949,6 +28723,7 @@ define bfloat @v_copysign_bf16_f64(bfloat %mag, double %sign.f64) {
; GFX7-LABEL: v_copysign_bf16_f64:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0x80000000, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 15
@@ -19998,6 +28773,7 @@ define bfloat @v_copysign_bf16_f16(bfloat %mag, half %sign.f16) {
; GCN-LABEL: v_copysign_bf16_f16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_cvt_f16_f32_e32 v1, v1
; GCN-NEXT: v_and_b32_e32 v1, 0x8000, v1
; GCN-NEXT: v_bfe_u32 v0, v0, 16, 15
@@ -20009,6 +28785,7 @@ define bfloat @v_copysign_bf16_f16(bfloat %mag, half %sign.f16) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7-NEXT: v_and_b32_e32 v1, 0x8000, v1
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
@@ -20055,18 +28832,22 @@ define bfloat @v_copysign_bf16_f16(bfloat %mag, half %sign.f16) {
define amdgpu_ps i32 @s_copysign_bf16_bf16(bfloat inreg %mag, bfloat inreg %sign) {
; GCN-LABEL: s_copysign_bf16_bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_and_b32 s1, s1, 0x80000000
-; GCN-NEXT: s_lshr_b32 s1, s1, 16
-; GCN-NEXT: s_bfe_u32 s0, s0, 0xf0010
-; GCN-NEXT: s_or_b32 s0, s0, s1
+; GCN-NEXT: v_mul_f32_e64 v0, 1.0, s0
+; GCN-NEXT: s_and_b32 s0, s1, 0x80000000
+; GCN-NEXT: s_lshr_b32 s0, s0, 16
+; GCN-NEXT: v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT: v_or_b32_e32 v0, s0, v0
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
; GCN-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: s_copysign_bf16_bf16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_and_b32 s1, s1, 0x80000000
-; GFX7-NEXT: s_lshr_b32 s1, s1, 16
-; GFX7-NEXT: s_bfe_u32 s0, s0, 0xf0010
-; GFX7-NEXT: s_or_b32 s0, s0, s1
+; GFX7-NEXT: v_mul_f32_e64 v0, 1.0, s0
+; GFX7-NEXT: s_and_b32 s0, s1, 0x80000000
+; GFX7-NEXT: s_lshr_b32 s0, s0, 16
+; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 15
+; GFX7-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_copysign_bf16_bf16:
@@ -20120,18 +28901,22 @@ define amdgpu_ps i32 @s_copysign_bf16_bf16(bfloat inreg %mag, bfloat inreg %sign
define amdgpu_ps i32 @s_copysign_bf16_f32(bfloat inreg %mag, float inreg %sign.f32) {
; GCN-LABEL: s_copysign_bf16_f32:
; GCN: ; %bb.0:
-; GCN-NEXT: s_and_b32 s1, s1, 0x80000000
-; GCN-NEXT: s_lshr_b32 s1, s1, 16
-; GCN-NEXT: s_bfe_u32 s0, s0, 0xf0010
-; GCN-NEXT: s_or_b32 s0, s0, s1
+; GCN-NEXT: v_mul_f32_e64 v0, 1.0, s0
+; GCN-NEXT: s_and_b32 s0, s1, 0x80000000
+; GCN-NEXT: s_lshr_b32 s0, s0, 16
+; GCN-NEXT: v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT: v_or_b32_e32 v0, s0, v0
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
; GCN-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: s_copysign_bf16_f32:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_and_b32 s1, s1, 0x80000000
-; GFX7-NEXT: s_lshr_b32 s1, s1, 16
-; GFX7-NEXT: s_bfe_u32 s0, s0, 0xf0010
-; GFX7-NEXT: s_or_b32 s0, s0, s1
+; GFX7-NEXT: v_mul_f32_e64 v0, 1.0, s0
+; GFX7-NEXT: s_and_b32 s0, s1, 0x80000000
+; GFX7-NEXT: s_lshr_b32 s0, s0, 16
+; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 15
+; GFX7-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_copysign_bf16_f32:
@@ -20189,18 +28974,22 @@ define amdgpu_ps i32 @s_copysign_bf16_f32(bfloat inreg %mag, float inreg %sign.f
define amdgpu_ps i32 @s_copysign_bf16_f64(bfloat inreg %mag, double inreg %sign.f64) {
; GCN-LABEL: s_copysign_bf16_f64:
; GCN: ; %bb.0:
-; GCN-NEXT: s_and_b32 s1, s2, 0x80000000
-; GCN-NEXT: s_lshr_b32 s1, s1, 16
-; GCN-NEXT: s_bfe_u32 s0, s0, 0xf0010
-; GCN-NEXT: s_or_b32 s0, s0, s1
+; GCN-NEXT: v_mul_f32_e64 v0, 1.0, s0
+; GCN-NEXT: s_and_b32 s0, s2, 0x80000000
+; GCN-NEXT: s_lshr_b32 s0, s0, 16
+; GCN-NEXT: v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT: v_or_b32_e32 v0, s0, v0
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
; GCN-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: s_copysign_bf16_f64:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_and_b32 s1, s2, 0x80000000
-; GFX7-NEXT: s_lshr_b32 s1, s1, 16
-; GFX7-NEXT: s_bfe_u32 s0, s0, 0xf0010
-; GFX7-NEXT: s_or_b32 s0, s0, s1
+; GFX7-NEXT: v_mul_f32_e64 v0, 1.0, s0
+; GFX7-NEXT: s_and_b32 s0, s2, 0x80000000
+; GFX7-NEXT: s_lshr_b32 s0, s0, 16
+; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 15
+; GFX7-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_copysign_bf16_f64:
@@ -20258,19 +29047,21 @@ define amdgpu_ps i32 @s_copysign_bf16_f64(bfloat inreg %mag, double inreg %sign.
define amdgpu_ps i32 @s_copysign_bf16_f16(bfloat inreg %mag, half inreg %sign.f16) {
; GCN-LABEL: s_copysign_bf16_f16:
; GCN: ; %bb.0:
-; GCN-NEXT: v_cvt_f16_f32_e32 v0, s1
-; GCN-NEXT: v_and_b32_e32 v0, 0x8000, v0
-; GCN-NEXT: s_bfe_u32 s0, s0, 0xf0010
-; GCN-NEXT: v_or_b32_e32 v0, s0, v0
+; GCN-NEXT: v_mul_f32_e64 v0, 1.0, s0
+; GCN-NEXT: v_cvt_f16_f32_e32 v1, s1
+; GCN-NEXT: v_and_b32_e32 v1, 0x8000, v1
+; GCN-NEXT: v_bfe_u32 v0, v0, 16, 15
+; GCN-NEXT: v_or_b32_e32 v0, v0, v1
; GCN-NEXT: v_readfirstlane_b32 s0, v0
; GCN-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: s_copysign_bf16_f16:
; GFX7: ; %bb.0:
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, s1
-; GFX7-NEXT: s_bfe_u32 s0, s0, 0xf0010
+; GFX7-NEXT: v_mul_f32_e64 v1, 1.0, s0
+; GFX7-NEXT: v_bfe_u32 v1, v1, 16, 15
; GFX7-NEXT: v_and_b32_e32 v0, 0x8000, v0
-; GFX7-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
;
@@ -20439,8 +29230,9 @@ define half @v_copysign_f16_bf16(half %mag, bfloat %sign.bf16) {
; GCN-LABEL: v_copysign_f16_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: v_cvt_f32_f16_e32 v0, v0
; GCN-NEXT: v_cvt_f32_f16_e32 v1, v1
; GCN-NEXT: s_brev_b32 s4, -2
@@ -20451,10 +29243,11 @@ define half @v_copysign_f16_bf16(half %mag, bfloat %sign.bf16) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: s_brev_b32 s4, -2
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT: s_brev_b32 s4, -2
; GFX7-NEXT: v_bfi_b32 v0, s4, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -20491,12 +29284,13 @@ define half @v_copysign_f16_bf16(half %mag, bfloat %sign.bf16) {
define amdgpu_ps i32 @s_copysign_f16_bf16(half inreg %mag, bfloat inreg %sign.bf16) {
; GCN-LABEL: s_copysign_f16_bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_lshr_b32 s1, s1, 16
-; GCN-NEXT: v_cvt_f16_f32_e32 v0, s0
+; GCN-NEXT: v_mul_f32_e64 v0, 1.0, s1
+; GCN-NEXT: v_cvt_f16_f32_e32 v1, s0
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_cvt_f32_f16_e32 v1, v1
; GCN-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GCN-NEXT: v_cvt_f32_f16_e32 v1, s1
; GCN-NEXT: s_brev_b32 s0, -2
-; GCN-NEXT: v_bfi_b32 v0, s0, v0, v1
+; GCN-NEXT: v_bfi_b32 v0, s0, v1, v0
; GCN-NEXT: v_cvt_f16_f32_e32 v0, v0
; GCN-NEXT: v_readfirstlane_b32 s0, v0
; GCN-NEXT: ; return to shader part epilog
@@ -20504,10 +29298,11 @@ define amdgpu_ps i32 @s_copysign_f16_bf16(half inreg %mag, bfloat inreg %sign.bf
; GFX7-LABEL: s_copysign_f16_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, s0
-; GFX7-NEXT: s_lshr_b32 s0, s1, 16
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, s0
-; GFX7-NEXT: s_brev_b32 s0, -2
+; GFX7-NEXT: v_mul_f32_e64 v1, 1.0, s1
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT: s_brev_b32 s0, -2
; GFX7-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
@@ -20677,6 +29472,7 @@ define i16 @v_fptosi_bf16_to_i16(bfloat %x) {
; GCN-LABEL: v_fptosi_bf16_to_i16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -20684,6 +29480,7 @@ define i16 @v_fptosi_bf16_to_i16(bfloat %x) {
; GFX7-LABEL: v_fptosi_bf16_to_i16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -20724,6 +29521,8 @@ define <2 x i16> @v_fptosi_v2bf16_to_v2i16(<2 x bfloat> %x) {
; GCN-LABEL: v_fptosi_v2bf16_to_v2i16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cvt_i32_f32_e32 v1, v1
@@ -20737,6 +29536,8 @@ define <2 x i16> @v_fptosi_v2bf16_to_v2i16(<2 x bfloat> %x) {
; GFX7-LABEL: v_fptosi_v2bf16_to_v2i16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cvt_i32_f32_e32 v1, v1
@@ -20797,6 +29598,9 @@ define <3 x i16> @v_fptosi_v3bf16_to_v3i16(<3 x bfloat> %x) {
; GCN-LABEL: v_fptosi_v3bf16_to_v3i16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -20813,6 +29617,9 @@ define <3 x i16> @v_fptosi_v3bf16_to_v3i16(<3 x bfloat> %x) {
; GFX7-LABEL: v_fptosi_v3bf16_to_v3i16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cvt_i32_f32_e32 v1, v1
@@ -20884,6 +29691,10 @@ define <4 x i16> @v_fptosi_v4bf16_to_v4i16(<4 x bfloat> %x) {
; GCN-LABEL: v_fptosi_v4bf16_to_v4i16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
@@ -20905,6 +29716,10 @@ define <4 x i16> @v_fptosi_v4bf16_to_v4i16(<4 x bfloat> %x) {
; GFX7-LABEL: v_fptosi_v4bf16_to_v4i16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
@@ -20994,6 +29809,7 @@ define i32 @v_fptosi_bf16_to_i32(bfloat %x) {
; GCN-LABEL: v_fptosi_bf16_to_i32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_cvt_i32_f32_e32 v0, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -21001,6 +29817,7 @@ define i32 @v_fptosi_bf16_to_i32(bfloat %x) {
; GFX7-LABEL: v_fptosi_bf16_to_i32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -21041,6 +29858,8 @@ define <2 x i32> @v_fptosi_v2bf16_to_v2i32(<2 x bfloat> %x) {
; GCN-LABEL: v_fptosi_v2bf16_to_v2i32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_cvt_i32_f32_e32 v0, v0
@@ -21050,6 +29869,8 @@ define <2 x i32> @v_fptosi_v2bf16_to_v2i32(<2 x bfloat> %x) {
; GFX7-LABEL: v_fptosi_v2bf16_to_v2i32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_cvt_i32_f32_e32 v0, v0
@@ -21102,6 +29923,9 @@ define <3 x i32> @v_fptosi_v3bf16_to_v3i32(<3 x bfloat> %x) {
; GCN-LABEL: v_fptosi_v3bf16_to_v3i32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -21113,6 +29937,9 @@ define <3 x i32> @v_fptosi_v3bf16_to_v3i32(<3 x bfloat> %x) {
; GFX7-LABEL: v_fptosi_v3bf16_to_v3i32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -21178,6 +30005,10 @@ define <4 x i32> @v_fptosi_v4bf16_to_v4i32(<4 x bfloat> %x) {
; GCN-LABEL: v_fptosi_v4bf16_to_v4i32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -21191,6 +30022,10 @@ define <4 x i32> @v_fptosi_v4bf16_to_v4i32(<4 x bfloat> %x) {
; GFX7-LABEL: v_fptosi_v4bf16_to_v4i32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -21266,9 +30101,10 @@ define i64 @v_fptosi_bf16_to_i64(bfloat %x) {
; GCN-LABEL: v_fptosi_bf16_to_i64:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, 0x2f800000
; GCN-NEXT: s_mov_b32 s5, 0xcf800000
+; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_trunc_f32_e32 v0, v0
; GCN-NEXT: v_mul_f32_e64 v1, |v0|, s4
; GCN-NEXT: v_ashrrev_i32_e32 v2, 31, v0
@@ -21285,6 +30121,7 @@ define i64 @v_fptosi_bf16_to_i64(bfloat %x) {
; GFX7-LABEL: v_fptosi_bf16_to_i64:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_trunc_f32_e32 v0, v0
; GFX7-NEXT: s_mov_b32 s4, 0x2f800000
@@ -21385,9 +30222,11 @@ define <2 x i64> @v_fptosi_v2bf16_to_v2i64(<2 x bfloat> %x) {
; GCN-LABEL: v_fptosi_v2bf16_to_v2i64:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, 0x2f800000
; GCN-NEXT: s_mov_b32 s5, 0xcf800000
+; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_trunc_f32_e32 v0, v0
; GCN-NEXT: v_trunc_f32_e32 v1, v1
@@ -21416,6 +30255,7 @@ define <2 x i64> @v_fptosi_v2bf16_to_v2i64(<2 x bfloat> %x) {
; GFX7-LABEL: v_fptosi_v2bf16_to_v2i64:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_trunc_f32_e32 v0, v0
; GFX7-NEXT: s_mov_b32 s4, 0x2f800000
@@ -21424,13 +30264,14 @@ define <2 x i64> @v_fptosi_v2bf16_to_v2i64(<2 x bfloat> %x) {
; GFX7-NEXT: s_mov_b32 s5, 0xcf800000
; GFX7-NEXT: v_fma_f32 v3, v2, s5, |v0|
; GFX7-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_ashrrev_i32_e32 v4, 31, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX7-NEXT: v_xor_b32_e32 v0, v3, v4
; GFX7-NEXT: v_trunc_f32_e32 v3, v1
; GFX7-NEXT: v_mul_f32_e64 v1, |v3|, s4
; GFX7-NEXT: v_floor_f32_e32 v1, v1
+; GFX7-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX7-NEXT: v_fma_f32 v5, v1, s5, |v3|
; GFX7-NEXT: v_cvt_u32_f32_e32 v5, v5
; GFX7-NEXT: v_cvt_u32_f32_e32 v6, v1
@@ -21578,9 +30419,12 @@ define <3 x i64> @v_fptosi_v3bf16_to_v3i64(<3 x bfloat> %x) {
; GCN-LABEL: v_fptosi_v3bf16_to_v3i64:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, 0x2f800000
; GCN-NEXT: s_mov_b32 s5, 0xcf800000
+; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GCN-NEXT: v_trunc_f32_e32 v0, v0
@@ -21621,6 +30465,7 @@ define <3 x i64> @v_fptosi_v3bf16_to_v3i64(<3 x bfloat> %x) {
; GFX7-LABEL: v_fptosi_v3bf16_to_v3i64:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_trunc_f32_e32 v0, v0
; GFX7-NEXT: s_mov_b32 s4, 0x2f800000
@@ -21629,15 +30474,17 @@ define <3 x i64> @v_fptosi_v3bf16_to_v3i64(<3 x bfloat> %x) {
; GFX7-NEXT: s_mov_b32 s5, 0xcf800000
; GFX7-NEXT: v_fma_f32 v4, v3, s5, |v0|
; GFX7-NEXT: v_cvt_u32_f32_e32 v4, v4
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_ashrrev_i32_e32 v5, 31, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX7-NEXT: v_xor_b32_e32 v0, v4, v5
; GFX7-NEXT: v_trunc_f32_e32 v4, v1
; GFX7-NEXT: v_mul_f32_e64 v1, |v4|, s4
+; GFX7-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX7-NEXT: v_floor_f32_e32 v1, v1
; GFX7-NEXT: v_fma_f32 v6, v1, s5, |v4|
; GFX7-NEXT: v_cvt_u32_f32_e32 v6, v6
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_xor_b32_e32 v3, v3, v5
; GFX7-NEXT: v_sub_i32_e32 v0, vcc, v0, v5
; GFX7-NEXT: v_cvt_u32_f32_e32 v7, v1
@@ -21843,9 +30690,13 @@ define <4 x i64> @v_fptosi_v4bf16_to_v4i64(<4 x bfloat> %x) {
; GCN-LABEL: v_fptosi_v4bf16_to_v4i64:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GCN-NEXT: s_mov_b32 s4, 0x2f800000
; GCN-NEXT: s_mov_b32 s5, 0xcf800000
+; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
@@ -21898,57 +30749,60 @@ define <4 x i64> @v_fptosi_v4bf16_to_v4i64(<4 x bfloat> %x) {
; GFX7-LABEL: v_fptosi_v4bf16_to_v4i64:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_trunc_f32_e32 v0, v0
; GFX7-NEXT: s_mov_b32 s4, 0x2f800000
-; GFX7-NEXT: v_mul_f32_e64 v4, |v0|, s4
-; GFX7-NEXT: v_floor_f32_e32 v4, v4
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e64 v3, |v0|, s4
+; GFX7-NEXT: v_floor_f32_e32 v3, v3
; GFX7-NEXT: s_mov_b32 s5, 0xcf800000
-; GFX7-NEXT: v_fma_f32 v5, v4, s5, |v0|
+; GFX7-NEXT: v_fma_f32 v5, v3, s5, |v0|
; GFX7-NEXT: v_cvt_u32_f32_e32 v5, v5
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_ashrrev_i32_e32 v6, 31, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_cvt_u32_f32_e32 v4, v4
; GFX7-NEXT: v_xor_b32_e32 v0, v5, v6
; GFX7-NEXT: v_trunc_f32_e32 v5, v1
; GFX7-NEXT: v_mul_f32_e64 v1, |v5|, s4
+; GFX7-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX7-NEXT: v_floor_f32_e32 v1, v1
; GFX7-NEXT: v_fma_f32 v7, v1, s5, |v5|
; GFX7-NEXT: v_cvt_u32_f32_e32 v7, v7
-; GFX7-NEXT: v_xor_b32_e32 v4, v4, v6
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_xor_b32_e32 v3, v3, v6
; GFX7-NEXT: v_sub_i32_e32 v0, vcc, v0, v6
; GFX7-NEXT: v_cvt_u32_f32_e32 v8, v1
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v4, v6, vcc
-; GFX7-NEXT: v_ashrrev_i32_e32 v4, 31, v5
+; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GFX7-NEXT: v_ashrrev_i32_e32 v3, 31, v5
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: v_xor_b32_e32 v6, v7, v4
+; GFX7-NEXT: v_xor_b32_e32 v6, v7, v3
; GFX7-NEXT: v_trunc_f32_e32 v7, v2
; GFX7-NEXT: v_mul_f32_e64 v2, |v7|, s4
; GFX7-NEXT: v_floor_f32_e32 v2, v2
-; GFX7-NEXT: v_xor_b32_e32 v5, v8, v4
+; GFX7-NEXT: v_xor_b32_e32 v5, v8, v3
; GFX7-NEXT: v_fma_f32 v8, v2, s5, |v7|
-; GFX7-NEXT: v_cvt_u32_f32_e32 v9, v8
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX7-NEXT: v_cvt_u32_f32_e32 v10, v2
-; GFX7-NEXT: v_sub_i32_e32 v2, vcc, v6, v4
-; GFX7-NEXT: v_trunc_f32_e32 v3, v3
-; GFX7-NEXT: v_subb_u32_e32 v8, vcc, v5, v4, vcc
+; GFX7-NEXT: v_cvt_u32_f32_e32 v8, v8
+; GFX7-NEXT: v_cvt_u32_f32_e32 v9, v2
+; GFX7-NEXT: v_sub_i32_e32 v2, vcc, v6, v3
+; GFX7-NEXT: v_subb_u32_e32 v3, vcc, v5, v3, vcc
; GFX7-NEXT: v_ashrrev_i32_e32 v5, 31, v7
-; GFX7-NEXT: v_mul_f32_e64 v7, |v3|, s4
-; GFX7-NEXT: v_floor_f32_e32 v7, v7
-; GFX7-NEXT: v_xor_b32_e32 v4, v9, v5
-; GFX7-NEXT: v_fma_f32 v9, v7, s5, |v3|
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: v_xor_b32_e32 v7, v8, v5
+; GFX7-NEXT: v_trunc_f32_e32 v8, v4
+; GFX7-NEXT: v_mul_f32_e64 v4, |v8|, s4
+; GFX7-NEXT: v_floor_f32_e32 v4, v4
+; GFX7-NEXT: v_xor_b32_e32 v6, v9, v5
+; GFX7-NEXT: v_fma_f32 v9, v4, s5, |v8|
; GFX7-NEXT: v_cvt_u32_f32_e32 v9, v9
-; GFX7-NEXT: v_cvt_u32_f32_e32 v7, v7
-; GFX7-NEXT: v_xor_b32_e32 v6, v10, v5
-; GFX7-NEXT: v_sub_i32_e32 v4, vcc, v4, v5
-; GFX7-NEXT: v_ashrrev_i32_e32 v3, 31, v3
+; GFX7-NEXT: v_cvt_u32_f32_e32 v10, v4
+; GFX7-NEXT: v_sub_i32_e32 v4, vcc, v7, v5
+; GFX7-NEXT: v_ashrrev_i32_e32 v7, 31, v8
; GFX7-NEXT: v_subb_u32_e32 v5, vcc, v6, v5, vcc
-; GFX7-NEXT: v_xor_b32_e32 v6, v9, v3
-; GFX7-NEXT: v_xor_b32_e32 v7, v7, v3
-; GFX7-NEXT: v_sub_i32_e32 v6, vcc, v6, v3
-; GFX7-NEXT: v_subb_u32_e32 v7, vcc, v7, v3, vcc
-; GFX7-NEXT: v_mov_b32_e32 v3, v8
+; GFX7-NEXT: v_xor_b32_e32 v6, v9, v7
+; GFX7-NEXT: v_xor_b32_e32 v8, v10, v7
+; GFX7-NEXT: v_sub_i32_e32 v6, vcc, v6, v7
+; GFX7-NEXT: v_subb_u32_e32 v7, vcc, v8, v7, vcc
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fptosi_v4bf16_to_v4i64:
@@ -22198,6 +31052,13 @@ define bfloat @v_sitofp_i16_to_bf16(i16 %x) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_and_b32_e32 v1, 0x80000000, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_or_b32_e32 v1, 0x400000, v1
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -22205,6 +31066,13 @@ define bfloat @v_sitofp_i16_to_bf16(i16 %x) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -22212,6 +31080,12 @@ define bfloat @v_sitofp_i16_to_bf16(i16 %x) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -22219,8 +31093,16 @@ define bfloat @v_sitofp_i16_to_bf16(i16 %x) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = sitofp i16 %x to bfloat
@@ -22253,39 +31135,90 @@ define <2 x bfloat> @v_sitofp_v2i16_to_v2bf16(<2 x i16> %x) {
; GFX8-LABEL: v_sitofp_v2i16_to_v2bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_i32_sdwa v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX8-NEXT: v_cvt_f32_i32_sdwa v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX8-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_sitofp_v2i16_to_v2bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v2, v2, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_sitofp_v2i16_to_v2bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v1, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_and_or_b32 v5, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_sitofp_v2i16_to_v2bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_ashrrev_i32_e32 v1, 16, v0
-; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-NEXT: v_bfe_i32 v1, v0, 0, 16
+; GFX11-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v1, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_and_or_b32 v5, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = sitofp <2 x i16> %x to <2 x bfloat>
ret <2 x bfloat> %op
@@ -22323,32 +31256,89 @@ define <3 x bfloat> @v_sitofp_v3i16_to_v3bf16(<3 x i16> %x) {
; GFX8-LABEL: v_sitofp_v3i16_to_v3bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX8-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX8-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX8-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX8-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v4
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v4
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX8-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v0
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16
+; GFX8-NEXT: v_alignbit_b32 v0, v0, v2, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_sitofp_v3i16_to_v3bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
; GFX9-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v2, v2, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v2, v2, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: v_perm_b32 v0, v2, v0, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v2, s4
; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_sitofp_v3i16_to_v3bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX10-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v2, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v4, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc_lo
; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = sitofp <3 x i16> %x to <3 x bfloat>
@@ -22393,55 +31383,150 @@ define <4 x bfloat> @v_sitofp_v4i16_to_v4bf16(<4 x i16> %x) {
; GFX8-LABEL: v_sitofp_v4i16_to_v4bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_i32_sdwa v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_cvt_f32_i32_sdwa v3, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX8-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX8-NEXT: v_alignbit_b32 v1, v2, v1, 16
+; GFX8-NEXT: v_cvt_f32_i32_sdwa v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX8-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v2
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v2
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX8-NEXT: v_bfe_u32 v4, v5, 16, 1
+; GFX8-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v5
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v5
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX8-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
+; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_sitofp_v4i16_to_v4bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX9-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v3, v3, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v5
+; GFX9-NEXT: v_add3_u32 v3, v3, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
-; GFX9-NEXT: v_perm_b32 v0, v3, v0, s4
-; GFX9-NEXT: v_perm_b32 v1, v2, v1, s4
+; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
+; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_sitofp_v4i16_to_v4bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX10-NEXT: v_perm_b32 v0, v3, v0, 0x7060302
-; GFX10-NEXT: v_perm_b32 v1, v2, v1, 0x7060302
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_and_or_b32 v6, v3, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX10-NEXT: v_and_or_b32 v11, v0, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_add3_u32 v10, v10, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v9, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v7, v6, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc_lo
+; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_sitofp_v4i16_to_v4bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_ashrrev_i32_e32 v2, 16, v1
-; GFX11-NEXT: v_ashrrev_i32_e32 v3, 16, v0
-; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_bfe_i32 v2, v1, 0, 16
+; GFX11-NEXT: v_bfe_i32 v3, v0, 0, 16
+; GFX11-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: v_ashrrev_i32_e32 v1, 16, v1
; GFX11-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX11-NEXT: v_cvt_f32_i32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_cvt_f32_i32_e32 v1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v0, v3, v0, 0x7060302
-; GFX11-NEXT: v_perm_b32 v1, v2, v1, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v2, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_and_or_b32 v6, v3, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-NEXT: v_and_or_b32 v11, v0, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v10, v10, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v9, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v6, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v0, v0, v3, 0x7060302
+; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = sitofp <4 x i16> %x to <4 x bfloat>
ret <4 x bfloat> %op
@@ -22466,6 +31551,13 @@ define bfloat @v_sitofp_i32_to_bf16(i32 %x) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_and_b32_e32 v1, 0x80000000, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_or_b32_e32 v1, 0x400000, v1
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -22473,6 +31565,13 @@ define bfloat @v_sitofp_i32_to_bf16(i32 %x) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -22480,6 +31579,12 @@ define bfloat @v_sitofp_i32_to_bf16(i32 %x) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -22487,7 +31592,14 @@ define bfloat @v_sitofp_i32_to_bf16(i32 %x) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = sitofp i32 %x to bfloat
@@ -22516,8 +31628,22 @@ define <2 x bfloat> @v_sitofp_v2i32_to_v2bf16(<2 x i32> %x) {
; GFX8-LABEL: v_sitofp_v2i32_to_v2bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX8-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX8-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v0
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -22527,6 +31653,19 @@ define <2 x bfloat> @v_sitofp_v2i32_to_v2bf16(<2 x i32> %x) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX9-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v2, v2, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -22536,6 +31675,17 @@ define <2 x bfloat> @v_sitofp_v2i32_to_v2bf16(<2 x i32> %x) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX10-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_and_or_b32 v5, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -22544,6 +31694,19 @@ define <2 x bfloat> @v_sitofp_v2i32_to_v2bf16(<2 x i32> %x) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX11-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_and_or_b32 v5, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -22578,19 +31741,60 @@ define <3 x bfloat> @v_sitofp_v3i32_to_v3bf16(<3 x i32> %x) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX8-NEXT: v_cvt_f32_i32_e32 v3, v1
; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16
+; GFX8-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX8-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v2
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v2
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
+; GFX8-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
+; GFX8-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v1
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_sitofp_v3i32_to_v3bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_cvt_f32_i32_e32 v1, v1
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v3, v3, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4
; GFX9-NEXT: v_alignbit_b32 v1, s4, v2, 16
@@ -22601,8 +31805,24 @@ define <3 x bfloat> @v_sitofp_v3i32_to_v3bf16(<3 x i32> %x) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX10-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v4, v2, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc_lo
; GFX10-NEXT: v_alignbit_b32 v1, s4, v2, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = sitofp <3 x i32> %x to <3 x bfloat>
@@ -22639,10 +31859,39 @@ define <4 x bfloat> @v_sitofp_v4i32_to_v4bf16(<4 x i32> %x) {
; GFX8-LABEL: v_sitofp_v4i32_to_v4bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX8-NEXT: v_cvt_f32_i32_e32 v3, v3
-; GFX8-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX8-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v2
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v3
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v4, vcc
+; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX8-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v0
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc
+; GFX8-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v1
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16
@@ -22653,9 +31902,34 @@ define <4 x bfloat> @v_sitofp_v4i32_to_v4bf16(<4 x i32> %x) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, v3
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v4, v4, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX9-NEXT: v_cvt_f32_i32_e32 v1, v1
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, v3
+; GFX9-NEXT: v_add3_u32 v4, v4, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4
; GFX9-NEXT: v_perm_b32 v1, v3, v2, s4
@@ -22666,9 +31940,30 @@ define <4 x bfloat> @v_sitofp_v4i32_to_v4bf16(<4 x i32> %x) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX10-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX10-NEXT: v_cvt_f32_i32_e32 v3, v3
+; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_and_or_b32 v9, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_and_or_b32 v11, v1, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v4, v10, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v6, v3, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v7, v3, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v11, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX10-NEXT: v_perm_b32 v1, v3, v2, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -22677,10 +31972,32 @@ define <4 x bfloat> @v_sitofp_v4i32_to_v4bf16(<4 x i32> %x) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: v_cvt_f32_i32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX11-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v2, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v9, v0, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-NEXT: v_and_or_b32 v11, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_and_or_b32 v6, v3, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v4, v10, v1, 0x7fff
+; GFX11-NEXT: v_add3_u32 v5, v7, v3, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v4, v11, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_perm_b32 v1, v3, v2, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = sitofp <4 x i32> %x to <4 x bfloat>
@@ -22739,6 +32056,13 @@ define bfloat @v_sitofp_i64_to_bf16(i64 %x) {
; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX8-NEXT: v_sub_u32_e32 v1, vcc, 32, v2
; GFX8-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -22752,11 +32076,18 @@ define bfloat @v_sitofp_i64_to_bf16(i64 %x) {
; GFX9-NEXT: v_add_u32_e32 v3, -1, v3
; GFX9-NEXT: v_min_u32_e32 v2, v3, v2
; GFX9-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX9-NEXT: v_sub_u32_e32 v1, 32, v2
; GFX9-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -22765,6 +32096,7 @@ define bfloat @v_sitofp_i64_to_bf16(i64 %x) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_xor_b32_e32 v2, v0, v1
; GFX10-NEXT: v_ffbh_i32_e32 v3, v1
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_ashrrev_i32_e32 v2, 31, v2
; GFX10-NEXT: v_add_nc_u32_e32 v3, -1, v3
; GFX10-NEXT: v_add_nc_u32_e32 v2, 32, v2
@@ -22775,6 +32107,11 @@ define bfloat @v_sitofp_i64_to_bf16(i64 %x) {
; GFX10-NEXT: v_sub_nc_u32_e32 v1, 32, v2
; GFX10-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX10-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -22783,6 +32120,7 @@ define bfloat @v_sitofp_i64_to_bf16(i64 %x) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_xor_b32_e32 v2, v0, v1
; GFX11-NEXT: v_cls_i32_e32 v3, v1
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_ashrrev_i32_e32 v2, 31, v2
; GFX11-NEXT: v_add_nc_u32_e32 v3, -1, v3
@@ -22798,6 +32136,13 @@ define bfloat @v_sitofp_i64_to_bf16(i64 %x) {
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = sitofp i64 %x to bfloat
@@ -22877,23 +32222,38 @@ define <2 x bfloat> @v_sitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX8-NEXT: v_add_u32_e32 v5, vcc, 32, v5
; GFX8-NEXT: v_min_u32_e32 v4, v4, v5
; GFX8-NEXT: v_lshlrev_b64 v[0:1], v4, v[0:1]
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
; GFX8-NEXT: v_min_u32_e32 v0, 1, v0
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX8-NEXT: v_sub_u32_e32 v1, vcc, 32, v4
+; GFX8-NEXT: v_ldexp_f32 v4, v0, v1
+; GFX8-NEXT: v_bfe_u32 v0, v4, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v4
; GFX8-NEXT: v_xor_b32_e32 v1, v2, v3
-; GFX8-NEXT: v_cvt_f32_i32_e32 v5, v0
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v0
; GFX8-NEXT: v_ffbh_i32_e32 v0, v3
; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v1
; GFX8-NEXT: v_add_u32_e32 v0, vcc, -1, v0
; GFX8-NEXT: v_add_u32_e32 v1, vcc, 32, v1
-; GFX8-NEXT: v_min_u32_e32 v6, v0, v1
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v6, v[2:3]
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v4
+; GFX8-NEXT: v_min_u32_e32 v7, v0, v1
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v7, v[2:3]
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX8-NEXT: v_min_u32_e32 v0, 1, v0
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX8-NEXT: v_ldexp_f32 v1, v5, v2
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v6
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc
+; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v7
; GFX8-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -22908,24 +32268,37 @@ define <2 x bfloat> @v_sitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX9-NEXT: v_add_u32_e32 v5, 32, v5
; GFX9-NEXT: v_min_u32_e32 v4, v4, v5
; GFX9-NEXT: v_lshlrev_b64 v[0:1], v4, v[0:1]
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX9-NEXT: v_or_b32_e32 v5, v1, v0
+; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX9-NEXT: v_sub_u32_e32 v1, 32, v4
+; GFX9-NEXT: v_ldexp_f32 v4, v0, v1
+; GFX9-NEXT: v_bfe_u32 v0, v4, 16, 1
; GFX9-NEXT: v_xor_b32_e32 v1, v2, v3
+; GFX9-NEXT: v_add3_u32 v5, v0, v4, s4
; GFX9-NEXT: v_ffbh_i32_e32 v0, v3
; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v1
; GFX9-NEXT: v_add_u32_e32 v0, -1, v0
; GFX9-NEXT: v_add_u32_e32 v1, 32, v1
-; GFX9-NEXT: v_min_u32_e32 v6, v0, v1
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v6, v[2:3]
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v5
+; GFX9-NEXT: v_min_u32_e32 v7, v0, v1
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[2:3]
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX9-NEXT: v_sub_u32_e32 v1, 32, v4
-; GFX9-NEXT: v_ldexp_f32 v1, v2, v1
-; GFX9-NEXT: v_sub_u32_e32 v2, 32, v6
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc
+; GFX9-NEXT: v_sub_u32_e32 v2, 32, v7
; GFX9-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -22936,6 +32309,7 @@ define <2 x bfloat> @v_sitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX10-NEXT: v_xor_b32_e32 v5, v2, v3
; GFX10-NEXT: v_ffbh_i32_e32 v6, v1
; GFX10-NEXT: v_ffbh_i32_e32 v7, v3
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v4
; GFX10-NEXT: v_ashrrev_i32_e32 v5, 31, v5
; GFX10-NEXT: v_add_nc_u32_e32 v6, -1, v6
@@ -22956,6 +32330,16 @@ define <2 x bfloat> @v_sitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX10-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX10-NEXT: v_ldexp_f32 v0, v0, v2
; GFX10-NEXT: v_ldexp_f32 v1, v1, v3
+; GFX10-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_and_or_b32 v5, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -22966,10 +32350,9 @@ define <2 x bfloat> @v_sitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX11-NEXT: v_xor_b32_e32 v5, v2, v3
; GFX11-NEXT: v_cls_i32_e32 v6, v1
; GFX11-NEXT: v_cls_i32_e32 v7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: v_ashrrev_i32_e32 v4, 31, v4
; GFX11-NEXT: v_ashrrev_i32_e32 v5, 31, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_add_nc_u32_e32 v6, -1, v6
; GFX11-NEXT: v_add_nc_u32_e32 v7, -1, v7
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -22995,6 +32378,18 @@ define <2 x bfloat> @v_sitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_ldexp_f32 v0, v0, v2
; GFX11-NEXT: v_ldexp_f32 v1, v1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_and_or_b32 v5, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -23101,36 +32496,58 @@ define <3 x bfloat> @v_sitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 32, v7
; GFX8-NEXT: v_min_u32_e32 v6, v6, v7
; GFX8-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
+; GFX8-NEXT: v_xor_b32_e32 v8, v0, v1
; GFX8-NEXT: v_min_u32_e32 v4, 1, v4
; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
; GFX8-NEXT: v_cvt_f32_i32_e32 v4, v4
; GFX8-NEXT: v_sub_u32_e32 v5, vcc, 32, v6
-; GFX8-NEXT: v_ldexp_f32 v6, v4, v5
-; GFX8-NEXT: v_xor_b32_e32 v5, v0, v1
-; GFX8-NEXT: v_ffbh_i32_e32 v4, v1
-; GFX8-NEXT: v_ashrrev_i32_e32 v5, 31, v5
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, -1, v4
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 32, v5
-; GFX8-NEXT: v_min_u32_e32 v7, v4, v5
-; GFX8-NEXT: v_lshlrev_b64 v[4:5], v7, v[0:1]
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX8-NEXT: v_min_u32_e32 v0, 1, v4
-; GFX8-NEXT: v_or_b32_e32 v0, v5, v0
-; GFX8-NEXT: v_xor_b32_e32 v5, v2, v3
-; GFX8-NEXT: v_ffbh_i32_e32 v4, v3
-; GFX8-NEXT: v_ashrrev_i32_e32 v5, 31, v5
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, -1, v4
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 32, v5
-; GFX8-NEXT: v_min_u32_e32 v4, v4, v5
-; GFX8-NEXT: v_lshlrev_b64 v[2:3], v4, v[2:3]
+; GFX8-NEXT: v_ffbh_i32_e32 v7, v1
+; GFX8-NEXT: v_ldexp_f32 v4, v4, v5
+; GFX8-NEXT: v_ashrrev_i32_e32 v8, 31, v8
+; GFX8-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, -1, v7
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v8
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v4
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_min_u32_e32 v7, v7, v8
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v4
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX8-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX8-NEXT: v_sub_u32_e32 v4, vcc, 32, v7
+; GFX8-NEXT: v_xor_b32_e32 v7, v2, v3
+; GFX8-NEXT: v_ffbh_i32_e32 v6, v3
+; GFX8-NEXT: v_ashrrev_i32_e32 v7, 31, v7
; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, -1, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 32, v7
+; GFX8-NEXT: v_min_u32_e32 v6, v6, v7
+; GFX8-NEXT: v_lshlrev_b64 v[2:3], v6, v[2:3]
+; GFX8-NEXT: v_ldexp_f32 v0, v0, v4
; GFX8-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX8-NEXT: v_bfe_u32 v4, v0, 16, 1
; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v0
; GFX8-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX8-NEXT: v_sub_u32_e32 v3, vcc, 32, v4
-; GFX8-NEXT: v_sub_u32_e32 v5, vcc, 32, v7
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX8-NEXT: v_sub_u32_e32 v3, vcc, 32, v6
; GFX8-NEXT: v_ldexp_f32 v2, v2, v3
-; GFX8-NEXT: v_ldexp_f32 v0, v0, v5
+; GFX8-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v2
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -23144,37 +32561,56 @@ define <3 x bfloat> @v_sitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX9-NEXT: v_add_u32_e32 v6, -1, v6
; GFX9-NEXT: v_add_u32_e32 v7, 32, v7
; GFX9-NEXT: v_min_u32_e32 v6, v6, v7
+; GFX9-NEXT: v_xor_b32_e32 v8, v0, v1
; GFX9-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
-; GFX9-NEXT: v_xor_b32_e32 v7, v0, v1
+; GFX9-NEXT: v_ffbh_i32_e32 v7, v1
+; GFX9-NEXT: v_ashrrev_i32_e32 v8, 31, v8
+; GFX9-NEXT: v_add_u32_e32 v7, -1, v7
+; GFX9-NEXT: v_add_u32_e32 v8, 32, v8
; GFX9-NEXT: v_min_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_min_u32_e32 v7, v7, v8
; GFX9-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX9-NEXT: v_sub_u32_e32 v5, 32, v6
-; GFX9-NEXT: v_ffbh_i32_e32 v6, v1
-; GFX9-NEXT: v_ashrrev_i32_e32 v7, 31, v7
-; GFX9-NEXT: v_add_u32_e32 v6, -1, v6
-; GFX9-NEXT: v_add_u32_e32 v7, 32, v7
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
; GFX9-NEXT: v_cvt_f32_i32_e32 v4, v4
-; GFX9-NEXT: v_min_u32_e32 v6, v6, v7
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1]
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT: v_sub_u32_e32 v5, 32, v6
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX9-NEXT: v_ldexp_f32 v4, v4, v5
-; GFX9-NEXT: v_or_b32_e32 v5, v1, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v5, v5, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_sub_u32_e32 v1, 32, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_ldexp_f32 v5, v0, v1
+; GFX9-NEXT: v_bfe_u32 v0, v5, 16, 1
; GFX9-NEXT: v_xor_b32_e32 v1, v2, v3
+; GFX9-NEXT: v_add3_u32 v6, v0, v5, s4
; GFX9-NEXT: v_ffbh_i32_e32 v0, v3
; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v1
; GFX9-NEXT: v_add_u32_e32 v0, -1, v0
; GFX9-NEXT: v_add_u32_e32 v1, 32, v1
-; GFX9-NEXT: v_min_u32_e32 v7, v0, v1
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[2:3]
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v5
+; GFX9-NEXT: v_min_u32_e32 v8, v0, v1
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3]
+; GFX9-NEXT: v_and_b32_e32 v7, 0x80000000, v5
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX9-NEXT: v_sub_u32_e32 v1, 32, v6
-; GFX9-NEXT: v_ldexp_f32 v1, v2, v1
-; GFX9-NEXT: v_sub_u32_e32 v2, 32, v7
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
+; GFX9-NEXT: v_sub_u32_e32 v2, 32, v8
; GFX9-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: v_alignbit_b32 v1, s4, v4, 16
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -23182,44 +32618,60 @@ define <3 x bfloat> @v_sitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX10-LABEL: v_sitofp_v3i64_to_v3bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_xor_b32_e32 v7, v0, v1
-; GFX10-NEXT: v_xor_b32_e32 v8, v4, v5
+; GFX10-NEXT: v_xor_b32_e32 v8, v0, v1
+; GFX10-NEXT: v_xor_b32_e32 v7, v4, v5
; GFX10-NEXT: v_xor_b32_e32 v9, v2, v3
-; GFX10-NEXT: v_ffbh_i32_e32 v6, v5
; GFX10-NEXT: v_ffbh_i32_e32 v10, v1
+; GFX10-NEXT: v_ffbh_i32_e32 v6, v5
+; GFX10-NEXT: v_ashrrev_i32_e32 v8, 31, v8
; GFX10-NEXT: v_ashrrev_i32_e32 v7, 31, v7
; GFX10-NEXT: v_ffbh_i32_e32 v11, v3
; GFX10-NEXT: v_ashrrev_i32_e32 v9, 31, v9
-; GFX10-NEXT: v_ashrrev_i32_e32 v8, 31, v8
-; GFX10-NEXT: v_add_nc_u32_e32 v6, -1, v6
; GFX10-NEXT: v_add_nc_u32_e32 v10, -1, v10
+; GFX10-NEXT: v_add_nc_u32_e32 v8, 32, v8
+; GFX10-NEXT: v_add_nc_u32_e32 v6, -1, v6
; GFX10-NEXT: v_add_nc_u32_e32 v7, 32, v7
; GFX10-NEXT: v_add_nc_u32_e32 v11, -1, v11
; GFX10-NEXT: v_add_nc_u32_e32 v9, 32, v9
-; GFX10-NEXT: v_add_nc_u32_e32 v8, 32, v8
-; GFX10-NEXT: v_min_u32_e32 v7, v10, v7
-; GFX10-NEXT: v_min_u32_e32 v9, v11, v9
-; GFX10-NEXT: v_min_u32_e32 v6, v6, v8
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], v9, v[2:3]
+; GFX10-NEXT: v_min_u32_e32 v8, v10, v8
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_min_u32_e32 v6, v6, v7
+; GFX10-NEXT: v_min_u32_e32 v7, v11, v9
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1]
; GFX10-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
; GFX10-NEXT: v_sub_nc_u32_e32 v6, 32, v6
+; GFX10-NEXT: v_lshlrev_b64 v[2:3], v7, v[2:3]
; GFX10-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX10-NEXT: v_min_u32_e32 v2, 1, v2
; GFX10-NEXT: v_min_u32_e32 v4, 1, v4
+; GFX10-NEXT: v_min_u32_e32 v2, 1, v2
; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX10-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX10-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX10-NEXT: v_sub_nc_u32_e32 v3, 32, v7
-; GFX10-NEXT: v_sub_nc_u32_e32 v4, 32, v9
+; GFX10-NEXT: v_or_b32_e32 v1, v5, v4
+; GFX10-NEXT: v_sub_nc_u32_e32 v4, 32, v7
+; GFX10-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX10-NEXT: v_sub_nc_u32_e32 v3, 32, v8
; GFX10-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX10-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX10-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX10-NEXT: v_ldexp_f32 v0, v0, v3
-; GFX10-NEXT: v_ldexp_f32 v1, v1, v4
-; GFX10-NEXT: v_ldexp_f32 v2, v2, v6
-; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
-; GFX10-NEXT: v_alignbit_b32 v1, s4, v2, 16
+; GFX10-NEXT: v_ldexp_f32 v1, v1, v6
+; GFX10-NEXT: v_ldexp_f32 v2, v2, v4
+; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v2, s4, 0x400000
+; GFX10-NEXT: v_and_or_b32 v6, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = sitofp <3 x i64> %x to <3 x bfloat>
ret <3 x bfloat> %op
@@ -23350,48 +32802,77 @@ define <4 x bfloat> @v_sitofp_v4i64_to_v4bf16(<4 x i64> %x) {
; GFX8-NEXT: v_add_u32_e32 v9, vcc, 32, v9
; GFX8-NEXT: v_min_u32_e32 v8, v8, v9
; GFX8-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5]
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
; GFX8-NEXT: v_min_u32_e32 v4, 1, v4
; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX8-NEXT: v_cvt_f32_i32_e32 v4, v4
+; GFX8-NEXT: v_sub_u32_e32 v5, vcc, 32, v8
+; GFX8-NEXT: v_ldexp_f32 v8, v4, v5
+; GFX8-NEXT: v_bfe_u32 v4, v8, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v8
; GFX8-NEXT: v_xor_b32_e32 v5, v6, v7
-; GFX8-NEXT: v_cvt_f32_i32_e32 v9, v4
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v4
; GFX8-NEXT: v_ffbh_i32_e32 v4, v7
; GFX8-NEXT: v_ashrrev_i32_e32 v5, 31, v5
; GFX8-NEXT: v_add_u32_e32 v4, vcc, -1, v4
; GFX8-NEXT: v_add_u32_e32 v5, vcc, 32, v5
-; GFX8-NEXT: v_min_u32_e32 v10, v4, v5
-; GFX8-NEXT: v_lshlrev_b64 v[4:5], v10, v[6:7]
-; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 32, v8
+; GFX8-NEXT: v_min_u32_e32 v11, v4, v5
+; GFX8-NEXT: v_lshlrev_b64 v[4:5], v11, v[6:7]
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v8
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v10
; GFX8-NEXT: v_min_u32_e32 v4, 1, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v6, vcc
+; GFX8-NEXT: v_xor_b32_e32 v9, v0, v1
+; GFX8-NEXT: v_ffbh_i32_e32 v8, v1
+; GFX8-NEXT: v_ashrrev_i32_e32 v9, 31, v9
; GFX8-NEXT: v_cvt_f32_i32_e32 v4, v4
-; GFX8-NEXT: v_ldexp_f32 v5, v9, v6
-; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 32, v10
-; GFX8-NEXT: v_xor_b32_e32 v7, v0, v1
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, -1, v8
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 32, v9
+; GFX8-NEXT: v_min_u32_e32 v8, v8, v9
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1]
+; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 32, v11
; GFX8-NEXT: v_ldexp_f32 v4, v4, v6
-; GFX8-NEXT: v_ffbh_i32_e32 v6, v1
-; GFX8-NEXT: v_ashrrev_i32_e32 v7, 31, v7
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, -1, v6
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 32, v7
-; GFX8-NEXT: v_min_u32_e32 v6, v6, v7
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1]
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX8-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, s4, v6
+; GFX8-NEXT: v_and_b32_e32 v7, 0x80000000, v4
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc
+; GFX8-NEXT: v_sub_u32_e32 v1, vcc, 32, v8
+; GFX8-NEXT: v_ldexp_f32 v6, v0, v1
+; GFX8-NEXT: v_bfe_u32 v0, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v6
; GFX8-NEXT: v_xor_b32_e32 v1, v2, v3
-; GFX8-NEXT: v_cvt_f32_i32_e32 v7, v0
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, s4, v0
; GFX8-NEXT: v_ffbh_i32_e32 v0, v3
; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v1
; GFX8-NEXT: v_add_u32_e32 v0, vcc, -1, v0
; GFX8-NEXT: v_add_u32_e32 v1, vcc, 32, v1
-; GFX8-NEXT: v_min_u32_e32 v8, v0, v1
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3]
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v6
+; GFX8-NEXT: v_min_u32_e32 v9, v0, v1
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v9, v[2:3]
+; GFX8-NEXT: v_and_b32_e32 v8, 0x80000000, v6
; GFX8-NEXT: v_min_u32_e32 v0, 1, v0
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX8-NEXT: v_ldexp_f32 v1, v7, v2
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v8
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v8
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
+; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v9
; GFX8-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX8-NEXT: v_alignbit_b32 v1, v4, v5, 16
@@ -23407,50 +32888,75 @@ define <4 x bfloat> @v_sitofp_v4i64_to_v4bf16(<4 x i64> %x) {
; GFX9-NEXT: v_add_u32_e32 v9, 32, v9
; GFX9-NEXT: v_min_u32_e32 v8, v8, v9
; GFX9-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5]
-; GFX9-NEXT: v_sub_u32_e32 v8, 32, v8
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_min_u32_e32 v4, 1, v4
; GFX9-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, v4
+; GFX9-NEXT: v_sub_u32_e32 v5, 32, v8
+; GFX9-NEXT: v_ldexp_f32 v8, v4, v5
+; GFX9-NEXT: v_bfe_u32 v4, v8, 16, 1
; GFX9-NEXT: v_xor_b32_e32 v5, v6, v7
-; GFX9-NEXT: v_cvt_f32_i32_e32 v9, v4
+; GFX9-NEXT: v_add3_u32 v9, v4, v8, s4
; GFX9-NEXT: v_ffbh_i32_e32 v4, v7
; GFX9-NEXT: v_ashrrev_i32_e32 v5, 31, v5
; GFX9-NEXT: v_add_u32_e32 v4, -1, v4
; GFX9-NEXT: v_add_u32_e32 v5, 32, v5
-; GFX9-NEXT: v_min_u32_e32 v10, v4, v5
-; GFX9-NEXT: v_lshlrev_b64 v[4:5], v10, v[6:7]
-; GFX9-NEXT: v_ldexp_f32 v6, v9, v8
-; GFX9-NEXT: v_xor_b32_e32 v8, v0, v1
+; GFX9-NEXT: v_min_u32_e32 v11, v4, v5
+; GFX9-NEXT: v_lshlrev_b64 v[4:5], v11, v[6:7]
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v8
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v10
; GFX9-NEXT: v_min_u32_e32 v4, 1, v4
-; GFX9-NEXT: v_ffbh_i32_e32 v7, v1
-; GFX9-NEXT: v_ashrrev_i32_e32 v8, 31, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GFX9-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX9-NEXT: v_add_u32_e32 v7, -1, v7
-; GFX9-NEXT: v_add_u32_e32 v8, 32, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v9, v6, vcc
+; GFX9-NEXT: v_xor_b32_e32 v9, v0, v1
+; GFX9-NEXT: v_ffbh_i32_e32 v8, v1
+; GFX9-NEXT: v_ashrrev_i32_e32 v9, 31, v9
+; GFX9-NEXT: v_add_u32_e32 v8, -1, v8
+; GFX9-NEXT: v_add_u32_e32 v9, 32, v9
+; GFX9-NEXT: v_min_u32_e32 v8, v8, v9
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1]
; GFX9-NEXT: v_cvt_f32_i32_e32 v4, v4
-; GFX9-NEXT: v_min_u32_e32 v7, v7, v8
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
-; GFX9-NEXT: v_sub_u32_e32 v5, 32, v10
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX9-NEXT: v_ldexp_f32 v4, v4, v5
-; GFX9-NEXT: v_or_b32_e32 v5, v1, v0
+; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT: v_sub_u32_e32 v6, 32, v11
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX9-NEXT: v_ldexp_f32 v4, v4, v6
+; GFX9-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v7, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v6, v6, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_sub_u32_e32 v1, 32, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc
+; GFX9-NEXT: v_ldexp_f32 v6, v0, v1
+; GFX9-NEXT: v_bfe_u32 v0, v6, 16, 1
; GFX9-NEXT: v_xor_b32_e32 v1, v2, v3
+; GFX9-NEXT: v_add3_u32 v7, v0, v6, s4
; GFX9-NEXT: v_ffbh_i32_e32 v0, v3
; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v1
; GFX9-NEXT: v_add_u32_e32 v0, -1, v0
; GFX9-NEXT: v_add_u32_e32 v1, 32, v1
-; GFX9-NEXT: v_min_u32_e32 v8, v0, v1
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3]
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v5
+; GFX9-NEXT: v_min_u32_e32 v9, v0, v1
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v9, v[2:3]
+; GFX9-NEXT: v_and_b32_e32 v8, 0x80000000, v6
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX9-NEXT: v_sub_u32_e32 v1, 32, v7
-; GFX9-NEXT: v_ldexp_f32 v1, v2, v1
-; GFX9-NEXT: v_sub_u32_e32 v2, 32, v8
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
+; GFX9-NEXT: v_sub_u32_e32 v2, 32, v9
; GFX9-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_perm_b32 v1, v4, v6, s4
+; GFX9-NEXT: v_perm_b32 v1, v4, v5, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_sitofp_v4i64_to_v4bf16:
@@ -23460,16 +32966,16 @@ define <4 x bfloat> @v_sitofp_v4i64_to_v4bf16(<4 x i64> %x) {
; GFX10-NEXT: v_ffbh_i32_e32 v9, v5
; GFX10-NEXT: v_xor_b32_e32 v11, v6, v7
; GFX10-NEXT: v_xor_b32_e32 v13, v0, v1
-; GFX10-NEXT: v_xor_b32_e32 v14, v2, v3
+; GFX10-NEXT: v_ffbh_i32_e32 v10, v7
; GFX10-NEXT: v_ashrrev_i32_e32 v8, 31, v8
; GFX10-NEXT: v_add_nc_u32_e32 v9, -1, v9
-; GFX10-NEXT: v_ffbh_i32_e32 v10, v7
; GFX10-NEXT: v_ffbh_i32_e32 v12, v1
+; GFX10-NEXT: v_xor_b32_e32 v14, v2, v3
; GFX10-NEXT: v_ashrrev_i32_e32 v11, 31, v11
; GFX10-NEXT: v_add_nc_u32_e32 v8, 32, v8
-; GFX10-NEXT: v_ashrrev_i32_e32 v14, 31, v14
; GFX10-NEXT: v_add_nc_u32_e32 v10, -1, v10
; GFX10-NEXT: v_add_nc_u32_e32 v12, -1, v12
+; GFX10-NEXT: v_ashrrev_i32_e32 v14, 31, v14
; GFX10-NEXT: v_add_nc_u32_e32 v11, 32, v11
; GFX10-NEXT: v_min_u32_e32 v8, v9, v8
; GFX10-NEXT: v_ashrrev_i32_e32 v9, 31, v13
@@ -23479,33 +32985,54 @@ define <4 x bfloat> @v_sitofp_v4i64_to_v4bf16(<4 x i64> %x) {
; GFX10-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5]
; GFX10-NEXT: v_add_nc_u32_e32 v9, 32, v9
; GFX10-NEXT: v_add_nc_u32_e32 v13, -1, v13
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_lshlrev_b64 v[6:7], v10, v[6:7]
; GFX10-NEXT: v_min_u32_e32 v9, v12, v9
; GFX10-NEXT: v_min_u32_e32 v11, v13, v14
; GFX10-NEXT: v_min_u32_e32 v4, 1, v4
-; GFX10-NEXT: v_min_u32_e32 v6, 1, v6
; GFX10-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1]
; GFX10-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
; GFX10-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX10-NEXT: v_or_b32_e32 v5, v7, v6
-; GFX10-NEXT: v_sub_nc_u32_e32 v6, 32, v11
+; GFX10-NEXT: v_min_u32_e32 v5, 1, v6
+; GFX10-NEXT: v_sub_nc_u32_e32 v6, 32, v8
; GFX10-NEXT: v_min_u32_e32 v0, 1, v0
; GFX10-NEXT: v_min_u32_e32 v2, 1, v2
-; GFX10-NEXT: v_sub_nc_u32_e32 v7, 32, v10
+; GFX10-NEXT: v_cvt_f32_i32_e32 v4, v4
+; GFX10-NEXT: v_or_b32_e32 v5, v7, v5
; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX10-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX10-NEXT: v_cvt_f32_i32_e32 v1, v4
-; GFX10-NEXT: v_sub_nc_u32_e32 v3, 32, v8
-; GFX10-NEXT: v_cvt_f32_i32_e32 v4, v5
+; GFX10-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX10-NEXT: v_ldexp_f32 v2, v4, v6
+; GFX10-NEXT: v_cvt_f32_i32_e32 v3, v5
+; GFX10-NEXT: v_sub_nc_u32_e32 v4, 32, v10
; GFX10-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX10-NEXT: v_sub_nc_u32_e32 v5, 32, v9
-; GFX10-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX10-NEXT: v_ldexp_f32 v1, v1, v3
-; GFX10-NEXT: v_ldexp_f32 v3, v4, v7
+; GFX10-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX10-NEXT: v_sub_nc_u32_e32 v6, 32, v11
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_ldexp_f32 v3, v3, v4
; GFX10-NEXT: v_ldexp_f32 v0, v0, v5
-; GFX10-NEXT: v_ldexp_f32 v2, v2, v6
-; GFX10-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX10-NEXT: v_and_or_b32 v5, v2, s4, 0x400000
+; GFX10-NEXT: v_ldexp_f32 v1, v1, v6
+; GFX10-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX10-NEXT: v_and_or_b32 v9, v3, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX10-NEXT: v_add3_u32 v4, v6, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v5, v7, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v6, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v7, v8, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v1, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v4, v9, vcc_lo
+; GFX10-NEXT: v_perm_b32 v1, v3, v2, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_sitofp_v4i64_to_v4bf16:
@@ -23515,16 +33042,16 @@ define <4 x bfloat> @v_sitofp_v4i64_to_v4bf16(<4 x i64> %x) {
; GFX11-NEXT: v_cls_i32_e32 v9, v5
; GFX11-NEXT: v_xor_b32_e32 v11, v6, v7
; GFX11-NEXT: v_xor_b32_e32 v13, v0, v1
-; GFX11-NEXT: v_xor_b32_e32 v14, v2, v3
+; GFX11-NEXT: v_cls_i32_e32 v10, v7
; GFX11-NEXT: v_ashrrev_i32_e32 v8, 31, v8
; GFX11-NEXT: v_add_nc_u32_e32 v9, -1, v9
-; GFX11-NEXT: v_cls_i32_e32 v10, v7
; GFX11-NEXT: v_cls_i32_e32 v12, v1
+; GFX11-NEXT: v_xor_b32_e32 v14, v2, v3
; GFX11-NEXT: v_ashrrev_i32_e32 v11, 31, v11
; GFX11-NEXT: v_add_nc_u32_e32 v8, 32, v8
-; GFX11-NEXT: v_ashrrev_i32_e32 v14, 31, v14
; GFX11-NEXT: v_add_nc_u32_e32 v10, -1, v10
; GFX11-NEXT: v_add_nc_u32_e32 v12, -1, v12
+; GFX11-NEXT: v_ashrrev_i32_e32 v14, 31, v14
; GFX11-NEXT: v_add_nc_u32_e32 v11, 32, v11
; GFX11-NEXT: v_min_u32_e32 v8, v9, v8
; GFX11-NEXT: v_ashrrev_i32_e32 v9, 31, v13
@@ -23534,40 +33061,61 @@ define <4 x bfloat> @v_sitofp_v4i64_to_v4bf16(<4 x i64> %x) {
; GFX11-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5]
; GFX11-NEXT: v_add_nc_u32_e32 v9, 32, v9
; GFX11-NEXT: v_add_nc_u32_e32 v13, -1, v13
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: v_lshlrev_b64 v[6:7], v10, v[6:7]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_min_u32_e32 v9, v12, v9
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_min_u32_e32 v11, v13, v14
; GFX11-NEXT: v_min_u32_e32 v4, 1, v4
-; GFX11-NEXT: v_min_u32_e32 v6, 1, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1]
; GFX11-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX11-NEXT: v_or_b32_e32 v5, v7, v6
-; GFX11-NEXT: v_sub_nc_u32_e32 v6, 32, v11
+; GFX11-NEXT: v_min_u32_e32 v5, 1, v6
+; GFX11-NEXT: v_sub_nc_u32_e32 v6, 32, v8
; GFX11-NEXT: v_min_u32_e32 v0, 1, v0
; GFX11-NEXT: v_min_u32_e32 v2, 1, v2
-; GFX11-NEXT: v_sub_nc_u32_e32 v7, 32, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cvt_f32_i32_e32 v4, v4
+; GFX11-NEXT: v_or_b32_e32 v5, v7, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX11-NEXT: v_cvt_f32_i32_e32 v1, v4
-; GFX11-NEXT: v_sub_nc_u32_e32 v3, 32, v8
-; GFX11-NEXT: v_cvt_f32_i32_e32 v4, v5
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_ldexp_f32 v2, v4, v6
+; GFX11-NEXT: v_cvt_f32_i32_e32 v3, v5
+; GFX11-NEXT: v_sub_nc_u32_e32 v4, 32, v10
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX11-NEXT: v_sub_nc_u32_e32 v5, 32, v9
-; GFX11-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX11-NEXT: v_ldexp_f32 v1, v1, v3
-; GFX11-NEXT: v_ldexp_f32 v3, v4, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX11-NEXT: v_sub_nc_u32_e32 v6, 32, v11
+; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-NEXT: v_ldexp_f32 v3, v3, v4
; GFX11-NEXT: v_ldexp_f32 v0, v0, v5
-; GFX11-NEXT: v_ldexp_f32 v2, v2, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX11-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX11-NEXT: v_and_or_b32 v5, v2, s0, 0x400000
+; GFX11-NEXT: v_ldexp_f32 v1, v1, v6
+; GFX11-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-NEXT: v_and_or_b32 v9, v3, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-NEXT: v_add3_u32 v4, v6, v3, 0x7fff
+; GFX11-NEXT: v_add3_u32 v5, v7, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v6, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v7, v8, v1, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v8, v1, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v4, v9, vcc_lo
+; GFX11-NEXT: v_perm_b32 v1, v3, v2, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = sitofp <4 x i64> %x to <4 x bfloat>
ret <4 x bfloat> %op
@@ -23594,6 +33142,13 @@ define bfloat @v_uitofp_i16_to_bf16(i16 %x) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -23601,6 +33156,13 @@ define bfloat @v_uitofp_i16_to_bf16(i16 %x) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -23608,6 +33170,12 @@ define bfloat @v_uitofp_i16_to_bf16(i16 %x) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -23615,8 +33183,16 @@ define bfloat @v_uitofp_i16_to_bf16(i16 %x) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = uitofp i16 %x to bfloat
@@ -23649,10 +33225,24 @@ define <2 x bfloat> @v_uitofp_v2i16_to_v2bf16(<2 x i16> %x) {
; GFX8-LABEL: v_uitofp_v2i16_to_v2bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_u32_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX8-NEXT: v_cvt_f32_u32_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX8-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v1
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_uitofp_v2i16_to_v2bf16:
@@ -23660,6 +33250,19 @@ define <2 x bfloat> @v_uitofp_v2i16_to_v2bf16(<2 x i16> %x) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_u32_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
; GFX9-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v2, v2, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -23669,6 +33272,17 @@ define <2 x bfloat> @v_uitofp_v2i16_to_v2bf16(<2 x i16> %x) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_u32_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
; GFX10-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v1, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_and_or_b32 v5, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -23677,9 +33291,22 @@ define <2 x bfloat> @v_uitofp_v2i16_to_v2bf16(<2 x i16> %x) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v1, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_and_or_b32 v5, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -23719,20 +33346,61 @@ define <3 x bfloat> @v_uitofp_v3i16_to_v3bf16(<3 x i16> %x) {
; GFX8-LABEL: v_uitofp_v3i16_to_v3bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX8-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX8-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_cvt_f32_u32_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX8-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX8-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s4, v2
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v4
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX8-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v0
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16
+; GFX8-NEXT: v_alignbit_b32 v0, v0, v2, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_uitofp_v3i16_to_v3bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX9-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX9-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: v_cvt_f32_u32_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v2, v2, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v2, v2, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v2, s4
; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16
@@ -23743,8 +33411,24 @@ define <3 x bfloat> @v_uitofp_v3i16_to_v3bf16(<3 x i16> %x) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
; GFX10-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v2, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v6, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = uitofp <3 x i16> %x to <3 x bfloat>
@@ -23789,23 +33473,77 @@ define <4 x bfloat> @v_uitofp_v4i16_to_v4bf16(<4 x i16> %x) {
; GFX8-LABEL: v_uitofp_v4i16_to_v4bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_u32_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_cvt_f32_u32_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX8-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v0, v3, v0, 16
-; GFX8-NEXT: v_alignbit_b32 v1, v2, v1, 16
+; GFX8-NEXT: v_cvt_f32_u32_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX8-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_cvt_f32_u32_sdwa v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v2
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX8-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX8-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v5
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v5
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX8-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
+; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_uitofp_v4i16_to_v4bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_u32_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX9-NEXT: v_cvt_f32_u32_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
-; GFX9-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX9-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_cvt_f32_u32_sdwa v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v3, v3, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX9-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v5
+; GFX9-NEXT: v_add3_u32 v3, v3, v5, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
@@ -23816,9 +33554,30 @@ define <4 x bfloat> @v_uitofp_v4i16_to_v4bf16(<4 x i16> %x) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_u32_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
; GFX10-NEXT: v_cvt_f32_u32_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX10-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX10-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX10-NEXT: v_and_or_b32 v5, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_and_or_b32 v9, v3, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX10-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX10-NEXT: v_and_or_b32 v11, v0, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_add3_u32 v10, v10, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v7, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX10-NEXT: v_perm_b32 v0, v0, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -23826,17 +33585,42 @@ define <4 x bfloat> @v_uitofp_v4i16_to_v4bf16(<4 x i16> %x) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v1
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cvt_f32_u32_e32 v2, v2
+; GFX11-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v5, v2, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-NEXT: v_and_or_b32 v7, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v0
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_cvt_f32_u32_e32 v2, v2
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
; GFX11-NEXT: v_cvt_f32_u32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11-NEXT: v_cvt_f32_u32_e32 v1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-NEXT: v_and_or_b32 v9, v3, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_and_or_b32 v11, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-NEXT: v_add3_u32 v10, v10, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_perm_b32 v0, v0, v3, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = uitofp <4 x i16> %x to <4 x bfloat>
@@ -23862,6 +33646,13 @@ define bfloat @v_uitofp_i32_to_bf16(i32 %x) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_and_b32_e32 v1, 0x80000000, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_or_b32_e32 v1, 0x400000, v1
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -23869,6 +33660,13 @@ define bfloat @v_uitofp_i32_to_bf16(i32 %x) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -23876,6 +33674,12 @@ define bfloat @v_uitofp_i32_to_bf16(i32 %x) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -23883,7 +33687,14 @@ define bfloat @v_uitofp_i32_to_bf16(i32 %x) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = uitofp i32 %x to bfloat
@@ -23912,8 +33723,22 @@ define <2 x bfloat> @v_uitofp_v2i32_to_v2bf16(<2 x i32> %x) {
; GFX8-LABEL: v_uitofp_v2i32_to_v2bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX8-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX8-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v0
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -23923,6 +33748,19 @@ define <2 x bfloat> @v_uitofp_v2i32_to_v2bf16(<2 x i32> %x) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v2, v2, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -23932,6 +33770,17 @@ define <2 x bfloat> @v_uitofp_v2i32_to_v2bf16(<2 x i32> %x) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX10-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_and_or_b32 v5, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -23940,6 +33789,19 @@ define <2 x bfloat> @v_uitofp_v2i32_to_v2bf16(<2 x i32> %x) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX11-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_and_or_b32 v5, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -23974,19 +33836,60 @@ define <3 x bfloat> @v_uitofp_v3i32_to_v3bf16(<3 x i32> %x) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cvt_f32_u32_e32 v2, v2
-; GFX8-NEXT: v_cvt_f32_u32_e32 v3, v1
; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16
+; GFX8-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX8-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v2
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v2
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v3, vcc
+; GFX8-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
+; GFX8-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v1
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_uitofp_v3i32_to_v3bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v1
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v2
+; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v3, v3, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4
; GFX9-NEXT: v_alignbit_b32 v1, s4, v2, 16
@@ -23997,8 +33900,24 @@ define <3 x bfloat> @v_uitofp_v3i32_to_v3bf16(<3 x i32> %x) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX10-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_cvt_f32_u32_e32 v2, v2
+; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v4, v2, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v6, v4, vcc_lo
; GFX10-NEXT: v_alignbit_b32 v1, s4, v2, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = uitofp <3 x i32> %x to <3 x bfloat>
@@ -24035,10 +33954,39 @@ define <4 x bfloat> @v_uitofp_v4i32_to_v4bf16(<4 x i32> %x) {
; GFX8-LABEL: v_uitofp_v4i32_to_v4bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX8-NEXT: v_cvt_f32_u32_e32 v3, v3
-; GFX8-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX8-NEXT: v_cvt_f32_u32_e32 v2, v2
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v2
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v3
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v4, vcc
+; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX8-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v0
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc
+; GFX8-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v1
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16
@@ -24049,9 +33997,34 @@ define <4 x bfloat> @v_uitofp_v4i32_to_v4bf16(<4 x i32> %x) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v3
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX9-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v4, v4, v2, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v1
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v3
+; GFX9-NEXT: v_add3_u32 v4, v4, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v4, v4, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4
; GFX9-NEXT: v_perm_b32 v1, v3, v2, s4
@@ -24062,9 +34035,30 @@ define <4 x bfloat> @v_uitofp_v4i32_to_v4bf16(<4 x i32> %x) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX10-NEXT: v_cvt_f32_u32_e32 v3, v3
+; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_and_or_b32 v9, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_and_or_b32 v11, v1, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v4, v10, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v6, v3, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v7, v3, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v11, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX10-NEXT: v_perm_b32 v1, v3, v2, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -24073,10 +34067,32 @@ define <4 x bfloat> @v_uitofp_v4i32_to_v4bf16(<4 x i32> %x) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: v_cvt_f32_u32_e32 v3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX11-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v2, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v9, v0, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-NEXT: v_and_or_b32 v11, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_and_or_b32 v6, v3, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v4, v10, v1, 0x7fff
+; GFX11-NEXT: v_add3_u32 v5, v7, v3, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v4, v11, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_perm_b32 v1, v3, v2, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = uitofp <4 x i32> %x to <4 x bfloat>
@@ -24123,6 +34139,13 @@ define bfloat @v_uitofp_i64_to_bf16(i64 %x) {
; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX8-NEXT: v_sub_u32_e32 v1, vcc, 32, v2
; GFX8-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -24132,11 +34155,18 @@ define bfloat @v_uitofp_i64_to_bf16(i64 %x) {
; GFX9-NEXT: v_ffbh_u32_e32 v2, v1
; GFX9-NEXT: v_min_u32_e32 v2, 32, v2
; GFX9-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX9-NEXT: v_sub_u32_e32 v1, 32, v2
; GFX9-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -24144,6 +34174,7 @@ define bfloat @v_uitofp_i64_to_bf16(i64 %x) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v2, v1
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_min_u32_e32 v2, 32, v2
; GFX10-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
; GFX10-NEXT: v_min_u32_e32 v0, 1, v0
@@ -24151,6 +34182,11 @@ define bfloat @v_uitofp_i64_to_bf16(i64 %x) {
; GFX10-NEXT: v_sub_nc_u32_e32 v1, 32, v2
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX10-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -24158,6 +34194,7 @@ define bfloat @v_uitofp_i64_to_bf16(i64 %x) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_clz_i32_u32_e32 v2, v1
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_min_u32_e32 v2, 32, v2
; GFX11-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
@@ -24168,7 +34205,13 @@ define bfloat @v_uitofp_i64_to_bf16(i64 %x) {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX11-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = uitofp i64 %x to bfloat
@@ -24230,17 +34273,31 @@ define <2 x bfloat> @v_uitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX8-NEXT: v_lshlrev_b64 v[0:1], v4, v[0:1]
; GFX8-NEXT: v_min_u32_e32 v0, 1, v0
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: v_cvt_f32_u32_e32 v5, v0
+; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX8-NEXT: v_sub_u32_e32 v1, vcc, 32, v4
+; GFX8-NEXT: v_ldexp_f32 v4, v0, v1
+; GFX8-NEXT: v_bfe_u32 v0, v4, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v4
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v0
; GFX8-NEXT: v_ffbh_u32_e32 v0, v3
-; GFX8-NEXT: v_min_u32_e32 v6, 32, v0
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v6, v[2:3]
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v4
+; GFX8-NEXT: v_min_u32_e32 v7, 32, v0
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v7, v[2:3]
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX8-NEXT: v_min_u32_e32 v0, 1, v0
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX8-NEXT: v_ldexp_f32 v1, v5, v2
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v6
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc
+; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v7
; GFX8-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -24251,20 +34308,33 @@ define <2 x bfloat> @v_uitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX9-NEXT: v_ffbh_u32_e32 v4, v1
; GFX9-NEXT: v_min_u32_e32 v4, 32, v4
; GFX9-NEXT: v_lshlrev_b64 v[0:1], v4, v[0:1]
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX9-NEXT: v_or_b32_e32 v5, v1, v0
+; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX9-NEXT: v_sub_u32_e32 v1, 32, v4
+; GFX9-NEXT: v_ldexp_f32 v4, v0, v1
+; GFX9-NEXT: v_bfe_u32 v0, v4, 16, 1
+; GFX9-NEXT: v_add3_u32 v5, v0, v4, s4
; GFX9-NEXT: v_ffbh_u32_e32 v0, v3
-; GFX9-NEXT: v_min_u32_e32 v6, 32, v0
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v6, v[2:3]
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v5
+; GFX9-NEXT: v_min_u32_e32 v7, 32, v0
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[2:3]
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v4
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX9-NEXT: v_sub_u32_e32 v1, 32, v4
-; GFX9-NEXT: v_ldexp_f32 v1, v2, v1
-; GFX9-NEXT: v_sub_u32_e32 v2, 32, v6
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc
+; GFX9-NEXT: v_sub_u32_e32 v2, 32, v7
; GFX9-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -24273,6 +34343,7 @@ define <2 x bfloat> @v_uitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v4, v1
; GFX10-NEXT: v_ffbh_u32_e32 v5, v3
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_min_u32_e32 v4, 32, v4
; GFX10-NEXT: v_min_u32_e32 v5, 32, v5
; GFX10-NEXT: v_lshlrev_b64 v[0:1], v4, v[0:1]
@@ -24287,6 +34358,16 @@ define <2 x bfloat> @v_uitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX10-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX10-NEXT: v_ldexp_f32 v0, v0, v2
; GFX10-NEXT: v_ldexp_f32 v1, v1, v3
+; GFX10-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_and_or_b32 v5, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -24295,6 +34376,7 @@ define <2 x bfloat> @v_uitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_clz_i32_u32_e32 v4, v1
; GFX11-NEXT: v_clz_i32_u32_e32 v5, v3
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_min_u32_e32 v4, 32, v4
; GFX11-NEXT: v_min_u32_e32 v5, 32, v5
@@ -24315,6 +34397,18 @@ define <2 x bfloat> @v_uitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_ldexp_f32 v0, v0, v2
; GFX11-NEXT: v_ldexp_f32 v1, v1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_and_or_b32 v5, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -24393,28 +34487,50 @@ define <3 x bfloat> @v_uitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX8-NEXT: v_ffbh_u32_e32 v6, v5
; GFX8-NEXT: v_min_u32_e32 v6, 32, v6
; GFX8-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
+; GFX8-NEXT: v_ffbh_u32_e32 v7, v1
; GFX8-NEXT: v_min_u32_e32 v4, 1, v4
; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
; GFX8-NEXT: v_cvt_f32_u32_e32 v4, v4
; GFX8-NEXT: v_sub_u32_e32 v5, vcc, 32, v6
-; GFX8-NEXT: v_ldexp_f32 v6, v4, v5
-; GFX8-NEXT: v_ffbh_u32_e32 v4, v1
-; GFX8-NEXT: v_min_u32_e32 v7, 32, v4
-; GFX8-NEXT: v_lshlrev_b64 v[4:5], v7, v[0:1]
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX8-NEXT: v_min_u32_e32 v0, 1, v4
-; GFX8-NEXT: v_ffbh_u32_e32 v4, v3
-; GFX8-NEXT: v_min_u32_e32 v4, 32, v4
-; GFX8-NEXT: v_lshlrev_b64 v[2:3], v4, v[2:3]
-; GFX8-NEXT: v_or_b32_e32 v0, v5, v0
+; GFX8-NEXT: v_min_u32_e32 v7, 32, v7
+; GFX8-NEXT: v_ldexp_f32 v4, v4, v5
+; GFX8-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v4
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v4
+; GFX8-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX8-NEXT: v_ffbh_u32_e32 v6, v3
+; GFX8-NEXT: v_min_u32_e32 v6, 32, v6
+; GFX8-NEXT: v_lshlrev_b64 v[2:3], v6, v[2:3]
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX8-NEXT: v_sub_u32_e32 v4, vcc, 32, v7
+; GFX8-NEXT: v_ldexp_f32 v0, v0, v4
; GFX8-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX8-NEXT: v_bfe_u32 v4, v0, 16, 1
; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v0
; GFX8-NEXT: v_cvt_f32_u32_e32 v2, v2
-; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX8-NEXT: v_sub_u32_e32 v3, vcc, 32, v4
-; GFX8-NEXT: v_sub_u32_e32 v5, vcc, 32, v7
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX8-NEXT: v_sub_u32_e32 v3, vcc, 32, v6
; GFX8-NEXT: v_ldexp_f32 v2, v2, v3
-; GFX8-NEXT: v_ldexp_f32 v0, v0, v5
+; GFX8-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v2
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v2
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -24425,28 +34541,47 @@ define <3 x bfloat> @v_uitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX9-NEXT: v_ffbh_u32_e32 v6, v5
; GFX9-NEXT: v_min_u32_e32 v6, 32, v6
; GFX9-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
-; GFX9-NEXT: s_mov_b32 s4, 0x7060302
+; GFX9-NEXT: v_ffbh_u32_e32 v7, v1
; GFX9-NEXT: v_min_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_min_u32_e32 v7, 32, v7
; GFX9-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX9-NEXT: v_sub_u32_e32 v5, 32, v6
-; GFX9-NEXT: v_ffbh_u32_e32 v6, v1
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
; GFX9-NEXT: v_cvt_f32_u32_e32 v4, v4
-; GFX9-NEXT: v_min_u32_e32 v6, 32, v6
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1]
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT: v_sub_u32_e32 v5, 32, v6
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX9-NEXT: v_ldexp_f32 v4, v4, v5
-; GFX9-NEXT: v_or_b32_e32 v5, v1, v0
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v5, v5, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_sub_u32_e32 v1, 32, v7
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
+; GFX9-NEXT: v_ldexp_f32 v5, v0, v1
+; GFX9-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX9-NEXT: v_add3_u32 v6, v0, v5, s4
; GFX9-NEXT: v_ffbh_u32_e32 v0, v3
-; GFX9-NEXT: v_min_u32_e32 v7, 32, v0
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[2:3]
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v5
+; GFX9-NEXT: v_min_u32_e32 v8, 32, v0
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3]
+; GFX9-NEXT: v_and_b32_e32 v7, 0x80000000, v5
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX9-NEXT: v_sub_u32_e32 v1, 32, v6
-; GFX9-NEXT: v_ldexp_f32 v1, v2, v1
-; GFX9-NEXT: v_sub_u32_e32 v2, 32, v7
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
+; GFX9-NEXT: v_sub_u32_e32 v2, 32, v8
; GFX9-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: v_alignbit_b32 v1, s4, v4, 16
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -24455,31 +34590,47 @@ define <3 x bfloat> @v_uitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v6, v1
-; GFX10-NEXT: v_ffbh_u32_e32 v7, v3
-; GFX10-NEXT: v_ffbh_u32_e32 v8, v5
+; GFX10-NEXT: v_ffbh_u32_e32 v8, v3
+; GFX10-NEXT: v_ffbh_u32_e32 v7, v5
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_min_u32_e32 v6, 32, v6
-; GFX10-NEXT: v_min_u32_e32 v7, 32, v7
; GFX10-NEXT: v_min_u32_e32 v8, 32, v8
+; GFX10-NEXT: v_min_u32_e32 v7, 32, v7
; GFX10-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1]
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], v7, v[2:3]
-; GFX10-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5]
-; GFX10-NEXT: v_sub_nc_u32_e32 v8, 32, v8
+; GFX10-NEXT: v_lshlrev_b64 v[2:3], v8, v[2:3]
+; GFX10-NEXT: v_lshlrev_b64 v[4:5], v7, v[4:5]
+; GFX10-NEXT: v_sub_nc_u32_e32 v7, 32, v7
; GFX10-NEXT: v_min_u32_e32 v0, 1, v0
; GFX10-NEXT: v_min_u32_e32 v2, 1, v2
; GFX10-NEXT: v_min_u32_e32 v4, 1, v4
; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX10-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX10-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX10-NEXT: v_or_b32_e32 v2, v3, v2
; GFX10-NEXT: v_sub_nc_u32_e32 v3, 32, v6
-; GFX10-NEXT: v_sub_nc_u32_e32 v4, 32, v7
+; GFX10-NEXT: v_or_b32_e32 v1, v5, v4
+; GFX10-NEXT: v_sub_nc_u32_e32 v4, 32, v8
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX10-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX10-NEXT: v_cvt_f32_u32_e32 v2, v2
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX10-NEXT: v_ldexp_f32 v0, v0, v3
-; GFX10-NEXT: v_ldexp_f32 v1, v1, v4
-; GFX10-NEXT: v_ldexp_f32 v2, v2, v8
-; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
-; GFX10-NEXT: v_alignbit_b32 v1, s4, v2, 16
+; GFX10-NEXT: v_ldexp_f32 v2, v2, v4
+; GFX10-NEXT: v_ldexp_f32 v1, v1, v7
+; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX10-NEXT: v_and_or_b32 v7, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v2, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v6, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = uitofp <3 x i64> %x to <3 x bfloat>
ret <3 x bfloat> %op
@@ -24574,36 +34725,65 @@ define <4 x bfloat> @v_uitofp_v4i64_to_v4bf16(<4 x i64> %x) {
; GFX8-NEXT: v_ffbh_u32_e32 v8, v5
; GFX8-NEXT: v_min_u32_e32 v8, 32, v8
; GFX8-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5]
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
; GFX8-NEXT: v_min_u32_e32 v4, 1, v4
; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX8-NEXT: v_cvt_f32_u32_e32 v9, v4
+; GFX8-NEXT: v_cvt_f32_u32_e32 v4, v4
+; GFX8-NEXT: v_sub_u32_e32 v5, vcc, 32, v8
+; GFX8-NEXT: v_ldexp_f32 v8, v4, v5
+; GFX8-NEXT: v_bfe_u32 v4, v8, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v8
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, s4, v4
; GFX8-NEXT: v_ffbh_u32_e32 v4, v7
-; GFX8-NEXT: v_min_u32_e32 v10, 32, v4
-; GFX8-NEXT: v_lshlrev_b64 v[4:5], v10, v[6:7]
-; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 32, v8
+; GFX8-NEXT: v_min_u32_e32 v11, 32, v4
+; GFX8-NEXT: v_lshlrev_b64 v[4:5], v11, v[6:7]
+; GFX8-NEXT: v_and_b32_e32 v10, 0x80000000, v8
; GFX8-NEXT: v_min_u32_e32 v4, 1, v4
; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
; GFX8-NEXT: v_cvt_f32_u32_e32 v4, v4
-; GFX8-NEXT: v_ldexp_f32 v5, v9, v6
-; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 32, v10
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX8-NEXT: v_ffbh_u32_e32 v8, v1
+; GFX8-NEXT: v_min_u32_e32 v8, 32, v8
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v10
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1]
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v6, vcc
+; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 32, v11
; GFX8-NEXT: v_ldexp_f32 v4, v4, v6
-; GFX8-NEXT: v_ffbh_u32_e32 v6, v1
-; GFX8-NEXT: v_min_u32_e32 v6, 32, v6
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1]
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX8-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: v_cvt_f32_u32_e32 v7, v0
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, s4, v6
+; GFX8-NEXT: v_and_b32_e32 v7, 0x80000000, v4
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc
+; GFX8-NEXT: v_sub_u32_e32 v1, vcc, 32, v8
+; GFX8-NEXT: v_ldexp_f32 v6, v0, v1
+; GFX8-NEXT: v_bfe_u32 v0, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, s4, v0
; GFX8-NEXT: v_ffbh_u32_e32 v0, v3
-; GFX8-NEXT: v_min_u32_e32 v8, 32, v0
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3]
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v6
+; GFX8-NEXT: v_min_u32_e32 v9, 32, v0
+; GFX8-NEXT: v_lshlrev_b64 v[0:1], v9, v[2:3]
+; GFX8-NEXT: v_and_b32_e32 v8, 0x80000000, v6
; GFX8-NEXT: v_min_u32_e32 v0, 1, v0
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX8-NEXT: v_ldexp_f32 v1, v7, v2
-; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v8
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v8
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
+; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v9
; GFX8-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX8-NEXT: v_alignbit_b32 v1, v4, v5, 16
@@ -24615,99 +34795,145 @@ define <4 x bfloat> @v_uitofp_v4i64_to_v4bf16(<4 x i64> %x) {
; GFX9-NEXT: v_ffbh_u32_e32 v8, v5
; GFX9-NEXT: v_min_u32_e32 v8, 32, v8
; GFX9-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5]
-; GFX9-NEXT: v_sub_u32_e32 v8, 32, v8
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_min_u32_e32 v4, 1, v4
; GFX9-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX9-NEXT: v_cvt_f32_u32_e32 v9, v4
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, v4
+; GFX9-NEXT: v_sub_u32_e32 v5, 32, v8
+; GFX9-NEXT: v_ldexp_f32 v8, v4, v5
+; GFX9-NEXT: v_bfe_u32 v4, v8, 16, 1
+; GFX9-NEXT: v_add3_u32 v9, v4, v8, s4
; GFX9-NEXT: v_ffbh_u32_e32 v4, v7
-; GFX9-NEXT: v_min_u32_e32 v10, 32, v4
-; GFX9-NEXT: v_lshlrev_b64 v[4:5], v10, v[6:7]
-; GFX9-NEXT: v_ffbh_u32_e32 v7, v1
+; GFX9-NEXT: v_min_u32_e32 v11, 32, v4
+; GFX9-NEXT: v_lshlrev_b64 v[4:5], v11, v[6:7]
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_ffbh_u32_e32 v8, v1
; GFX9-NEXT: v_min_u32_e32 v4, 1, v4
+; GFX9-NEXT: v_min_u32_e32 v8, 32, v8
; GFX9-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1]
; GFX9-NEXT: v_cvt_f32_u32_e32 v4, v4
-; GFX9-NEXT: v_min_u32_e32 v7, 32, v7
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
-; GFX9-NEXT: v_sub_u32_e32 v5, 32, v10
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX9-NEXT: v_ldexp_f32 v4, v4, v5
-; GFX9-NEXT: v_or_b32_e32 v5, v1, v0
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v10
+; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v9, v6, vcc
+; GFX9-NEXT: v_sub_u32_e32 v6, 32, v11
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX9-NEXT: v_ldexp_f32 v4, v4, v6
+; GFX9-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v7, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v6, v6, v4, s4
+; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_sub_u32_e32 v1, 32, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc
+; GFX9-NEXT: v_ldexp_f32 v6, v0, v1
+; GFX9-NEXT: v_bfe_u32 v0, v6, 16, 1
+; GFX9-NEXT: v_add3_u32 v7, v0, v6, s4
; GFX9-NEXT: v_ffbh_u32_e32 v0, v3
-; GFX9-NEXT: v_ldexp_f32 v6, v9, v8
-; GFX9-NEXT: v_min_u32_e32 v8, 32, v0
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3]
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v5
+; GFX9-NEXT: v_min_u32_e32 v9, 32, v0
+; GFX9-NEXT: v_lshlrev_b64 v[0:1], v9, v[2:3]
+; GFX9-NEXT: v_and_b32_e32 v8, 0x80000000, v6
; GFX9-NEXT: v_min_u32_e32 v0, 1, v0
; GFX9-NEXT: v_or_b32_e32 v0, v1, v0
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX9-NEXT: v_sub_u32_e32 v1, 32, v7
-; GFX9-NEXT: v_ldexp_f32 v1, v2, v1
-; GFX9-NEXT: v_sub_u32_e32 v2, 32, v8
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
+; GFX9-NEXT: v_sub_u32_e32 v2, 32, v9
; GFX9-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX9-NEXT: v_perm_b32 v1, v4, v6, s4
+; GFX9-NEXT: v_perm_b32 v1, v4, v5, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_uitofp_v4i64_to_v4bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v8, v5
-; GFX10-NEXT: v_ffbh_u32_e32 v9, v1
-; GFX10-NEXT: v_ffbh_u32_e32 v10, v3
-; GFX10-NEXT: v_ffbh_u32_e32 v11, v7
+; GFX10-NEXT: v_ffbh_u32_e32 v10, v1
+; GFX10-NEXT: v_ffbh_u32_e32 v11, v3
+; GFX10-NEXT: v_ffbh_u32_e32 v9, v7
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_min_u32_e32 v8, 32, v8
-; GFX10-NEXT: v_min_u32_e32 v9, 32, v9
; GFX10-NEXT: v_min_u32_e32 v10, 32, v10
; GFX10-NEXT: v_min_u32_e32 v11, 32, v11
+; GFX10-NEXT: v_min_u32_e32 v9, 32, v9
; GFX10-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5]
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1]
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], v10, v[2:3]
-; GFX10-NEXT: v_lshlrev_b64 v[6:7], v11, v[6:7]
+; GFX10-NEXT: v_lshlrev_b64 v[0:1], v10, v[0:1]
+; GFX10-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
+; GFX10-NEXT: v_lshlrev_b64 v[6:7], v9, v[6:7]
; GFX10-NEXT: v_sub_nc_u32_e32 v8, 32, v8
+; GFX10-NEXT: v_sub_nc_u32_e32 v9, 32, v9
; GFX10-NEXT: v_min_u32_e32 v4, 1, v4
; GFX10-NEXT: v_min_u32_e32 v0, 1, v0
; GFX10-NEXT: v_min_u32_e32 v2, 1, v2
; GFX10-NEXT: v_min_u32_e32 v6, 1, v6
; GFX10-NEXT: v_or_b32_e32 v4, v5, v4
; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX10-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX10-NEXT: v_or_b32_e32 v3, v7, v6
-; GFX10-NEXT: v_sub_nc_u32_e32 v1, 32, v11
-; GFX10-NEXT: v_cvt_f32_u32_e32 v4, v4
+; GFX10-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX10-NEXT: v_sub_nc_u32_e32 v5, 32, v10
+; GFX10-NEXT: v_sub_nc_u32_e32 v3, 32, v11
+; GFX10-NEXT: v_cvt_f32_u32_e32 v2, v4
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX10-NEXT: v_sub_nc_u32_e32 v5, 32, v9
-; GFX10-NEXT: v_cvt_f32_u32_e32 v2, v2
-; GFX10-NEXT: v_sub_nc_u32_e32 v6, 32, v10
-; GFX10-NEXT: v_cvt_f32_u32_e32 v3, v3
-; GFX10-NEXT: v_ldexp_f32 v4, v4, v8
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX10-NEXT: v_or_b32_e32 v6, v7, v6
+; GFX10-NEXT: v_ldexp_f32 v2, v2, v8
; GFX10-NEXT: v_ldexp_f32 v0, v0, v5
-; GFX10-NEXT: v_ldexp_f32 v2, v2, v6
-; GFX10-NEXT: v_ldexp_f32 v1, v3, v1
-; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
-; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX10-NEXT: v_ldexp_f32 v1, v1, v3
+; GFX10-NEXT: v_cvt_f32_u32_e32 v4, v6
+; GFX10-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX10-NEXT: v_and_or_b32 v5, v2, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_ldexp_f32 v4, v4, v9
+; GFX10-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v3, v9, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v5, v1, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v9, v4, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v6, v9, vcc_lo
+; GFX10-NEXT: v_perm_b32 v1, v3, v2, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_uitofp_v4i64_to_v4bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_clz_i32_u32_e32 v8, v5
-; GFX11-NEXT: v_clz_i32_u32_e32 v9, v1
-; GFX11-NEXT: v_clz_i32_u32_e32 v10, v3
-; GFX11-NEXT: v_clz_i32_u32_e32 v11, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_clz_i32_u32_e32 v10, v1
+; GFX11-NEXT: v_clz_i32_u32_e32 v11, v3
+; GFX11-NEXT: v_clz_i32_u32_e32 v9, v7
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: v_min_u32_e32 v8, 32, v8
-; GFX11-NEXT: v_min_u32_e32 v9, 32, v9
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_min_u32_e32 v10, 32, v10
; GFX11-NEXT: v_min_u32_e32 v11, 32, v11
+; GFX11-NEXT: v_min_u32_e32 v9, 32, v9
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5]
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1]
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v10, v[0:1]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], v10, v[2:3]
-; GFX11-NEXT: v_lshlrev_b64 v[6:7], v11, v[6:7]
+; GFX11-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3]
+; GFX11-NEXT: v_lshlrev_b64 v[6:7], v9, v[6:7]
; GFX11-NEXT: v_sub_nc_u32_e32 v8, 32, v8
+; GFX11-NEXT: v_sub_nc_u32_e32 v9, 32, v9
; GFX11-NEXT: v_min_u32_e32 v4, 1, v4
; GFX11-NEXT: v_min_u32_e32 v0, 1, v0
; GFX11-NEXT: v_min_u32_e32 v2, 1, v2
@@ -24715,24 +34941,45 @@ define <4 x bfloat> @v_uitofp_v4i64_to_v4bf16(<4 x i64> %x) {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_or_b32_e32 v4, v5, v4
; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX11-NEXT: v_or_b32_e32 v3, v7, v6
-; GFX11-NEXT: v_sub_nc_u32_e32 v1, 32, v11
-; GFX11-NEXT: v_cvt_f32_u32_e32 v4, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-NEXT: v_sub_nc_u32_e32 v5, 32, v10
+; GFX11-NEXT: v_or_b32_e32 v6, v7, v6
+; GFX11-NEXT: v_cvt_f32_u32_e32 v2, v4
+; GFX11-NEXT: v_sub_nc_u32_e32 v3, 32, v11
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11-NEXT: v_sub_nc_u32_e32 v5, 32, v9
-; GFX11-NEXT: v_cvt_f32_u32_e32 v2, v2
-; GFX11-NEXT: v_sub_nc_u32_e32 v6, 32, v10
-; GFX11-NEXT: v_cvt_f32_u32_e32 v3, v3
-; GFX11-NEXT: v_ldexp_f32 v4, v4, v8
+; GFX11-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX11-NEXT: v_cvt_f32_u32_e32 v4, v6
+; GFX11-NEXT: v_ldexp_f32 v2, v2, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_ldexp_f32 v0, v0, v5
+; GFX11-NEXT: v_ldexp_f32 v1, v1, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_ldexp_f32 v2, v2, v6
-; GFX11-NEXT: v_ldexp_f32 v1, v3, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
-; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
+; GFX11-NEXT: v_ldexp_f32 v4, v4, v9
+; GFX11-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v5, v2, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v8, v0, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v3, v9, v1, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v5, v1, s0, 0x400000
+; GFX11-NEXT: v_and_or_b32 v9, v4, s0, 0x400000
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v6, v9, vcc_lo
+; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, v3, v2, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = uitofp <4 x i64> %x to <4 x bfloat>
ret <4 x bfloat> %op
@@ -24742,6 +34989,8 @@ define bfloat @v_select_bf16(i1 %cond, bfloat %a, bfloat %b) {
; GCN-LABEL: v_select_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
@@ -24752,6 +35001,8 @@ define bfloat @v_select_bf16(i1 %cond, bfloat %a, bfloat %b) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GFX7-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -24797,8 +35048,9 @@ define bfloat @v_select_fneg_lhs_bf16(i1 %cond, bfloat %a, bfloat %b) {
; GCN-LABEL: v_select_fneg_lhs_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GCN-NEXT: v_and_b32_e32 v0, 1, v0
-; GCN-NEXT: v_xor_b32_e32 v1, 0x80000000, v1
+; GCN-NEXT: v_mul_f32_e32 v1, -1.0, v1
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -24808,7 +35060,8 @@ define bfloat @v_select_fneg_lhs_bf16(i1 %cond, bfloat %a, bfloat %b) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX7-NEXT: v_xor_b32_e32 v1, 0x80000000, v1
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, -1.0, v1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GFX7-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -24859,8 +35112,9 @@ define bfloat @v_select_fneg_rhs_bf16(i1 %cond, bfloat %a, bfloat %b) {
; GCN-LABEL: v_select_fneg_rhs_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v0, 1, v0
-; GCN-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
+; GCN-NEXT: v_mul_f32_e32 v2, -1.0, v2
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -24870,7 +35124,8 @@ define bfloat @v_select_fneg_rhs_bf16(i1 %cond, bfloat %a, bfloat %b) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX7-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v2, -1.0, v2
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GFX7-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -24921,11 +35176,15 @@ define <2 x bfloat> @v_select_v2bf16(i1 %cond, <2 x bfloat> %a, <2 x bfloat> %b)
; GCN-LABEL: v_select_v2bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GCN-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
; GCN-NEXT: v_cndmask_b32_e32 v0, v3, v1, vcc
@@ -24936,6 +35195,10 @@ define <2 x bfloat> @v_select_v2bf16(i1 %cond, <2 x bfloat> %a, <2 x bfloat> %b)
; GFX7-LABEL: v_select_v2bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
@@ -25006,7 +35269,11 @@ define <2 x bfloat> @v_vselect_v2bf16(<2 x i1> %cond, <2 x bfloat> %a, <2 x bflo
; GCN-LABEL: v_vselect_v2bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GCN-NEXT: v_and_b32_e32 v0, 1, v0
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_and_b32_e32 v1, 1, v1
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
; GCN-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
@@ -25021,7 +35288,11 @@ define <2 x bfloat> @v_vselect_v2bf16(<2 x i1> %cond, <2 x bfloat> %a, <2 x bflo
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, 1, v1
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
@@ -25094,20 +35365,20 @@ define <2 x bfloat> @v_vselect_v2bf16(<2 x i1> %cond, <2 x bfloat> %a, <2 x bflo
define amdgpu_ps i32 @s_select_bf16(bfloat inreg %a, bfloat inreg %b, i32 %c) {
; GCN-LABEL: s_select_bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s0
+; GCN-NEXT: v_mul_f32_e64 v1, 1.0, s0
+; GCN-NEXT: v_mul_f32_e64 v2, 1.0, s1
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GCN-NEXT: v_readfirstlane_b32 s0, v0
; GCN-NEXT: ; return to shader part epilog
;
; GFX7-LABEL: s_select_bf16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
+; GFX7-NEXT: v_mul_f32_e64 v1, 1.0, s0
+; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: ; return to shader part epilog
@@ -25162,17 +35433,17 @@ define amdgpu_ps i32 @s_select_bf16(bfloat inreg %a, bfloat inreg %b, i32 %c) {
define amdgpu_ps i32 @s_select_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b, i32 %c) {
; GCN-LABEL: s_select_v2bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_lshr_b32 s1, s1, 16
-; GCN-NEXT: s_lshr_b32 s0, s0, 16
-; GCN-NEXT: s_lshr_b32 s2, s2, 16
-; GCN-NEXT: s_lshr_b32 s3, s3, 16
-; GCN-NEXT: v_mov_b32_e32 v1, s2
-; GCN-NEXT: v_mov_b32_e32 v2, s0
+; GCN-NEXT: v_mul_f32_e64 v1, 1.0, s1
+; GCN-NEXT: v_mul_f32_e64 v2, 1.0, s3
+; GCN-NEXT: v_mul_f32_e64 v3, 1.0, s0
+; GCN-NEXT: v_mul_f32_e64 v4, 1.0, s2
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; GCN-NEXT: v_mov_b32_e32 v1, s3
-; GCN-NEXT: v_mov_b32_e32 v2, s1
-; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
; GCN-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GCN-NEXT: v_or_b32_e32 v0, v0, v1
; GCN-NEXT: v_readfirstlane_b32 s0, v0
@@ -25180,17 +35451,17 @@ define amdgpu_ps i32 @s_select_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX7-LABEL: s_select_v2bf16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_lshr_b32 s0, s0, 16
-; GFX7-NEXT: s_lshr_b32 s2, s2, 16
-; GFX7-NEXT: s_lshr_b32 s1, s1, 16
-; GFX7-NEXT: s_lshr_b32 s3, s3, 16
-; GFX7-NEXT: v_mov_b32_e32 v1, s2
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
+; GFX7-NEXT: v_mul_f32_e64 v1, 1.0, s1
+; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s3
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e64 v3, 1.0, s0
+; GFX7-NEXT: v_mul_f32_e64 v4, 1.0, s2
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; GFX7-NEXT: v_mov_b32_e32 v1, s3
-; GFX7-NEXT: v_mov_b32_e32 v2, s1
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
@@ -25265,14 +35536,14 @@ define amdgpu_ps i32 @s_select_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
define amdgpu_ps i32 @s_vselect_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg %b, <2 x i32> %c) {
; GCN-LABEL: s_vselect_v2bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v2, s3
-; GCN-NEXT: v_mov_b32_e32 v3, s1
+; GCN-NEXT: v_mul_f32_e64 v2, 1.0, s0
+; GCN-NEXT: v_mul_f32_e64 v3, 1.0, s2
+; GCN-NEXT: v_mul_f32_e64 v4, 1.0, s1
+; GCN-NEXT: v_mul_f32_e64 v5, 1.0, s3
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
-; GCN-NEXT: v_mov_b32_e32 v2, s2
-; GCN-NEXT: v_mov_b32_e32 v3, s0
+; GCN-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GCN-NEXT: v_or_b32_e32 v0, v0, v1
@@ -25281,14 +35552,14 @@ define amdgpu_ps i32 @s_vselect_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX7-LABEL: s_vselect_v2bf16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: v_mov_b32_e32 v2, s3
-; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mul_f32_e64 v4, 1.0, s1
+; GFX7-NEXT: v_mul_f32_e64 v5, 1.0, s3
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s0
+; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s0
+; GFX7-NEXT: v_mul_f32_e64 v3, 1.0, s2
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
@@ -25369,37 +35640,49 @@ define <3 x bfloat> @v_select_v3bf16(i1 %cond, <3 x bfloat> %a, <3 x bfloat> %b)
; GCN-LABEL: v_select_v3bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GCN-NEXT: v_and_b32_e32 v0, 1, v0
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GCN-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GCN-NEXT: v_alignbit_b32 v2, v5, v4, 16
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GCN-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v0, v6, v3, vcc
+; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_select_v3bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v4, 16
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v2, v2, v4, 16
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v3, vcc
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_select_v3bf16:
@@ -25445,11 +35728,19 @@ define <4 x bfloat> @v_select_v4bf16(i1 %cond, <4 x bfloat> %a, <4 x bfloat> %b)
; GCN-LABEL: v_select_v4bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GCN-NEXT: v_alignbit_b32 v2, v6, v5, 16
; GCN-NEXT: v_alignbit_b32 v3, v4, v3, 16
@@ -25466,15 +35757,23 @@ define <4 x bfloat> @v_select_v4bf16(i1 %cond, <4 x bfloat> %a, <4 x bfloat> %b)
; GFX7-LABEL: v_select_v4bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v8
-; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v8
; GFX7-NEXT: v_alignbit_b32 v2, v2, v5, 16
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v7
+; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
@@ -25527,13 +35826,25 @@ define <6 x bfloat> @v_select_v6bf16(i1 %cond, <6 x bfloat> %a, <6 x bfloat> %b)
; GCN-LABEL: v_select_v6bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GCN-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; GCN-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GCN-NEXT: v_alignbit_b32 v2, v8, v7, 16
; GCN-NEXT: v_alignbit_b32 v3, v4, v3, 16
@@ -25555,19 +35866,31 @@ define <6 x bfloat> @v_select_v6bf16(i1 %cond, <6 x bfloat> %a, <6 x bfloat> %b)
; GFX7-LABEL: v_select_v6bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v8
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v10
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v10
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_alignbit_b32 v2, v2, v7, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v9
; GFX7-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v12
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v12
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v11
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v7, 16
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v9, 16
-; GFX7-NEXT: v_alignbit_b32 v6, v6, v11, 16
+; GFX7-NEXT: v_alignbit_b32 v6, v6, v7, 16
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GFX7-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
@@ -25627,6 +35950,23 @@ define <8 x bfloat> @v_select_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat> %b)
; GCN-LABEL: v_select_v8bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GCN-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v4
@@ -25635,7 +35975,6 @@ define <8 x bfloat> @v_select_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat> %b)
; GCN-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; GCN-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GCN-NEXT: v_alignbit_b32 v2, v10, v9, 16
; GCN-NEXT: v_alignbit_b32 v3, v4, v3, 16
@@ -25662,23 +36001,39 @@ define <8 x bfloat> @v_select_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat> %b)
; GFX7-LABEL: v_select_v8bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v10
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v10
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GFX7-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v12
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v12
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_alignbit_b32 v2, v2, v9, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v11
; GFX7-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v14
+; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v9, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v13
; GFX7-NEXT: v_alignbit_b32 v7, v8, v7, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v16
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v16
+; GFX7-NEXT: v_alignbit_b32 v6, v6, v9, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v15
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v9, 16
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v11, 16
-; GFX7-NEXT: v_alignbit_b32 v6, v6, v13, 16
-; GFX7-NEXT: v_alignbit_b32 v8, v8, v15, 16
+; GFX7-NEXT: v_alignbit_b32 v8, v8, v9, 16
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GFX7-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc
@@ -25744,47 +36099,81 @@ define <16 x bfloat> @v_select_v16bf16(i1 %cond, <16 x bfloat> %a, <16 x bfloat>
; GCN-LABEL: v_select_v16bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_and_b32_e32 v0, 1, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v18
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v17
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GCN-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; GCN-NEXT: v_alignbit_b32 v2, v2, v17, 16
+; GCN-NEXT: v_alignbit_b32 v2, v2, v3, 16
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v20
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v19
+; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT: v_alignbit_b32 v3, v3, v4, 16
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GCN-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v20
-; GCN-NEXT: v_alignbit_b32 v4, v4, v19, 16
+; GCN-NEXT: v_alignbit_b32 v4, v4, v5, 16
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v22
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v21
+; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT: v_alignbit_b32 v5, v5, v6, 16
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GCN-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v22
-; GCN-NEXT: v_alignbit_b32 v6, v6, v21, 16
+; GCN-NEXT: v_alignbit_b32 v6, v6, v7, 16
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v24
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v23
+; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT: v_alignbit_b32 v7, v7, v8, 16
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GCN-NEXT: v_alignbit_b32 v7, v8, v7, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v24
-; GCN-NEXT: v_alignbit_b32 v8, v8, v23, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; GCN-NEXT: v_alignbit_b32 v9, v10, v9, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v10, 16, v26
-; GCN-NEXT: v_alignbit_b32 v10, v10, v25, 16
+; GCN-NEXT: v_alignbit_b32 v8, v8, v9, 16
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v26
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v25
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v28
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v27
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v30
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v29
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT: v_alignbit_b32 v9, v9, v10, 16
+; GCN-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:4
; GCN-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; GCN-NEXT: v_lshrrev_b32_e32 v17, 16, v28
+; GCN-NEXT: v_alignbit_b32 v11, v12, v11, 16
+; GCN-NEXT: buffer_load_dword v12, off, s[0:3], s32
+; GCN-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; GCN-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; GCN-NEXT: v_lshrrev_b32_e32 v18, 16, v30
+; GCN-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; GCN-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; GCN-NEXT: v_alignbit_b32 v11, v12, v11, 16
-; GCN-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:4
-; GCN-NEXT: v_alignbit_b32 v12, v17, v27, 16
-; GCN-NEXT: buffer_load_dword v17, off, s[0:3], s32
-; GCN-NEXT: v_and_b32_e32 v0, 1, v0
+; GCN-NEXT: v_alignbit_b32 v17, v17, v18, 16
; GCN-NEXT: v_alignbit_b32 v13, v14, v13, 16
-; GCN-NEXT: v_alignbit_b32 v14, v18, v29, 16
+; GCN-NEXT: v_alignbit_b32 v14, v19, v20, 16
; GCN-NEXT: v_alignbit_b32 v15, v16, v15, 16
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GCN-NEXT: v_cndmask_b32_e32 v13, v14, v13, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v11, v12, v11, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v9, v10, v9, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v11, v17, v11, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v9, v9, v8, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v7, v7, v6, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v3, v3, v2, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v10
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v12
; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -25799,10 +36188,8 @@ define <16 x bfloat> @v_select_v16bf16(i1 %cond, <16 x bfloat> %a, <16 x bfloat>
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GCN-NEXT: v_lshlrev_b32_e32 v12, 16, v13
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_lshrrev_b32_e32 v14, 16, v19
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v14, v14, v17, 16
+; GCN-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GCN-NEXT: v_alignbit_b32 v14, v14, v16, 16
; GCN-NEXT: v_cndmask_b32_e32 v15, v14, v15, vcc
; GCN-NEXT: v_lshlrev_b32_e32 v14, 16, v15
; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
@@ -25811,44 +36198,74 @@ define <16 x bfloat> @v_select_v16bf16(i1 %cond, <16 x bfloat> %a, <16 x bfloat>
; GFX7-LABEL: v_select_v16bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_alignbit_b32 v11, v12, v11, 16
-; GFX7-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:4
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; GFX7-NEXT: buffer_load_dword v18, off, s[0:3], s32
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v18
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_alignbit_b32 v7, v8, v7, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v24
-; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GFX7-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v20
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v20
; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_alignbit_b32 v8, v8, v23, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GFX7-NEXT: v_alignbit_b32 v2, v2, v17, 16
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v19, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v19
; GFX7-NEXT: v_alignbit_b32 v5, v6, v5, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v22
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v22
+; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v17, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v21
+; GFX7-NEXT: v_alignbit_b32 v7, v8, v7, 16
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v24
+; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_alignbit_b32 v6, v6, v17, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v23
; GFX7-NEXT: v_alignbit_b32 v9, v10, v9, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v26
-; GFX7-NEXT: v_lshrrev_b32_e32 v17, 16, v28
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v26
+; GFX7-NEXT: v_alignbit_b32 v8, v8, v17, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v25
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_alignbit_b32 v10, v10, v17, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v28
+; GFX7-NEXT: v_alignbit_b32 v11, v12, v11, 16
+; GFX7-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:4
+; GFX7-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v27
+; GFX7-NEXT: v_alignbit_b32 v17, v17, v18, 16
+; GFX7-NEXT: buffer_load_dword v18, off, s[0:3], s32
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; GFX7-NEXT: v_lshrrev_b32_e32 v19, 16, v30
-; GFX7-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc
-; GFX7-NEXT: v_alignbit_b32 v6, v6, v21, 16
-; GFX7-NEXT: v_alignbit_b32 v10, v10, v25, 16
-; GFX7-NEXT: v_alignbit_b32 v17, v17, v27, 16
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GFX7-NEXT: v_alignbit_b32 v13, v14, v13, 16
-; GFX7-NEXT: v_alignbit_b32 v14, v19, v29, 16
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v30
+; GFX7-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v29
; GFX7-NEXT: v_alignbit_b32 v15, v16, v15, 16
+; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT: v_alignbit_b32 v14, v14, v19, 16
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GFX7-NEXT: v_cndmask_b32_e32 v13, v14, v13, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v11, v17, v11, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v9, v10, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
@@ -25860,17 +36277,19 @@ define <16 x bfloat> @v_select_v16bf16(i1 %cond, <16 x bfloat> %a, <16 x bfloat>
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v7
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v9
+; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v11
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v12
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v18
+; GFX7-NEXT: v_alignbit_b32 v12, v12, v16, 16
+; GFX7-NEXT: v_cndmask_b32_e32 v15, v12, v15, vcc
; GFX7-NEXT: v_lshlrev_b32_e32 v12, 16, v13
; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_alignbit_b32 v8, v8, v18, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v15, v8, v15, vcc
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v9
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; GFX7-NEXT: v_lshlrev_b32_e32 v14, 16, v15
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -25939,156 +36358,220 @@ define <32 x bfloat> @v_select_v32bf16(i1 %cond, <32 x bfloat> %a, <32 x bfloat>
; GCN-LABEL: v_select_v32bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GCN-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GCN-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GCN-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v8
-; GCN-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v10
-; GCN-NEXT: v_alignbit_b32 v5, v5, v9, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v12
-; GCN-NEXT: v_alignbit_b32 v6, v6, v11, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v14
-; GCN-NEXT: v_alignbit_b32 v7, v7, v13, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v16
-; GCN-NEXT: v_alignbit_b32 v8, v8, v15, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v18
-; GCN-NEXT: v_alignbit_b32 v9, v9, v17, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v10, 16, v20
-; GCN-NEXT: v_alignbit_b32 v10, v10, v19, 16
-; GCN-NEXT: v_lshrrev_b32_e32 v11, 16, v22
-; GCN-NEXT: v_alignbit_b32 v11, v11, v21, 16
-; GCN-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:12
-; GCN-NEXT: v_lshrrev_b32_e32 v12, 16, v24
-; GCN-NEXT: v_alignbit_b32 v12, v12, v23, 16
-; GCN-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:8
-; GCN-NEXT: v_lshrrev_b32_e32 v13, 16, v26
-; GCN-NEXT: v_alignbit_b32 v13, v13, v25, 16
-; GCN-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:20
-; GCN-NEXT: v_lshrrev_b32_e32 v14, 16, v28
-; GCN-NEXT: v_alignbit_b32 v14, v14, v27, 16
-; GCN-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:16
-; GCN-NEXT: v_lshrrev_b32_e32 v15, 16, v30
-; GCN-NEXT: v_alignbit_b32 v15, v15, v29, 16
-; GCN-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:28
; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GCN-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:24
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GCN-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v3
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v5
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GCN-NEXT: v_alignbit_b32 v2, v2, v3, 16
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v7
+; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT: v_alignbit_b32 v3, v3, v4, 16
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v9
+; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GCN-NEXT: v_alignbit_b32 v4, v4, v5, 16
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v11
+; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT: v_alignbit_b32 v5, v5, v6, 16
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v13
+; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GCN-NEXT: v_alignbit_b32 v6, v6, v7, 16
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v16
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v15
+; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT: v_alignbit_b32 v7, v7, v8, 16
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v18
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v17
+; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GCN-NEXT: v_alignbit_b32 v8, v8, v9, 16
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v20
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v19
+; GCN-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GCN-NEXT: v_alignbit_b32 v9, v9, v10, 16
+; GCN-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:12
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v22
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v21
+; GCN-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GCN-NEXT: v_alignbit_b32 v10, v10, v11, 16
+; GCN-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:8
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v24
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v23
+; GCN-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; GCN-NEXT: v_alignbit_b32 v11, v11, v12, 16
+; GCN-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:20
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v26
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v25
+; GCN-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GCN-NEXT: v_alignbit_b32 v12, v12, v13, 16
+; GCN-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:16
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v28
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v27
+; GCN-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GCN-NEXT: v_alignbit_b32 v13, v13, v14, 16
+; GCN-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:28
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v30
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v29
+; GCN-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GCN-NEXT: v_alignbit_b32 v14, v14, v20, 16
+; GCN-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:24
; GCN-NEXT: s_waitcnt vmcnt(5)
-; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v16
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GCN-NEXT: s_waitcnt vmcnt(4)
-; GCN-NEXT: v_alignbit_b32 v0, v0, v17, 16
-; GCN-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:36
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GCN-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GCN-NEXT: v_alignbit_b32 v15, v15, v16, 16
+; GCN-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:36
; GCN-NEXT: s_waitcnt vmcnt(4)
-; GCN-NEXT: v_lshrrev_b32_e32 v16, 16, v18
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v17
; GCN-NEXT: s_waitcnt vmcnt(3)
-; GCN-NEXT: v_alignbit_b32 v16, v16, v19, 16
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v18
+; GCN-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GCN-NEXT: v_alignbit_b32 v16, v16, v17, 16
; GCN-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:32
; GCN-NEXT: s_waitcnt vmcnt(3)
-; GCN-NEXT: v_lshrrev_b32_e32 v17, 16, v20
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v19
; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_alignbit_b32 v17, v17, v21, 16
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v20
+; GCN-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; GCN-NEXT: v_alignbit_b32 v17, v17, v19, 16
; GCN-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:44
-; GCN-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:40
-; GCN-NEXT: s_waitcnt vmcnt(3)
-; GCN-NEXT: v_lshrrev_b32_e32 v21, 16, v22
; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_alignbit_b32 v18, v21, v18, 16
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v21
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_lshrrev_b32_e32 v19, 16, v19
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GCN-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:40
+; GCN-NEXT: v_lshrrev_b32_e32 v20, 16, v20
+; GCN-NEXT: v_alignbit_b32 v18, v20, v18, 16
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v21
+; GCN-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:52
+; GCN-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:48
+; GCN-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; GCN-NEXT: v_alignbit_b32 v19, v19, v20, 16
-; GCN-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:52
-; GCN-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:48
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v21
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v22
; GCN-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:60
; GCN-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:56
-; GCN-NEXT: s_waitcnt vmcnt(3)
; GCN-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; GCN-NEXT: s_waitcnt vmcnt(2)
; GCN-NEXT: v_alignbit_b32 v20, v20, v21, 16
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_lshrrev_b32_e32 v21, 16, v22
+; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v22
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v23
+; GCN-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:68
+; GCN-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:64
+; GCN-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; GCN-NEXT: v_alignbit_b32 v21, v21, v22, 16
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v23
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v21, v21, v23, 16
-; GCN-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:68
-; GCN-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:64
+; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v24
; GCN-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:76
; GCN-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:72
-; GCN-NEXT: s_waitcnt vmcnt(3)
; GCN-NEXT: v_lshrrev_b32_e32 v22, 16, v22
-; GCN-NEXT: s_waitcnt vmcnt(2)
; GCN-NEXT: v_alignbit_b32 v22, v22, v23, 16
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_lshrrev_b32_e32 v23, 16, v24
+; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v24
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v23, v23, v25, 16
-; GCN-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:84
-; GCN-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:80
+; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v25
+; GCN-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:84
+; GCN-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:80
+; GCN-NEXT: v_lshrrev_b32_e32 v23, 16, v23
+; GCN-NEXT: v_alignbit_b32 v23, v23, v24, 16
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v24, 1.0, v25
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v26
; GCN-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:92
; GCN-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:88
-; GCN-NEXT: s_waitcnt vmcnt(3)
; GCN-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; GCN-NEXT: s_waitcnt vmcnt(2)
; GCN-NEXT: v_alignbit_b32 v24, v24, v25, 16
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_lshrrev_b32_e32 v25, 16, v26
+; GCN-NEXT: v_mul_f32_e32 v25, 1.0, v26
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v27
+; GCN-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:100
+; GCN-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:96
+; GCN-NEXT: v_lshrrev_b32_e32 v25, 16, v25
+; GCN-NEXT: v_alignbit_b32 v25, v25, v26, 16
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v27
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v25, v25, v27, 16
-; GCN-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:100
-; GCN-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:96
+; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v28
; GCN-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:108
; GCN-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:104
-; GCN-NEXT: s_waitcnt vmcnt(3)
; GCN-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; GCN-NEXT: s_waitcnt vmcnt(2)
; GCN-NEXT: v_alignbit_b32 v26, v26, v27, 16
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_lshrrev_b32_e32 v27, 16, v28
+; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v28
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v29
+; GCN-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:116
+; GCN-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:112
+; GCN-NEXT: v_lshrrev_b32_e32 v27, 16, v27
+; GCN-NEXT: v_alignbit_b32 v27, v27, v28, 16
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v29
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v27, v27, v29, 16
-; GCN-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:116
-; GCN-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:112
+; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v30
; GCN-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:124
; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120
-; GCN-NEXT: s_waitcnt vmcnt(3)
; GCN-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; GCN-NEXT: s_waitcnt vmcnt(2)
; GCN-NEXT: v_alignbit_b32 v28, v28, v29, 16
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_lshrrev_b32_e32 v29, 16, v30
+; GCN-NEXT: v_mul_f32_e32 v29, 1.0, v30
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v29, v29, v31, 16
-; GCN-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:4
-; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v31
+; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32
+; GCN-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; GCN-NEXT: v_alignbit_b32 v29, v29, v30, 16
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v30, 1.0, v31
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v31, 1.0, v32
; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:132
; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:128
-; GCN-NEXT: s_waitcnt vmcnt(3)
; GCN-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; GCN-NEXT: s_waitcnt vmcnt(2)
; GCN-NEXT: v_alignbit_b32 v30, v30, v31, 16
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_lshrrev_b32_e32 v31, 16, v32
+; GCN-NEXT: v_mul_f32_e32 v31, 1.0, v32
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_alignbit_b32 v31, v31, v33, 16
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v33
+; GCN-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; GCN-NEXT: v_alignbit_b32 v31, v31, v32, 16
; GCN-NEXT: v_cndmask_b32_e32 v31, v31, v30, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v29, v29, v15, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v28, v28, v14, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v27, v27, v13, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v26, v26, v12, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v25, v25, v11, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v24, v24, v10, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v23, v23, v9, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v15, v22, v8, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v13, v21, v7, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v11, v20, v6, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v9, v19, v5, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v7, v18, v4, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v5, v17, v3, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v3, v16, v2, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v29, v29, v14, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v28, v28, v13, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v27, v27, v12, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v26, v26, v11, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v25, v25, v10, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v24, v24, v9, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v23, v23, v8, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v22, v22, v7, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v13, v21, v6, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v11, v20, v5, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v9, v19, v4, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v7, v18, v3, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v5, v17, v2, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v3, v16, v1, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v1, v15, v0, vcc
; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -26103,8 +36586,8 @@ define <32 x bfloat> @v_select_v32bf16(i1 %cond, <32 x bfloat> %a, <32 x bfloat>
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GCN-NEXT: v_lshlrev_b32_e32 v12, 16, v13
; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GCN-NEXT: v_lshlrev_b32_e32 v14, 16, v15
-; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GCN-NEXT: v_lshlrev_b32_e32 v14, 16, v22
+; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v22
; GCN-NEXT: v_lshlrev_b32_e32 v16, 16, v23
; GCN-NEXT: v_and_b32_e32 v17, 0xffff0000, v23
; GCN-NEXT: v_lshlrev_b32_e32 v18, 16, v24
@@ -26126,173 +36609,241 @@ define <32 x bfloat> @v_select_v32bf16(i1 %cond, <32 x bfloat> %a, <32 x bfloat>
; GFX7-LABEL: v_select_v32bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_alignbit_b32 v1, v2, v1, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_alignbit_b32 v2, v2, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v8
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v7, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v10
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v12
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v14
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v16
-; GFX7-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:12
-; GFX7-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:16
-; GFX7-NEXT: v_alignbit_b32 v6, v6, v11, 16
-; GFX7-NEXT: v_alignbit_b32 v7, v7, v13, 16
-; GFX7-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:24
-; GFX7-NEXT: v_alignbit_b32 v8, v8, v15, 16
-; GFX7-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:40
-; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:72
-; GFX7-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:8
-; GFX7-NEXT: v_alignbit_b32 v5, v5, v9, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v9, 16, v18
-; GFX7-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; GFX7-NEXT: v_alignbit_b32 v9, v9, v17, 16
-; GFX7-NEXT: v_alignbit_b32 v25, v26, v25, 16
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v5
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 16
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v8
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v7
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v10
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GFX7-NEXT: v_alignbit_b32 v5, v5, v6, 16
+; GFX7-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:12
+; GFX7-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:16
+; GFX7-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:24
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:40
+; GFX7-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; GFX7-NEXT: v_alignbit_b32 v17, v18, v17, 16
+; GFX7-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:76
+; GFX7-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:8
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28
+; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GFX7-NEXT: v_lshrrev_b32_e32 v28, 16, v28
+; GFX7-NEXT: v_mul_f32_e32 v27, 1.0, v27
+; GFX7-NEXT: v_alignbit_b32 v13, v14, v13, 16
+; GFX7-NEXT: v_alignbit_b32 v27, v28, v27, 16
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
+; GFX7-NEXT: v_alignbit_b32 v11, v12, v11, 16
+; GFX7-NEXT: v_alignbit_b32 v23, v24, v23, 16
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GFX7-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GFX7-NEXT: v_alignbit_b32 v15, v16, v15, 16
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
+; GFX7-NEXT: v_lshrrev_b32_e32 v20, 16, v20
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; GFX7-NEXT: v_alignbit_b32 v19, v20, v19, 16
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GFX7-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GFX7-NEXT: v_alignbit_b32 v21, v22, v21, 16
+; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
+; GFX7-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_alignbit_b32 v25, v26, v25, 16
+; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30
; GFX7-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; GFX7-NEXT: v_mul_f32_e32 v29, 1.0, v29
; GFX7-NEXT: v_alignbit_b32 v29, v30, v29, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; GFX7-NEXT: v_alignbit_b32 v19, v20, v19, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; GFX7-NEXT: v_alignbit_b32 v23, v24, v23, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; GFX7-NEXT: v_alignbit_b32 v27, v28, v27, 16
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:32
-; GFX7-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:56
-; GFX7-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:108
-; GFX7-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:48
+; GFX7-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:32
+; GFX7-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:60
+; GFX7-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:116
+; GFX7-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52
+; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:100
+; GFX7-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:68
+; GFX7-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:84
; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:92
-; GFX7-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:64
+; GFX7-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:108
; GFX7-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:124
-; GFX7-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:84
-; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:100
-; GFX7-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:116
; GFX7-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:128
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32
; GFX7-NEXT: s_waitcnt vmcnt(14)
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GFX7-NEXT: s_waitcnt vmcnt(13)
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GFX7-NEXT: s_waitcnt vmcnt(12)
-; GFX7-NEXT: v_alignbit_b32 v10, v10, v11, 16
-; GFX7-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:20
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_alignbit_b32 v6, v6, v7, 16
+; GFX7-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:20
+; GFX7-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: s_waitcnt vmcnt(10)
-; GFX7-NEXT: v_lshrrev_b32_e32 v26, 16, v26
-; GFX7-NEXT: s_waitcnt vmcnt(8)
-; GFX7-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; GFX7-NEXT: s_waitcnt vmcnt(12)
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: s_waitcnt vmcnt(11)
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX7-NEXT: s_waitcnt vmcnt(9)
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX7-NEXT: s_waitcnt vmcnt(7)
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GFX7-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; GFX7-NEXT: s_waitcnt vmcnt(6)
-; GFX7-NEXT: v_lshrrev_b32_e32 v30, 16, v30
-; GFX7-NEXT: s_waitcnt vmcnt(5)
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; GFX7-NEXT: s_waitcnt vmcnt(4)
+; GFX7-NEXT: s_waitcnt vmcnt(5)
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
+; GFX7-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GFX7-NEXT: v_lshrrev_b32_e32 v24, 16, v24
-; GFX7-NEXT: s_waitcnt vmcnt(3)
+; GFX7-NEXT: s_waitcnt vmcnt(4)
+; GFX7-NEXT: v_mul_f32_e32 v26, 1.0, v26
+; GFX7-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; GFX7-NEXT: v_mul_f32_e32 v28, 1.0, v28
; GFX7-NEXT: v_lshrrev_b32_e32 v28, 16, v28
+; GFX7-NEXT: s_waitcnt vmcnt(3)
+; GFX7-NEXT: v_mul_f32_e32 v30, 1.0, v30
+; GFX7-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GFX7-NEXT: v_mul_f32_e32 v33, 1.0, v33
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; GFX7-NEXT: v_alignbit_b32 v11, v11, v12, 16
-; GFX7-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:28
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX7-NEXT: v_alignbit_b32 v7, v7, v8, 16
+; GFX7-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:28
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; GFX7-NEXT: v_alignbit_b32 v12, v12, v13, 16
-; GFX7-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:36
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX7-NEXT: v_alignbit_b32 v8, v8, v9, 16
+; GFX7-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:36
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; GFX7-NEXT: v_alignbit_b32 v13, v13, v14, 16
-; GFX7-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:44
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX7-NEXT: v_alignbit_b32 v9, v9, v10, 16
+; GFX7-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44
+; GFX7-NEXT: v_cndmask_b32_e32 v9, v9, v4, vcc
+; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v9
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; GFX7-NEXT: v_alignbit_b32 v14, v14, v15, 16
-; GFX7-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:52
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX7-NEXT: v_alignbit_b32 v10, v10, v31, 16
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:48
+; GFX7-NEXT: v_cndmask_b32_e32 v10, v10, v5, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v5, v8, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v3, v7, v2, vcc
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
+; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v10
+; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v10
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; GFX7-NEXT: v_alignbit_b32 v15, v15, v16, 16
-; GFX7-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:60
-; GFX7-NEXT: v_cndmask_b32_e32 v15, v15, v6, vcc
-; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v15
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GFX7-NEXT: v_alignbit_b32 v12, v12, v31, 16
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:56
+; GFX7-NEXT: v_cndmask_b32_e32 v11, v12, v11, vcc
+; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v11
+; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; GFX7-NEXT: v_alignbit_b32 v16, v16, v17, 16
-; GFX7-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:68
-; GFX7-NEXT: v_cndmask_b32_e32 v16, v16, v7, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v13, v4, vcc
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v16
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GFX7-NEXT: v_alignbit_b32 v14, v14, v31, 16
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:64
+; GFX7-NEXT: v_cndmask_b32_e32 v13, v14, v13, vcc
+; GFX7-NEXT: v_lshlrev_b32_e32 v12, 16, v13
+; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; GFX7-NEXT: v_alignbit_b32 v17, v17, v18, 16
-; GFX7-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:76
-; GFX7-NEXT: v_cndmask_b32_e32 v17, v17, v8, vcc
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GFX7-NEXT: v_alignbit_b32 v16, v16, v31, 16
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:72
+; GFX7-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc
+; GFX7-NEXT: v_lshlrev_b32_e32 v14, 16, v15
+; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GFX7-NEXT: v_alignbit_b32 v18, v18, v31, 16
; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:80
-; GFX7-NEXT: v_cndmask_b32_e32 v18, v18, v9, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v9, v14, v5, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v12, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v11, v2, vcc
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v9
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v15
-; GFX7-NEXT: v_lshlrev_b32_e32 v12, 16, v16
-; GFX7-NEXT: v_lshlrev_b32_e32 v14, 16, v17
-; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v17
-; GFX7-NEXT: v_lshlrev_b32_e32 v16, 16, v18
-; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v18
+; GFX7-NEXT: v_cndmask_b32_e32 v17, v18, v17, vcc
+; GFX7-NEXT: v_lshlrev_b32_e32 v16, 16, v17
+; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GFX7-NEXT: v_alignbit_b32 v20, v20, v31, 16
; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:88
; GFX7-NEXT: v_cndmask_b32_e32 v19, v20, v19, vcc
; GFX7-NEXT: v_lshlrev_b32_e32 v18, 16, v19
; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GFX7-NEXT: v_alignbit_b32 v22, v22, v31, 16
; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:96
; GFX7-NEXT: v_cndmask_b32_e32 v21, v22, v21, vcc
; GFX7-NEXT: v_lshlrev_b32_e32 v20, 16, v21
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GFX7-NEXT: v_alignbit_b32 v24, v24, v31, 16
; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:104
; GFX7-NEXT: v_cndmask_b32_e32 v23, v24, v23, vcc
; GFX7-NEXT: v_lshlrev_b32_e32 v22, 16, v23
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GFX7-NEXT: v_alignbit_b32 v26, v26, v31, 16
; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:112
; GFX7-NEXT: v_cndmask_b32_e32 v25, v26, v25, vcc
; GFX7-NEXT: v_lshlrev_b32_e32 v24, 16, v25
; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GFX7-NEXT: v_alignbit_b32 v28, v28, v31, 16
; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120
; GFX7-NEXT: v_cndmask_b32_e32 v27, v28, v27, vcc
; GFX7-NEXT: v_lshlrev_b32_e32 v26, 16, v27
; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GFX7-NEXT: v_alignbit_b32 v30, v30, v31, 16
; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
; GFX7-NEXT: v_cndmask_b32_e32 v29, v30, v29, vcc
; GFX7-NEXT: v_lshlrev_b32_e32 v28, 16, v29
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
; GFX7-NEXT: v_lshrrev_b32_e32 v31, 16, v31
; GFX7-NEXT: v_alignbit_b32 v31, v31, v32, 16
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:132
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_lshrrev_b32_e32 v32, 16, v32
; GFX7-NEXT: v_alignbit_b32 v32, v32, v33, 16
; GFX7-NEXT: v_cndmask_b32_e32 v31, v32, v31, vcc
@@ -26408,18 +36959,20 @@ define <32 x bfloat> @v_select_v32bf16(i1 %cond, <32 x bfloat> %a, <32 x bfloat>
define amdgpu_ps <2 x i32> @s_select_v3bf16(<3 x bfloat> inreg %a, <3 x bfloat> inreg %b, i32 %c) {
; GCN-LABEL: s_select_v3bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_lshr_b32 s1, s1, 16
-; GCN-NEXT: v_mov_b32_e32 v1, s0
-; GCN-NEXT: s_lshr_b32 s0, s4, 16
-; GCN-NEXT: v_mov_b32_e32 v2, s3
-; GCN-NEXT: s_lshr_b32 s2, s2, 16
-; GCN-NEXT: s_lshr_b32 s3, s5, 16
-; GCN-NEXT: v_alignbit_b32 v1, s1, v1, 16
-; GCN-NEXT: v_alignbit_b32 v2, s0, v2, 16
-; GCN-NEXT: v_mov_b32_e32 v3, s3
-; GCN-NEXT: v_mov_b32_e32 v4, s2
+; GCN-NEXT: v_mul_f32_e64 v1, 1.0, s1
+; GCN-NEXT: v_mul_f32_e64 v2, 1.0, s0
+; GCN-NEXT: v_mul_f32_e64 v3, 1.0, s4
+; GCN-NEXT: v_mul_f32_e64 v4, 1.0, s3
+; GCN-NEXT: v_mul_f32_e64 v5, 1.0, s2
+; GCN-NEXT: v_mul_f32_e64 v6, 1.0, s5
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GCN-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; GCN-NEXT: v_alignbit_b32 v2, v3, v4, 16
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v0, v6, v5, vcc
; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
; GCN-NEXT: v_readfirstlane_b32 s0, v1
; GCN-NEXT: v_readfirstlane_b32 s1, v0
@@ -26427,18 +36980,20 @@ define amdgpu_ps <2 x i32> @s_select_v3bf16(<3 x bfloat> inreg %a, <3 x bfloat>
;
; GFX7-LABEL: s_select_v3bf16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_lshr_b32 s1, s1, 16
-; GFX7-NEXT: v_mov_b32_e32 v1, s0
-; GFX7-NEXT: s_lshr_b32 s0, s4, 16
-; GFX7-NEXT: v_mov_b32_e32 v2, s3
-; GFX7-NEXT: v_alignbit_b32 v1, s1, v1, 16
-; GFX7-NEXT: v_alignbit_b32 v2, s0, v2, 16
-; GFX7-NEXT: s_lshr_b32 s0, s2, 16
-; GFX7-NEXT: s_lshr_b32 s1, s5, 16
-; GFX7-NEXT: v_mov_b32_e32 v3, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: v_mul_f32_e64 v1, 1.0, s1
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s0
+; GFX7-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s4
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e64 v3, 1.0, s3
+; GFX7-NEXT: v_alignbit_b32 v2, v2, v3, 16
+; GFX7-NEXT: v_mul_f32_e64 v3, 1.0, s2
+; GFX7-NEXT: v_mul_f32_e64 v4, 1.0, s5
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
; GFX7-NEXT: v_readfirstlane_b32 s0, v1
; GFX7-NEXT: v_readfirstlane_b32 s1, v0
@@ -26513,18 +37068,22 @@ define amdgpu_ps <2 x i32> @s_select_v3bf16(<3 x bfloat> inreg %a, <3 x bfloat>
define amdgpu_ps <2 x i32> @s_select_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat> inreg %b, i32 %c) {
; GCN-LABEL: s_select_v4bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_lshr_b32 s1, s1, 16
-; GCN-NEXT: v_mov_b32_e32 v1, s0
-; GCN-NEXT: s_lshr_b32 s0, s5, 16
-; GCN-NEXT: v_mov_b32_e32 v2, s4
-; GCN-NEXT: s_lshr_b32 s3, s3, 16
-; GCN-NEXT: v_mov_b32_e32 v3, s2
-; GCN-NEXT: s_lshr_b32 s2, s7, 16
-; GCN-NEXT: v_mov_b32_e32 v4, s6
-; GCN-NEXT: v_alignbit_b32 v1, s1, v1, 16
-; GCN-NEXT: v_alignbit_b32 v2, s0, v2, 16
-; GCN-NEXT: v_alignbit_b32 v3, s3, v3, 16
-; GCN-NEXT: v_alignbit_b32 v4, s2, v4, 16
+; GCN-NEXT: v_mul_f32_e64 v1, 1.0, s1
+; GCN-NEXT: v_mul_f32_e64 v2, 1.0, s0
+; GCN-NEXT: v_mul_f32_e64 v3, 1.0, s5
+; GCN-NEXT: v_mul_f32_e64 v4, 1.0, s4
+; GCN-NEXT: v_mul_f32_e64 v5, 1.0, s3
+; GCN-NEXT: v_mul_f32_e64 v6, 1.0, s2
+; GCN-NEXT: v_mul_f32_e64 v7, 1.0, s7
+; GCN-NEXT: v_mul_f32_e64 v8, 1.0, s6
+; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; GCN-NEXT: v_alignbit_b32 v2, v3, v4, 16
+; GCN-NEXT: v_alignbit_b32 v3, v5, v6, 16
+; GCN-NEXT: v_alignbit_b32 v4, v7, v8, 16
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
@@ -26534,18 +37093,22 @@ define amdgpu_ps <2 x i32> @s_select_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
;
; GFX7-LABEL: s_select_v4bf16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: v_mov_b32_e32 v1, s0
-; GFX7-NEXT: s_lshr_b32 s0, s5, 16
-; GFX7-NEXT: v_mov_b32_e32 v2, s4
-; GFX7-NEXT: v_alignbit_b32 v2, s0, v2, 16
-; GFX7-NEXT: s_lshr_b32 s0, s3, 16
-; GFX7-NEXT: v_mov_b32_e32 v3, s2
-; GFX7-NEXT: s_lshr_b32 s1, s1, 16
-; GFX7-NEXT: v_alignbit_b32 v3, s0, v3, 16
-; GFX7-NEXT: s_lshr_b32 s0, s7, 16
-; GFX7-NEXT: v_mov_b32_e32 v4, s6
-; GFX7-NEXT: v_alignbit_b32 v1, s1, v1, 16
-; GFX7-NEXT: v_alignbit_b32 v4, s0, v4, 16
+; GFX7-NEXT: v_mul_f32_e64 v1, 1.0, s1
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s0
+; GFX7-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s5
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_mul_f32_e64 v3, 1.0, s4
+; GFX7-NEXT: v_alignbit_b32 v2, v2, v3, 16
+; GFX7-NEXT: v_mul_f32_e64 v3, 1.0, s3
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e64 v4, 1.0, s2
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 16
+; GFX7-NEXT: v_mul_f32_e64 v4, 1.0, s7
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mul_f32_e64 v5, 1.0, s6
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
@@ -26616,22 +37179,22 @@ define amdgpu_ps <2 x i32> @s_select_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
define amdgpu_ps <2 x i32> @s_vselect_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat> inreg %b, <4 x i32> %c) {
; GCN-LABEL: s_vselect_v4bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: v_mov_b32_e32 v4, s7
-; GCN-NEXT: v_mov_b32_e32 v5, s3
+; GCN-NEXT: v_mul_f32_e64 v4, 1.0, s0
+; GCN-NEXT: v_mul_f32_e64 v5, 1.0, s4
+; GCN-NEXT: v_mul_f32_e64 v6, 1.0, s1
+; GCN-NEXT: v_mul_f32_e64 v7, 1.0, s5
+; GCN-NEXT: v_mul_f32_e64 v8, 1.0, s2
+; GCN-NEXT: v_mul_f32_e64 v9, 1.0, s6
+; GCN-NEXT: v_mul_f32_e64 v10, 1.0, s3
+; GCN-NEXT: v_mul_f32_e64 v11, 1.0, s7
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
-; GCN-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
-; GCN-NEXT: v_mov_b32_e32 v4, s6
-; GCN-NEXT: v_mov_b32_e32 v5, s2
+; GCN-NEXT: v_cndmask_b32_e32 v3, v11, v10, vcc
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GCN-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
-; GCN-NEXT: v_mov_b32_e32 v4, s5
-; GCN-NEXT: v_mov_b32_e32 v5, s1
+; GCN-NEXT: v_cndmask_b32_e32 v2, v9, v8, vcc
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GCN-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
-; GCN-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NEXT: v_mov_b32_e32 v5, s0
+; GCN-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
@@ -26644,27 +37207,27 @@ define amdgpu_ps <2 x i32> @s_vselect_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
;
; GFX7-LABEL: s_vselect_v4bf16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: v_mov_b32_e32 v4, s7
-; GFX7-NEXT: v_mov_b32_e32 v5, s3
+; GFX7-NEXT: v_mul_f32_e64 v10, 1.0, s3
+; GFX7-NEXT: v_mul_f32_e64 v11, 1.0, s7
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX7-NEXT: v_mov_b32_e32 v4, s6
-; GFX7-NEXT: v_mov_b32_e32 v5, s2
+; GFX7-NEXT: v_mul_f32_e64 v8, 1.0, s2
+; GFX7-NEXT: v_mul_f32_e64 v9, 1.0, s6
+; GFX7-NEXT: v_cndmask_b32_e32 v3, v11, v10, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX7-NEXT: v_mov_b32_e32 v3, s5
-; GFX7-NEXT: v_mov_b32_e32 v4, s1
+; GFX7-NEXT: v_mul_f32_e64 v6, 1.0, s1
+; GFX7-NEXT: v_mul_f32_e64 v7, 1.0, s5
+; GFX7-NEXT: v_cndmask_b32_e32 v2, v9, v8, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX7-NEXT: v_mov_b32_e32 v3, s4
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: v_mul_f32_e64 v4, 1.0, s0
+; GFX7-NEXT: v_mul_f32_e64 v5, 1.0, s4
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v7, v6, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
; GFX7-NEXT: v_readfirstlane_b32 s1, v2
@@ -26796,9 +37359,17 @@ define <4 x bfloat> @v_vselect_v4bf16(<4 x i1> %cond, <4 x bfloat> %a, <4 x bflo
; GCN-LABEL: v_vselect_v4bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: v_and_b32_e32 v0, 1, v0
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GCN-NEXT: v_and_b32_e32 v1, 1, v1
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GCN-NEXT: v_and_b32_e32 v2, 1, v2
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GCN-NEXT: v_and_b32_e32 v3, 1, v3
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
; GCN-NEXT: v_cndmask_b32_e32 v3, v11, v7, vcc
@@ -26819,13 +37390,21 @@ define <4 x bfloat> @v_vselect_v4bf16(<4 x i1> %cond, <4 x bfloat> %a, <4 x bflo
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v3, 1, v3
; GFX7-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
; GFX7-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GFX7-NEXT: v_cndmask_b32_e32 v3, v11, v7, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GFX7-NEXT: v_cndmask_b32_e32 v2, v10, v6, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GFX7-NEXT: v_cndmask_b32_e32 v1, v9, v5, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GFX7-NEXT: v_cndmask_b32_e32 v0, v8, v4, vcc
@@ -26938,13 +37517,29 @@ define <8 x bfloat> @v_vselect_v8bf16(<8 x i1> %cond, <8 x bfloat> %a, <8 x bflo
; GCN-LABEL: v_vselect_v8bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GCN-NEXT: v_and_b32_e32 v0, 1, v0
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GCN-NEXT: v_and_b32_e32 v1, 1, v1
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GCN-NEXT: v_and_b32_e32 v2, 1, v2
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GCN-NEXT: v_and_b32_e32 v3, 1, v3
+; GCN-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GCN-NEXT: v_and_b32_e32 v4, 1, v4
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GCN-NEXT: v_and_b32_e32 v5, 1, v5
+; GCN-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GCN-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GCN-NEXT: v_and_b32_e32 v6, 1, v6
+; GCN-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GCN-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GCN-NEXT: v_and_b32_e32 v7, 1, v7
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7
; GCN-NEXT: v_cndmask_b32_e32 v7, v23, v15, vcc
@@ -26977,25 +37572,41 @@ define <8 x bfloat> @v_vselect_v8bf16(<8 x i1> %cond, <8 x bfloat> %a, <8 x bflo
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v7, 1, v7
; GFX7-NEXT: v_and_b32_e32 v6, 1, v6
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
+; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7
; GFX7-NEXT: v_and_b32_e32 v5, 1, v5
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GFX7-NEXT: v_cndmask_b32_e32 v7, v23, v15, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
; GFX7-NEXT: v_and_b32_e32 v4, 1, v4
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GFX7-NEXT: v_cndmask_b32_e32 v6, v22, v14, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5
; GFX7-NEXT: v_and_b32_e32 v3, 1, v3
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: v_cndmask_b32_e32 v5, v21, v13, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4
; GFX7-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GFX7-NEXT: v_cndmask_b32_e32 v4, v20, v12, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
; GFX7-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GFX7-NEXT: v_cndmask_b32_e32 v3, v19, v11, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GFX7-NEXT: v_cndmask_b32_e32 v2, v18, v10, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GFX7-NEXT: v_cndmask_b32_e32 v1, v17, v9, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GFX7-NEXT: v_cndmask_b32_e32 v0, v16, v8, vcc
@@ -27214,72 +37825,104 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GCN-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v0
; GCN-NEXT: v_and_b32_e32 v0, 1, v9
; GCN-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v0
-; GCN-NEXT: v_and_b32_e32 v0, 1, v10
-; GCN-NEXT: v_and_b32_e32 v1, 1, v11
-; GCN-NEXT: v_and_b32_e32 v2, 1, v12
-; GCN-NEXT: v_and_b32_e32 v3, 1, v13
-; GCN-NEXT: v_and_b32_e32 v4, 1, v14
-; GCN-NEXT: v_and_b32_e32 v5, 1, v15
-; GCN-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v0
-; GCN-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4
-; GCN-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v1
-; GCN-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:12
-; GCN-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v2
-; GCN-NEXT: buffer_load_dword v2, off, s[0:3], s32
-; GCN-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v3
-; GCN-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:64
-; GCN-NEXT: v_cmp_eq_u32_e64 s[30:31], 1, v4
-; GCN-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:60
-; GCN-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v5
-; GCN-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:56
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_cndmask_b32_e64 v15, v3, v2, s[34:35]
-; GCN-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:52
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_cndmask_b32_e64 v14, v4, v30, s[30:31]
-; GCN-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:48
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_cndmask_b32_e64 v13, v5, v29, s[28:29]
-; GCN-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:44
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_cndmask_b32_e64 v12, v2, v28, s[26:27]
-; GCN-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:40
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_cndmask_b32_e64 v11, v3, v27, s[24:25]
-; GCN-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:36
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_cndmask_b32_e64 v10, v4, v26, s[22:23]
-; GCN-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_cndmask_b32_e64 v9, v2, v25, s[20:21]
-; GCN-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:28
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_cndmask_b32_e64 v8, v3, v24, s[18:19]
-; GCN-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:24
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v16
+; GCN-NEXT: v_and_b32_e32 v1, 1, v10
+; GCN-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v1
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v17
+; GCN-NEXT: v_and_b32_e32 v3, 1, v11
+; GCN-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v3
+; GCN-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:8
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v18
+; GCN-NEXT: v_and_b32_e32 v5, 1, v12
+; GCN-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v5
+; GCN-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:12
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v19
+; GCN-NEXT: v_and_b32_e32 v7, 1, v13
+; GCN-NEXT: v_and_b32_e32 v8, 1, v14
+; GCN-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v7
+; GCN-NEXT: buffer_load_dword v7, off, s[0:3], s32
+; GCN-NEXT: v_cmp_eq_u32_e64 s[30:31], 1, v8
+; GCN-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:64
+; GCN-NEXT: v_and_b32_e32 v9, 1, v15
+; GCN-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v9
+; GCN-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:60
; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_cndmask_b32_e64 v7, v4, v23, s[16:17]
-; GCN-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_cndmask_b32_e64 v6, v2, v22, s[14:15]
-; GCN-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:16
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_cndmask_b32_e64 v5, v3, v21, s[12:13]
-; GCN-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_cndmask_b32_e64 v4, v4, v20, s[10:11]
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_cndmask_b32_e64 v15, v8, v7, s[34:35]
+; GCN-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:56
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v30
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_cndmask_b32_e64 v14, v9, v8, s[30:31]
+; GCN-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:52
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v29
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_cndmask_b32_e64 v13, v7, v9, s[28:29]
+; GCN-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:48
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v28
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_cndmask_b32_e64 v12, v8, v9, s[26:27]
+; GCN-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:44
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v27
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_cndmask_b32_e64 v11, v7, v9, s[24:25]
+; GCN-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:40
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v26
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_cndmask_b32_e64 v10, v8, v9, s[22:23]
+; GCN-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:36
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v25
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_cndmask_b32_e64 v19, v2, v19, s[8:9]
-; GCN-NEXT: v_cndmask_b32_e64 v2, v1, v18, s[6:7]
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_cndmask_b32_e64 v9, v7, v9, s[20:21]
+; GCN-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:32
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v24
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_cndmask_b32_e64 v8, v8, v16, s[18:19]
+; GCN-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:28
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v23
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_cndmask_b32_e64 v7, v7, v17, s[16:17]
+; GCN-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:24
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v22
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GCN-NEXT: v_cndmask_b32_e64 v16, v16, v18, s[14:15]
+; GCN-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:16
+; GCN-NEXT: v_mul_f32_e32 v19, 1.0, v20
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v21
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; GCN-NEXT: v_cndmask_b32_e64 v17, v17, v20, s[12:13]
+; GCN-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:20
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cndmask_b32_e64 v1, v3, v17, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc
+; GCN-NEXT: v_mul_f32_e32 v20, 1.0, v20
+; GCN-NEXT: v_cndmask_b32_e64 v19, v20, v19, s[10:11]
+; GCN-NEXT: v_cndmask_b32_e64 v5, v18, v5, s[8:9]
+; GCN-NEXT: v_cndmask_b32_e64 v3, v6, v3, s[6:7]
+; GCN-NEXT: v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v19
-; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GCN-NEXT: v_and_b32_e32 v4, 0xffff0000, v19
+; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v17
+; GCN-NEXT: v_and_b32_e32 v6, 0xffff0000, v16
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GCN-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
@@ -27303,7 +37946,7 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX7-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; GFX7-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; GFX7-NEXT: s_mov_b64 exec, s[4:5]
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
@@ -27329,87 +37972,122 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX7-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v0
; GFX7-NEXT: v_and_b32_e32 v0, 1, v11
; GFX7-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 1, v12
-; GFX7-NEXT: v_writelane_b32 v32, s30, 0
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 1, v13
-; GFX7-NEXT: v_writelane_b32 v32, s31, 1
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 1, v14
-; GFX7-NEXT: v_writelane_b32 v32, s34, 2
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[30:31], 1, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 1, v15
-; GFX7-NEXT: v_writelane_b32 v32, s35, 3
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v0
-; GFX7-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4
-; GFX7-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8
-; GFX7-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12
-; GFX7-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16
-; GFX7-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20
-; GFX7-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24
-; GFX7-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:28
-; GFX7-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:32
-; GFX7-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:36
-; GFX7-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:40
-; GFX7-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44
-; GFX7-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:48
-; GFX7-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52
-; GFX7-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56
-; GFX7-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:60
-; GFX7-NEXT: buffer_load_dword v15, off, s[0:3], s32
-; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:64
-; GFX7-NEXT: s_waitcnt vmcnt(14)
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v16, vcc
-; GFX7-NEXT: v_cndmask_b32_e64 v1, v1, v17, s[4:5]
-; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v18, s[6:7]
-; GFX7-NEXT: s_waitcnt vmcnt(13)
-; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v19, s[8:9]
-; GFX7-NEXT: s_waitcnt vmcnt(12)
-; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v20, s[10:11]
-; GFX7-NEXT: s_waitcnt vmcnt(11)
-; GFX7-NEXT: v_cndmask_b32_e64 v5, v5, v21, s[12:13]
-; GFX7-NEXT: s_waitcnt vmcnt(10)
-; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, v22, s[14:15]
-; GFX7-NEXT: s_waitcnt vmcnt(9)
-; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v23, s[16:17]
-; GFX7-NEXT: s_waitcnt vmcnt(8)
-; GFX7-NEXT: v_cndmask_b32_e64 v8, v8, v24, s[18:19]
-; GFX7-NEXT: s_waitcnt vmcnt(7)
-; GFX7-NEXT: v_cndmask_b32_e64 v9, v9, v25, s[20:21]
-; GFX7-NEXT: s_waitcnt vmcnt(6)
-; GFX7-NEXT: v_cndmask_b32_e64 v10, v10, v26, s[22:23]
+; GFX7-NEXT: buffer_load_dword v0, off, s[0:3], s32
+; GFX7-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64
+; GFX7-NEXT: v_and_b32_e32 v2, 1, v12
+; GFX7-NEXT: v_writelane_b32 v31, s30, 0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v2
+; GFX7-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:60
+; GFX7-NEXT: v_and_b32_e32 v3, 1, v13
+; GFX7-NEXT: v_writelane_b32 v31, s31, 1
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v3
+; GFX7-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:56
+; GFX7-NEXT: v_and_b32_e32 v4, 1, v14
+; GFX7-NEXT: v_writelane_b32 v31, s34, 2
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[30:31], 1, v4
+; GFX7-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:52
+; GFX7-NEXT: v_and_b32_e32 v5, 1, v15
+; GFX7-NEXT: v_writelane_b32 v31, s35, 3
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v5
+; GFX7-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:48
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: s_waitcnt vmcnt(5)
-; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v27, s[24:25]
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: s_waitcnt vmcnt(4)
-; GFX7-NEXT: v_cndmask_b32_e64 v12, v12, v28, s[26:27]
-; GFX7-NEXT: s_waitcnt vmcnt(3)
-; GFX7-NEXT: v_cndmask_b32_e64 v13, v13, v29, s[28:29]
-; GFX7-NEXT: s_waitcnt vmcnt(2)
-; GFX7-NEXT: v_cndmask_b32_e64 v14, v14, v30, s[30:31]
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e64 v15, v31, v15, s[34:35]
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_cndmask_b32_e64 v15, v1, v0, s[34:35]
+; GFX7-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:44
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v30
+; GFX7-NEXT: s_waitcnt vmcnt(4)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_cndmask_b32_e64 v14, v2, v1, s[30:31]
+; GFX7-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:40
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v29
+; GFX7-NEXT: s_waitcnt vmcnt(4)
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_cndmask_b32_e64 v13, v3, v2, s[28:29]
+; GFX7-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:36
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v28
+; GFX7-NEXT: s_waitcnt vmcnt(4)
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_cndmask_b32_e64 v12, v4, v3, s[26:27]
+; GFX7-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:32
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v27
+; GFX7-NEXT: s_waitcnt vmcnt(4)
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_cndmask_b32_e64 v11, v5, v4, s[24:25]
+; GFX7-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:28
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v26
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX7-NEXT: v_readlane_b32 s35, v32, 3
-; GFX7-NEXT: v_readlane_b32 s34, v32, 2
-; GFX7-NEXT: v_readlane_b32 s31, v32, 1
-; GFX7-NEXT: v_readlane_b32 s30, v32, 0
+; GFX7-NEXT: v_readlane_b32 s35, v31, 3
+; GFX7-NEXT: v_readlane_b32 s34, v31, 2
+; GFX7-NEXT: v_readlane_b32 s31, v31, 1
+; GFX7-NEXT: v_readlane_b32 s30, v31, 0
+; GFX7-NEXT: s_waitcnt vmcnt(4)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_cndmask_b32_e64 v10, v0, v5, s[22:23]
+; GFX7-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:24
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v25
+; GFX7-NEXT: s_waitcnt vmcnt(4)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_cndmask_b32_e64 v9, v1, v5, s[20:21]
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v24
+; GFX7-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:4
+; GFX7-NEXT: s_waitcnt vmcnt(4)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_cndmask_b32_e64 v8, v2, v5, s[18:19]
+; GFX7-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:8
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v23
+; GFX7-NEXT: s_waitcnt vmcnt(4)
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_cndmask_b32_e64 v7, v3, v5, s[16:17]
+; GFX7-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:12
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v22
+; GFX7-NEXT: s_waitcnt vmcnt(4)
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_cndmask_b32_e64 v6, v4, v5, s[14:15]
+; GFX7-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:16
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v21
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; GFX7-NEXT: s_waitcnt vmcnt(4)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_cndmask_b32_e64 v5, v0, v5, s[12:13]
+; GFX7-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:20
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX7-NEXT: s_waitcnt vmcnt(4)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: s_waitcnt vmcnt(3)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_cndmask_b32_e64 v2, v2, v17, s[4:5]
+; GFX7-NEXT: s_waitcnt vmcnt(2)
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v18, s[6:7]
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_cndmask_b32_e64 v4, v4, v19, s[8:9]
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_cndmask_b32_e64 v20, v0, v20, s[10:11]
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v1, v16, vcc
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
+; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v20
; GFX7-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GFX7-NEXT: s_mov_b64 exec, s[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -27762,183 +38440,266 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
; GCN-LABEL: v_vselect_v32bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_and_b32_e32 v31, 1, v30
+; GCN-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GCN-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GCN-NEXT: v_and_b32_e32 v1, 1, v1
+; GCN-NEXT: v_and_b32_e32 v5, 1, v5
+; GCN-NEXT: v_and_b32_e32 v36, 1, v13
+; GCN-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:48
+; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:176
+; GCN-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:56
+; GCN-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:184
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:188
+; GCN-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64
+; GCN-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:192
; GCN-NEXT: v_and_b32_e32 v29, 1, v29
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:124
-; GCN-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:252
-; GCN-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:128
-; GCN-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:120
-; GCN-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:248
-; GCN-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:116
-; GCN-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:244
-; GCN-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:240
-; GCN-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:104
-; GCN-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:232
-; GCN-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:100
-; GCN-NEXT: v_and_b32_e32 v51, 1, v5
-; GCN-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:152
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v51
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v31
-; GCN-NEXT: s_waitcnt vmcnt(11)
-; GCN-NEXT: v_cndmask_b32_e64 v31, v34, v33, s[4:5]
+; GCN-NEXT: v_and_b32_e32 v30, 1, v30
+; GCN-NEXT: v_and_b32_e32 v48, 1, v28
+; GCN-NEXT: v_and_b32_e32 v50, 1, v27
+; GCN-NEXT: v_and_b32_e32 v52, 1, v26
+; GCN-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:92
+; GCN-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:220
+; GCN-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:96
+; GCN-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:224
+; GCN-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:104
+; GCN-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:232
+; GCN-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:108
+; GCN-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:236
+; GCN-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:112
+; GCN-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:240
+; GCN-NEXT: s_waitcnt expcnt(6)
+; GCN-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:120
+; GCN-NEXT: s_waitcnt expcnt(5)
+; GCN-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:248
+; GCN-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:124
+; GCN-NEXT: s_waitcnt expcnt(4)
+; GCN-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:252
+; GCN-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:128
+; GCN-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:256
+; GCN-NEXT: s_waitcnt vmcnt(14)
+; GCN-NEXT: v_mul_f32_e32 v41, 1.0, v37
+; GCN-NEXT: v_mul_f32_e32 v42, 1.0, v38
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v36
+; GCN-NEXT: s_waitcnt vmcnt(3)
+; GCN-NEXT: v_mul_f32_e32 v36, 1.0, v43
+; GCN-NEXT: s_waitcnt vmcnt(2)
+; GCN-NEXT: v_mul_f32_e32 v37, 1.0, v56
+; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v30
+; GCN-NEXT: v_cndmask_b32_e64 v30, v37, v36, s[4:5]
+; GCN-NEXT: buffer_load_dword v43, off, s[0:3], s32
+; GCN-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:116
+; GCN-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:244
+; GCN-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:100
+; GCN-NEXT: s_waitcnt expcnt(3)
+; GCN-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:228
+; GCN-NEXT: s_waitcnt expcnt(2)
+; GCN-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:84
+; GCN-NEXT: s_waitcnt expcnt(1)
+; GCN-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:212
+; GCN-NEXT: s_waitcnt expcnt(0)
+; GCN-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:68
+; GCN-NEXT: v_mul_f32_e32 v38, 1.0, v46
+; GCN-NEXT: v_mul_f32_e32 v46, 1.0, v47
+; GCN-NEXT: s_waitcnt vmcnt(6)
+; GCN-NEXT: v_mul_f32_e32 v36, 1.0, v36
+; GCN-NEXT: s_waitcnt vmcnt(5)
+; GCN-NEXT: v_mul_f32_e32 v37, 1.0, v37
+; GCN-NEXT: v_mul_f32_e32 v44, 1.0, v44
+; GCN-NEXT: v_mul_f32_e32 v45, 1.0, v45
+; GCN-NEXT: v_mul_f32_e32 v55, 1.0, v55
+; GCN-NEXT: v_mul_f32_e32 v40, 1.0, v40
; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v29
-; GCN-NEXT: s_waitcnt vmcnt(8)
-; GCN-NEXT: v_cndmask_b32_e64 v29, v36, v35, s[4:5]
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:220
-; GCN-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:96
-; GCN-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:108
-; GCN-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:236
-; GCN-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:112
-; GCN-NEXT: v_and_b32_e32 v18, 1, v18
-; GCN-NEXT: v_and_b32_e32 v22, 1, v22
-; GCN-NEXT: v_and_b32_e32 v26, 1, v26
-; GCN-NEXT: v_and_b32_e32 v28, 1, v28
-; GCN-NEXT: v_and_b32_e32 v27, 1, v27
+; GCN-NEXT: v_cndmask_b32_e64 v29, v46, v38, s[4:5]
+; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v48
+; GCN-NEXT: v_cndmask_b32_e64 v36, v37, v36, s[4:5]
+; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v50
+; GCN-NEXT: v_cndmask_b32_e64 v37, v45, v44, s[4:5]
+; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v52
+; GCN-NEXT: v_cndmask_b32_e64 v38, v40, v55, s[4:5]
+; GCN-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:8
+; GCN-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:136
+; GCN-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:12
+; GCN-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:140
+; GCN-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:16
+; GCN-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:144
+; GCN-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:24
+; GCN-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:152
+; GCN-NEXT: v_and_b32_e32 v9, 1, v9
+; GCN-NEXT: v_and_b32_e32 v17, 1, v17
+; GCN-NEXT: v_and_b32_e32 v21, 1, v21
; GCN-NEXT: v_and_b32_e32 v25, 1, v25
; GCN-NEXT: v_and_b32_e32 v24, 1, v24
; GCN-NEXT: v_and_b32_e32 v23, 1, v23
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v28
+; GCN-NEXT: v_and_b32_e32 v22, 1, v22
+; GCN-NEXT: v_mul_f32_e32 v53, 1.0, v53
+; GCN-NEXT: v_mul_f32_e32 v54, 1.0, v54
+; GCN-NEXT: s_waitcnt vmcnt(12)
+; GCN-NEXT: v_mul_f32_e32 v47, 1.0, v56
; GCN-NEXT: s_waitcnt vmcnt(11)
-; GCN-NEXT: v_cndmask_b32_e64 v28, v38, v37, s[4:5]
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v27
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cndmask_b32_e64 v27, v39, v51, s[4:5]
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v26
-; GCN-NEXT: v_cndmask_b32_e64 v26, v36, v35, s[4:5]
+; GCN-NEXT: v_mul_f32_e32 v56, 1.0, v57
+; GCN-NEXT: v_mul_f32_e32 v49, 1.0, v49
+; GCN-NEXT: v_mul_f32_e32 v51, 1.0, v51
+; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
+; GCN-NEXT: v_mul_f32_e32 v39, 1.0, v39
; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v25
-; GCN-NEXT: v_cndmask_b32_e64 v25, v49, v48, s[4:5]
-; GCN-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:228
-; GCN-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:224
-; GCN-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:88
-; GCN-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:216
-; GCN-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:84
-; GCN-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:212
-; GCN-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:208
-; GCN-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:72
+; GCN-NEXT: v_cndmask_b32_e64 v25, v54, v53, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v24
-; GCN-NEXT: s_waitcnt vmcnt(7)
-; GCN-NEXT: v_cndmask_b32_e64 v24, v35, v50, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v24, v56, v47, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v23
-; GCN-NEXT: s_waitcnt vmcnt(6)
-; GCN-NEXT: v_cndmask_b32_e64 v23, v36, v34, s[4:5]
-; GCN-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:76
-; GCN-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:204
-; GCN-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:80
-; GCN-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:92
-; GCN-NEXT: v_and_b32_e32 v21, 1, v21
+; GCN-NEXT: v_cndmask_b32_e64 v23, v51, v49, s[4:5]
+; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v22
+; GCN-NEXT: v_cndmask_b32_e64 v22, v39, v28, s[4:5]
+; GCN-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:72
+; GCN-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:200
+; GCN-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:76
+; GCN-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:204
+; GCN-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:80
+; GCN-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:208
+; GCN-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:88
+; GCN-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:216
; GCN-NEXT: v_and_b32_e32 v20, 1, v20
; GCN-NEXT: v_and_b32_e32 v19, 1, v19
-; GCN-NEXT: v_and_b32_e32 v17, 1, v17
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v22
+; GCN-NEXT: v_and_b32_e32 v18, 1, v18
+; GCN-NEXT: v_and_b32_e32 v16, 1, v16
+; GCN-NEXT: v_and_b32_e32 v15, 1, v15
+; GCN-NEXT: v_and_b32_e32 v14, 1, v14
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v47, 1.0, v47
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cndmask_b32_e64 v22, v33, v50, s[4:5]
+; GCN-NEXT: v_mul_f32_e32 v56, 1.0, v56
+; GCN-NEXT: v_mul_f32_e32 v57, 1.0, v58
+; GCN-NEXT: v_mul_f32_e32 v58, 1.0, v59
+; GCN-NEXT: v_mul_f32_e32 v53, 1.0, v53
+; GCN-NEXT: v_mul_f32_e32 v54, 1.0, v54
+; GCN-NEXT: v_mul_f32_e32 v49, 1.0, v49
+; GCN-NEXT: v_mul_f32_e32 v51, 1.0, v51
; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v21
-; GCN-NEXT: v_cndmask_b32_e64 v21, v38, v37, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v21, v56, v47, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v20
-; GCN-NEXT: v_cndmask_b32_e64 v20, v48, v39, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v20, v58, v57, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v19
-; GCN-NEXT: v_cndmask_b32_e64 v19, v49, v36, s[4:5]
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:200
-; GCN-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:68
-; GCN-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:196
-; GCN-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:192
-; GCN-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:56
-; GCN-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:184
-; GCN-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:52
-; GCN-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:180
+; GCN-NEXT: v_cndmask_b32_e64 v19, v54, v53, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v18
-; GCN-NEXT: v_cndmask_b32_e64 v18, v35, v34, s[4:5]
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v17
+; GCN-NEXT: v_cndmask_b32_e64 v18, v51, v49, s[4:5]
+; GCN-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:196
+; GCN-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:52
+; GCN-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:180
+; GCN-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:36
+; GCN-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:164
+; GCN-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:20
+; GCN-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:148
+; GCN-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:4
+; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
+; GCN-NEXT: v_mul_f32_e32 v39, 1.0, v39
+; GCN-NEXT: v_mul_f32_e32 v59, 1.0, v60
; GCN-NEXT: s_waitcnt vmcnt(7)
-; GCN-NEXT: v_cndmask_b32_e64 v17, v33, v51, s[4:5]
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:48
-; GCN-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:60
-; GCN-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:188
-; GCN-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:64
-; GCN-NEXT: v_and_b32_e32 v14, 1, v14
-; GCN-NEXT: v_and_b32_e32 v16, 1, v16
-; GCN-NEXT: v_and_b32_e32 v15, 1, v15
+; GCN-NEXT: v_mul_f32_e32 v49, 1.0, v49
+; GCN-NEXT: v_mul_f32_e32 v34, 1.0, v34
+; GCN-NEXT: v_mul_f32_e32 v35, 1.0, v35
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GCN-NEXT: v_mul_f32_e32 v33, 1.0, v33
+; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v17
+; GCN-NEXT: v_cndmask_b32_e64 v17, v39, v28, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v16
-; GCN-NEXT: s_waitcnt vmcnt(9)
-; GCN-NEXT: v_cndmask_b32_e64 v16, v37, v36, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v16, v49, v59, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v15
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cndmask_b32_e64 v15, v38, v51, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v15, v35, v34, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v14
-; GCN-NEXT: v_cndmask_b32_e64 v14, v35, v34, s[4:5]
-; GCN-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:176
+; GCN-NEXT: v_cndmask_b32_e64 v14, v33, v32, s[4:5]
+; GCN-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:28
+; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:156
+; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:32
+; GCN-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:160
; GCN-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:40
-; GCN-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:168
-; GCN-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36
-; GCN-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:164
-; GCN-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:160
-; GCN-NEXT: v_and_b32_e32 v10, 1, v10
-; GCN-NEXT: v_and_b32_e32 v13, 1, v13
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v13
-; GCN-NEXT: v_cndmask_b32_e64 v13, v48, v39, s[4:5]
-; GCN-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:44
-; GCN-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:172
+; GCN-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:168
+; GCN-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:44
+; GCN-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:172
; GCN-NEXT: v_and_b32_e32 v12, 1, v12
; GCN-NEXT: v_and_b32_e32 v11, 1, v11
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v12
-; GCN-NEXT: v_cndmask_b32_e64 v12, v50, v49, s[4:5]
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v11
-; GCN-NEXT: s_waitcnt vmcnt(7)
-; GCN-NEXT: v_cndmask_b32_e64 v11, v34, v33, s[4:5]
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v10
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cndmask_b32_e64 v10, v48, v39, s[4:5]
-; GCN-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12
-; GCN-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:140
-; GCN-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:16
-; GCN-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:28
-; GCN-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:156
-; GCN-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:32
-; GCN-NEXT: v_and_b32_e32 v2, 1, v2
-; GCN-NEXT: v_and_b32_e32 v6, 1, v6
-; GCN-NEXT: v_and_b32_e32 v9, 1, v9
+; GCN-NEXT: v_and_b32_e32 v10, 1, v10
; GCN-NEXT: v_and_b32_e32 v8, 1, v8
; GCN-NEXT: v_and_b32_e32 v7, 1, v7
+; GCN-NEXT: v_and_b32_e32 v6, 1, v6
; GCN-NEXT: v_and_b32_e32 v4, 1, v4
; GCN-NEXT: v_and_b32_e32 v3, 1, v3
-; GCN-NEXT: v_and_b32_e32 v1, 1, v1
+; GCN-NEXT: v_and_b32_e32 v2, 1, v2
; GCN-NEXT: v_and_b32_e32 v0, 1, v0
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v9
-; GCN-NEXT: v_cndmask_b32_e64 v9, v36, v35, s[4:5]
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v8
-; GCN-NEXT: v_cndmask_b32_e64 v8, v38, v37, s[4:5]
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v7
+; GCN-NEXT: v_cndmask_b32_e32 v41, v42, v41, vcc
+; GCN-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:132
+; GCN-NEXT: v_and_b32_e32 v43, 1, v43
+; GCN-NEXT: v_mul_f32_e32 v40, 1.0, v40
+; GCN-NEXT: v_mul_f32_e32 v44, 1.0, v44
+; GCN-NEXT: v_mul_f32_e32 v45, 1.0, v45
+; GCN-NEXT: v_mul_f32_e32 v46, 1.0, v46
+; GCN-NEXT: s_waitcnt vmcnt(4)
+; GCN-NEXT: v_mul_f32_e32 v35, 1.0, v35
+; GCN-NEXT: s_waitcnt vmcnt(3)
+; GCN-NEXT: v_mul_f32_e32 v39, 1.0, v39
+; GCN-NEXT: v_mul_f32_e32 v26, 1.0, v26
+; GCN-NEXT: v_mul_f32_e32 v27, 1.0, v27
+; GCN-NEXT: v_mul_f32_e32 v51, 1.0, v51
+; GCN-NEXT: v_mul_f32_e32 v53, 1.0, v53
+; GCN-NEXT: v_mul_f32_e32 v13, 1.0, v13
+; GCN-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GCN-NEXT: s_waitcnt vmcnt(2)
+; GCN-NEXT: v_mul_f32_e32 v49, 1.0, v49
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_mul_f32_e32 v59, 1.0, v59
+; GCN-NEXT: v_mul_f32_e32 v54, 1.0, v54
+; GCN-NEXT: v_mul_f32_e32 v47, 1.0, v47
+; GCN-NEXT: v_mul_f32_e32 v33, 1.0, v33
+; GCN-NEXT: v_mul_f32_e32 v34, 1.0, v34
+; GCN-NEXT: v_mul_f32_e32 v28, 1.0, v28
+; GCN-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GCN-NEXT: v_mul_f32_e32 v56, 1.0, v56
+; GCN-NEXT: v_mul_f32_e32 v57, 1.0, v57
+; GCN-NEXT: v_mul_f32_e32 v52, 1.0, v52
+; GCN-NEXT: v_mul_f32_e32 v55, 1.0, v55
+; GCN-NEXT: v_mul_f32_e32 v48, 1.0, v48
+; GCN-NEXT: v_mul_f32_e32 v50, 1.0, v50
+; GCN-NEXT: v_mul_f32_e32 v58, 1.0, v58
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cndmask_b32_e64 v7, v51, v50, s[4:5]
-; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v6
-; GCN-NEXT: v_cndmask_b32_e64 v6, v49, v48, s[4:5]
-; GCN-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:20
-; GCN-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:148
-; GCN-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:144
-; GCN-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:8
-; GCN-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:136
-; GCN-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:4
-; GCN-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:132
-; GCN-NEXT: buffer_load_dword v51, off, s[0:3], s32
-; GCN-NEXT: v_cndmask_b32_e32 v5, v32, v5, vcc
-; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:256
+; GCN-NEXT: v_mul_f32_e32 v42, 1.0, v42
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v12
+; GCN-NEXT: v_cndmask_b32_e32 v12, v53, v51, vcc
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v11
+; GCN-NEXT: v_cndmask_b32_e32 v11, v31, v13, vcc
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10
+; GCN-NEXT: v_cndmask_b32_e32 v10, v59, v49, vcc
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v9
+; GCN-NEXT: v_cndmask_b32_e32 v9, v39, v35, vcc
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v8
+; GCN-NEXT: v_cndmask_b32_e32 v8, v47, v54, vcc
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7
+; GCN-NEXT: v_cndmask_b32_e32 v7, v34, v33, vcc
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GCN-NEXT: v_cndmask_b32_e32 v6, v32, v28, vcc
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5
+; GCN-NEXT: v_cndmask_b32_e32 v5, v46, v45, vcc
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4
-; GCN-NEXT: s_waitcnt vmcnt(7)
-; GCN-NEXT: v_cndmask_b32_e32 v4, v36, v35, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v4, v57, v56, vcc
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
-; GCN-NEXT: s_waitcnt vmcnt(6)
-; GCN-NEXT: v_cndmask_b32_e32 v3, v37, v39, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v3, v55, v52, vcc
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
-; GCN-NEXT: v_cndmask_b32_e32 v2, v34, v33, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v2, v50, v48, vcc
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
-; GCN-NEXT: s_waitcnt vmcnt(4)
-; GCN-NEXT: v_cndmask_b32_e32 v1, v48, v38, vcc
+; GCN-NEXT: v_cndmask_b32_e32 v1, v44, v40, vcc
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_cndmask_b32_e32 v0, v50, v49, vcc
-; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v33, 1, v51
+; GCN-NEXT: v_cndmask_b32_e32 v0, v42, v58, vcc
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v43
+; GCN-NEXT: v_cndmask_b32_e32 v31, v27, v26, vcc
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -27952,7 +38713,7 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
; GCN-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GCN-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; GCN-NEXT: v_and_b32_e32 v13, 0xffff0000, v41
; GCN-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; GCN-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; GCN-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
@@ -27965,45 +38726,49 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
; GCN-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; GCN-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
; GCN-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; GCN-NEXT: v_and_b32_e32 v26, 0xffff0000, v38
+; GCN-NEXT: v_and_b32_e32 v27, 0xffff0000, v37
+; GCN-NEXT: v_and_b32_e32 v28, 0xffff0000, v36
; GCN-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v33
+; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; GCN-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; GCN-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GCN-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cndmask_b32_e32 v32, v32, v30, vcc
-; GCN-NEXT: v_and_b32_e32 v30, 0xffff0000, v31
-; GCN-NEXT: v_and_b32_e32 v31, 0xffff0000, v32
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_vselect_v32bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v24, 1, v24
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v24
+; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:228
+; GFX7-NEXT: v_and_b32_e32 v25, 1, v25
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v25
+; GFX7-NEXT: v_and_b32_e32 v30, 1, v30
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[12:13], 1, v30
+; GFX7-NEXT: v_and_b32_e32 v29, 1, v29
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[14:15], 1, v29
; GFX7-NEXT: v_and_b32_e32 v28, 1, v28
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v28
; GFX7-NEXT: v_and_b32_e32 v27, 1, v27
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[10:11], 1, v27
; GFX7-NEXT: v_and_b32_e32 v26, 1, v26
-; GFX7-NEXT: v_and_b32_e32 v25, 1, v25
-; GFX7-NEXT: v_and_b32_e32 v24, 1, v24
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v26
; GFX7-NEXT: v_and_b32_e32 v23, 1, v23
; GFX7-NEXT: v_and_b32_e32 v22, 1, v22
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v28
-; GFX7-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:124
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v27
-; GFX7-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:252
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[10:11], 1, v26
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[12:13], 1, v25
-; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:248
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[14:15], 1, v24
-; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:116
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v23
-; GFX7-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:244
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v22
-; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:240
-; GFX7-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:120
-; GFX7-NEXT: v_and_b32_e32 v30, 1, v30
-; GFX7-NEXT: v_and_b32_e32 v29, 1, v29
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v30
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v29
; GFX7-NEXT: v_and_b32_e32 v21, 1, v21
; GFX7-NEXT: v_and_b32_e32 v20, 1, v20
; GFX7-NEXT: v_and_b32_e32 v19, 1, v19
@@ -28026,183 +38791,286 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
; GFX7-NEXT: v_and_b32_e32 v2, 1, v2
; GFX7-NEXT: v_and_b32_e32 v1, 1, v1
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX7-NEXT: s_waitcnt vmcnt(5)
-; GFX7-NEXT: v_cndmask_b32_e64 v30, v27, v28, s[4:5]
-; GFX7-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:112
-; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:252
+; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:256
; GFX7-NEXT: s_waitcnt vmcnt(3)
-; GFX7-NEXT: v_cndmask_b32_e64 v28, v23, v24, s[6:7]
-; GFX7-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:236
-; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; GFX7-NEXT: v_and_b32_e32 v24, 1, v24
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v24
+; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:124
+; GFX7-NEXT: s_waitcnt vmcnt(3)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: s_waitcnt vmcnt(2)
-; GFX7-NEXT: v_cndmask_b32_e32 v29, v25, v26, vcc
-; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:108
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v31, 1.0, v31
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: v_cndmask_b32_e64 v30, v25, v24, s[12:13]
+; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:120
+; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:248
+; GFX7-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_cndmask_b32_e64 v29, v25, v24, s[14:15]
+; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:116
+; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:244
; GFX7-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; GFX7-NEXT: s_waitcnt vmcnt(2)
-; GFX7-NEXT: v_cndmask_b32_e64 v27, v22, v27, s[8:9]
-; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:104
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_cndmask_b32_e64 v28, v25, v24, s[16:17]
+; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:112
+; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:240
+; GFX7-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_cndmask_b32_e64 v27, v25, v24, s[10:11]
+; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:108
+; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:236
; GFX7-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_cndmask_b32_e64 v26, v23, v25, s[10:11]
-; GFX7-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:232
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_cndmask_b32_e64 v26, v25, v24, s[8:9]
+; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:104
+; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:232
; GFX7-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e64 v25, v23, v22, s[12:13]
-; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:100
-; GFX7-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:228
+; GFX7-NEXT: v_mul_f32_e32 v25, 1.0, v25
+; GFX7-NEXT: v_cndmask_b32_e64 v25, v25, v24, s[6:7]
+; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:128
; GFX7-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e64 v24, v23, v22, s[14:15]
-; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:224
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: v_cndmask_b32_e64 v31, v31, v24, s[4:5]
+; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:100
+; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v24, 1.0, v24
+; GFX7-NEXT: v_cndmask_b32_e32 v24, v32, v24, vcc
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v23
; GFX7-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:96
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:224
; GFX7-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v23, 1.0, v23
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e64 v23, v22, v23, s[16:17]
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GFX7-NEXT: v_cndmask_b32_e32 v23, v32, v23, vcc
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v22
; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:92
-; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:220
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:220
; GFX7-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v22, 1.0, v22
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e64 v22, v31, v22, s[18:19]
-; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v31, 1, v31
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v31
-; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:128
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:256
-; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e32 v31, v32, v31, vcc
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GFX7-NEXT: v_cndmask_b32_e32 v22, v32, v22, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v21
; GFX7-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:88
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:216
-; GFX7-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
+; GFX7-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v21, 1.0, v21
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v21, v32, v21, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v20
; GFX7-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:84
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:212
; GFX7-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v20, 1.0, v20
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v20, v32, v20, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v19
-; GFX7-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:208
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:80
+; GFX7-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:80
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:208
; GFX7-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v19, 1.0, v19
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e32 v19, v19, v32, vcc
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GFX7-NEXT: v_cndmask_b32_e32 v19, v32, v19, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v18
; GFX7-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:76
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:204
; GFX7-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v18, 1.0, v18
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v18, v32, v18, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v17
; GFX7-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:72
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:200
; GFX7-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v17, 1.0, v17
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v17, v32, v17, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v16
; GFX7-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:68
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:196
; GFX7-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v16, 1.0, v16
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v15
-; GFX7-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:192
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64
+; GFX7-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:192
; GFX7-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v15, 1.0, v15
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e32 v15, v15, v32, vcc
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GFX7-NEXT: v_cndmask_b32_e32 v15, v32, v15, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v14
; GFX7-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:60
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:188
; GFX7-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v14, 1.0, v14
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v14, v32, v14, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v13
; GFX7-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:184
; GFX7-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v13, 1.0, v13
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v13, v32, v13, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v12
; GFX7-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:180
; GFX7-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v12, 1.0, v12
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v12, v32, v12, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v11
-; GFX7-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:176
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48
+; GFX7-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:48
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:176
; GFX7-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e32 v11, v11, v32, vcc
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GFX7-NEXT: v_cndmask_b32_e32 v11, v32, v11, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10
; GFX7-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:172
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v10, v32, v10, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v9
; GFX7-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:40
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:168
; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v9, v32, v9, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v8
; GFX7-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:36
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:164
; GFX7-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v8, v32, v8, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7
-; GFX7-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160
-; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:32
+; GFX7-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:32
+; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v32, vcc
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v32, v7, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
; GFX7-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:28
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:156
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v6, v32, v6, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5
; GFX7-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:152
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v5, v32, v5, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4
; GFX7-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:148
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v4, v32, v4, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
; GFX7-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:144
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v3, v32, v3, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
; GFX7-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v2, v32, v2, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
; GFX7-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:136
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v1, v32, v1, vcc
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GFX7-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:132
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT: s_waitcnt vmcnt(1)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v32, 1.0, v32
; GFX7-NEXT: v_cndmask_b32_e32 v0, v32, v0, vcc
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -29183,6 +40051,9 @@ define bfloat @v_fma_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GCN-LABEL: v_fma_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -29193,6 +40064,9 @@ define bfloat @v_fma_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GFX7-LABEL: v_fma_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -29207,6 +40081,13 @@ define bfloat @v_fma_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -29217,6 +40098,13 @@ define bfloat @v_fma_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -29226,8 +40114,14 @@ define bfloat @v_fma_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX10-NEXT: v_bfe_u32 v0, v2, 16, 1
+; GFX10-NEXT: v_and_or_b32 v1, v2, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fma_bf16:
@@ -29236,9 +40130,17 @@ define bfloat @v_fma_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-NEXT: v_bfe_u32 v0, v2, 16, 1
+; GFX11-NEXT: v_and_or_b32 v1, v2, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
ret bfloat %op
@@ -29248,6 +40150,12 @@ define <2 x bfloat> @v_fma_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat>
; GCN-LABEL: v_fma_v2bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
@@ -29263,6 +40171,12 @@ define <2 x bfloat> @v_fma_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat>
; GFX7-LABEL: v_fma_v2bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
@@ -29281,11 +40195,25 @@ define <2 x bfloat> @v_fma_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat>
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX8-NEXT: v_fma_f32 v3, v5, v4, v3
+; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v3
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX8-NEXT: v_fma_f32 v0, v0, v1, v2
-; GFX8-NEXT: v_fma_f32 v3, v5, v4, v3
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -29296,11 +40224,24 @@ define <2 x bfloat> @v_fma_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat>
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX9-NEXT: v_fma_f32 v3, v5, v4, v3
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_fma_f32 v3, v5, v4, v3
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX9-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX9-NEXT: v_add3_u32 v4, v4, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -29314,9 +40255,20 @@ define <2 x bfloat> @v_fma_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat>
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_fmac_f32_e32 v3, v5, v4
; GFX10-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX10-NEXT: v_perm_b32 v0, v2, v3, 0x7060302
+; GFX10-NEXT: v_bfe_u32 v0, v3, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v3, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX10-NEXT: v_and_or_b32 v5, v2, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fma_v2bf16:
@@ -29328,9 +40280,23 @@ define <2 x bfloat> @v_fma_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat>
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_dual_fmac_f32 v2, v0, v1 :: v_dual_fmac_f32 v3, v5, v4
-; GFX11-NEXT: v_perm_b32 v0, v2, v3, 0x7060302
+; GFX11-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_bfe_u32 v0, v3, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v3, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_and_or_b32 v5, v2, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
ret <2 x bfloat> %op
@@ -29340,6 +40306,15 @@ define <3 x bfloat> @v_fma_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat>
; GCN-LABEL: v_fma_v3bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -29360,9 +40335,18 @@ define <3 x bfloat> @v_fma_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat>
; GFX7-LABEL: v_fma_v3bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
; GFX7-NEXT: v_fma_f32 v2, v2, v5, v8
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
@@ -29384,14 +40368,36 @@ define <3 x bfloat> @v_fma_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat>
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_fma_f32 v1, v1, v3, v5
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX8-NEXT: v_fma_f32 v3, v6, v5, v3
+; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v3
; GFX8-NEXT: v_fma_f32 v0, v0, v2, v4
-; GFX8-NEXT: v_fma_f32 v3, v6, v5, v3
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
@@ -29404,14 +40410,33 @@ define <3 x bfloat> @v_fma_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat>
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_fma_f32 v1, v1, v3, v5
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX9-NEXT: v_fma_f32 v3, v6, v5, v3
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_fma_f32 v3, v6, v5, v3
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v3
; GFX9-NEXT: v_fma_f32 v0, v0, v2, v4
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16
@@ -29420,20 +40445,36 @@ define <3 x bfloat> @v_fma_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat>
; GFX10-LABEL: v_fma_v3bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_fmac_f32_e32 v6, v8, v7
-; GFX10-NEXT: v_fmac_f32_e32 v4, v0, v2
; GFX10-NEXT: v_fmac_f32_e32 v5, v1, v3
-; GFX10-NEXT: v_perm_b32 v0, v4, v6, 0x7060302
-; GFX10-NEXT: v_alignbit_b32 v1, s4, v5, 16
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_fmac_f32_e32 v4, v0, v2
+; GFX10-NEXT: v_bfe_u32 v1, v6, 16, 1
+; GFX10-NEXT: v_and_or_b32 v3, v6, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX10-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX10-NEXT: v_add3_u32 v1, v1, v6, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v7, v4, s4, 0x400000
+; GFX10-NEXT: v_and_or_b32 v8, v5, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v0, v8, vcc_lo
+; GFX10-NEXT: v_perm_b32 v0, v2, v1, 0x7060302
+; GFX10-NEXT: v_alignbit_b32 v1, s4, v3, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = call <3 x bfloat> @llvm.fma.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c)
ret <3 x bfloat> %op
@@ -29443,6 +40484,18 @@ define <4 x bfloat> @v_fma_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat>
; GCN-LABEL: v_fma_v4bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
@@ -29468,13 +40521,25 @@ define <4 x bfloat> @v_fma_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat>
; GFX7-LABEL: v_fma_v4bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
; GFX7-NEXT: v_fma_f32 v3, v3, v7, v11
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v10
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GFX7-NEXT: v_fma_f32 v2, v2, v6, v7
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v9
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
@@ -29496,20 +40561,49 @@ define <4 x bfloat> @v_fma_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat>
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v5
; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX8-NEXT: v_fma_f32 v6, v8, v7, v6
+; GFX8-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v6
; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT: v_fma_f32 v6, v8, v7, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_and_b32_e32 v8, 0x80000000, v6
; GFX8-NEXT: v_fma_f32 v1, v1, v3, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v8
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX8-NEXT: v_fma_f32 v3, v7, v5, v3
+; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v7, 0x80000000, v3
; GFX8-NEXT: v_fma_f32 v0, v0, v2, v4
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_fma_f32 v3, v7, v5, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v6, 16
@@ -29521,19 +40615,44 @@ define <4 x bfloat> @v_fma_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat>
; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v5
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX9-NEXT: v_fma_f32 v6, v8, v7, v6
; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_fma_f32 v6, v8, v7, v6
+; GFX9-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v8, 0x80000000, v6
; GFX9-NEXT: v_fma_f32 v1, v1, v3, v5
+; GFX9-NEXT: v_add3_u32 v7, v7, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX9-NEXT: v_fma_f32 v3, v7, v5, v3
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_fma_f32 v3, v7, v5, v3
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v7, 0x80000000, v3
; GFX9-NEXT: v_fma_f32 v0, v0, v2, v4
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v6, s4
@@ -29547,43 +40666,89 @@ define <4 x bfloat> @v_fma_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat>
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v1
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v4
-; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX10-NEXT: v_fmac_f32_e32 v6, v8, v7
+; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_fmac_f32_e32 v6, v8, v7
-; GFX10-NEXT: v_fmac_f32_e32 v9, v11, v10
-; GFX10-NEXT: v_fmac_f32_e32 v4, v0, v2
+; GFX10-NEXT: v_bfe_u32 v10, v6, 16, 1
; GFX10-NEXT: v_fmac_f32_e32 v5, v1, v3
-; GFX10-NEXT: v_perm_b32 v0, v4, v9, 0x7060302
-; GFX10-NEXT: v_perm_b32 v1, v5, v6, 0x7060302
+; GFX10-NEXT: v_fmac_f32_e32 v7, v9, v8
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_fmac_f32_e32 v4, v0, v2
+; GFX10-NEXT: v_add3_u32 v0, v10, v6, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v1, v6, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX10-NEXT: v_bfe_u32 v3, v7, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX10-NEXT: v_and_or_b32 v9, v5, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc_lo
+; GFX10-NEXT: v_add3_u32 v0, v2, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v2, v3, v7, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v3, v7, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX10-NEXT: v_add3_u32 v6, v8, v4, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v4, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v0, v9, vcc_lo
+; GFX10-NEXT: v_perm_b32 v0, v3, v2, 0x7060302
+; GFX10-NEXT: v_perm_b32 v1, v4, v1, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fma_v4bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v0
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v0
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_fmac_f32 v9, v11, v10 :: v_dual_lshlrev_b32 v6, 16, v5
; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-NEXT: v_dual_fmac_f32 v4, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v5
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v0, v4, v9, 0x7060302
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_fmac_f32_e32 v5, v1, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_fmac_f32_e32 v6, v8, v7
-; GFX11-NEXT: v_perm_b32 v1, v5, v6, 0x7060302
+; GFX11-NEXT: v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v7, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-NEXT: v_and_or_b32 v1, v6, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-NEXT: v_fmac_f32_e32 v4, v0, v2
+; GFX11-NEXT: v_add3_u32 v0, v10, v6, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc_lo
+; GFX11-NEXT: v_fmac_f32_e32 v7, v9, v8
+; GFX11-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-NEXT: v_add3_u32 v0, v2, v5, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v9, v5, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_bfe_u32 v3, v7, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-NEXT: v_add3_u32 v6, v8, v4, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v8, v4, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v2, v3, v7, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v3, v7, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v0, v9, vcc_lo
+; GFX11-NEXT: v_perm_b32 v0, v3, v2, 0x7060302
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, v4, v1, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <4 x bfloat> @llvm.fma.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c)
ret <4 x bfloat> %op
@@ -29598,6 +40763,9 @@ define bfloat @v_fmuladd_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GCN-LABEL: v_fmuladd_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GCN-NEXT: v_mul_f32_e32 v0, v0, v1
@@ -29610,8 +40778,11 @@ define bfloat @v_fmuladd_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GFX7-LABEL: v_fmuladd_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
@@ -29625,9 +40796,23 @@ define bfloat @v_fmuladd_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -29637,9 +40822,22 @@ define bfloat @v_fmuladd_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v3, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v3
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -29648,10 +40846,21 @@ define bfloat @v_fmuladd_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v3, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc_lo
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v2, v0, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -29660,11 +40869,25 @@ define bfloat @v_fmuladd_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mul_f32 v0, v0, v1 :: v_dual_lshlrev_b32 v1, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v3, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v1, v3 :: v_dual_lshlrev_b32 v1, 16, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v2, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call bfloat @llvm.fmuladd.bf16(bfloat %a, bfloat %b, bfloat %c)
@@ -29675,6 +40898,12 @@ define <2 x bfloat> @v_fmuladd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfl
; GCN-LABEL: v_fmuladd_v2bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
@@ -29694,10 +40923,16 @@ define <2 x bfloat> @v_fmuladd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfl
; GFX7-LABEL: v_fmuladd_v2bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_mul_f32_e32 v1, v1, v3
; GFX7-NEXT: v_mul_f32_e32 v0, v0, v2
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
@@ -29715,16 +40950,45 @@ define <2 x bfloat> @v_fmuladd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfl
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v3
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v3
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v3
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX8-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, s4, v1
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX8-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -29734,16 +40998,41 @@ define <2 x bfloat> @v_fmuladd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfl
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v4, v4, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX9-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v3
+; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX9-NEXT: v_add3_u32 v4, v4, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
-; GFX9-NEXT: v_add_f32_e32 v3, v3, v4
; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -29755,14 +41044,35 @@ define <2 x bfloat> @v_fmuladd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfl
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_mul_f32_e32 v3, v4, v3
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-NEXT: v_bfe_u32 v1, v3, 16, 1
+; GFX10-NEXT: v_and_or_b32 v5, v3, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_and_or_b32 v6, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v1, v1, v3, 0x7fff
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX10-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo
+; GFX10-NEXT: v_add_f32_e32 v1, v1, v3
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_add_f32_e32 v1, v3, v1
+; GFX10-NEXT: v_and_or_b32 v4, v1, s4, 0x400000
; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v5, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -29773,16 +41083,43 @@ define <2 x bfloat> @v_fmuladd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfl
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mul_f32 v0, v0, v1 :: v_dual_lshlrev_b32 v1, 16, v2
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mul_f32_e32 v3, v4, v3
+; GFX11-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v3, 16, 1
+; GFX11-NEXT: v_and_or_b32 v5, v3, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v1, v1, v3, 0x7fff
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v6, v0, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-NEXT: v_dual_cndmask_b32 v0, v4, v6 :: v_dual_add_f32 v1, v1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v4, v1, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11-NEXT: v_mul_f32_e32 v3, v4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v1, v3, v1
+; GFX11-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v5, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <2 x bfloat> @llvm.fmuladd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
@@ -29793,6 +41130,15 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
; GCN-LABEL: v_fmuladd_v3bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GCN-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GCN-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GCN-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
@@ -29819,12 +41165,21 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
; GFX7-LABEL: v_fmuladd_v3bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
; GFX7-NEXT: v_mul_f32_e32 v2, v2, v5
; GFX7-NEXT: v_mul_f32_e32 v1, v1, v4
; GFX7-NEXT: v_mul_f32_e32 v0, v0, v3
@@ -29848,21 +41203,64 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX8-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX8-NEXT: v_mul_f32_e32 v3, v5, v3
+; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX8-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v6, 0x80000000, v3
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_mul_f32_e32 v3, v5, v3
+; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s4, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
@@ -29874,21 +41272,58 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX9-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX9-NEXT: v_mul_f32_e32 v3, v5, v3
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX9-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v3
+; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX9-NEXT: v_add_f32_e32 v3, v3, v5
; GFX9-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16
@@ -29904,19 +41339,50 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX10-NEXT: v_mul_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX10-NEXT: s_brev_b32 s4, 1
; GFX10-NEXT: v_mul_f32_e32 v3, v7, v6
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX10-NEXT: v_and_or_b32 v8, v1, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v9, v3, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v10, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v6, v9, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v7, v10, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v6, v1, s4, 0x400000
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v5
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v1, v1, v5
-; GFX10-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX10-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX10-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16
+; GFX10-NEXT: v_and_or_b32 v7, v2, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16
; GFX10-NEXT: s_setpc_b64 s[30:31]
%op = call <3 x bfloat> @llvm.fmuladd.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c)
ret <3 x bfloat> %op
@@ -29926,6 +41392,18 @@ define <4 x bfloat> @v_fmuladd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfl
; GCN-LABEL: v_fmuladd_v4bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GCN-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GCN-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GCN-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GCN-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GCN-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GCN-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GCN-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GCN-NEXT: v_mul_f32_e32 v11, 1.0, v11
+; GCN-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GCN-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GCN-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GCN-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GCN-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
@@ -29959,6 +41437,14 @@ define <4 x bfloat> @v_fmuladd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfl
; GFX7-LABEL: v_fmuladd_v4bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v7
; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
@@ -29967,6 +41453,10 @@ define <4 x bfloat> @v_fmuladd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfl
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_mul_f32_e32 v8, 1.0, v8
+; GFX7-NEXT: v_mul_f32_e32 v9, 1.0, v9
+; GFX7-NEXT: v_mul_f32_e32 v10, 1.0, v10
+; GFX7-NEXT: v_mul_f32_e32 v11, 1.0, v11
; GFX7-NEXT: v_mul_f32_e32 v3, v3, v7
; GFX7-NEXT: v_mul_f32_e32 v2, v2, v6
; GFX7-NEXT: v_mul_f32_e32 v1, v1, v5
@@ -29994,29 +41484,86 @@ define <4 x bfloat> @v_fmuladd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfl
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX8-NEXT: v_mul_f32_e32 v6, v7, v6
+; GFX8-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_and_b32_e32 v8, 0x80000000, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v8
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v7
+; GFX8-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX8-NEXT: s_movk_i32 s4, 0x7fff
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v6
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, s4, v7
+; GFX8-NEXT: v_and_b32_e32 v8, 0x80000000, v6
; GFX8-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v8
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT: v_and_b32_e32 v7, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v7, vcc
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
-; GFX8-NEXT: v_mul_f32_e32 v6, v7, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v5
; GFX8-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX8-NEXT: v_mul_f32_e32 v3, v5, v3
+; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v7, 0x80000000, v3
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX8-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT: v_and_b32_e32 v7, 0x80000000, v3
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_mul_f32_e32 v3, v5, v3
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s4, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_add_f32_e32 v6, v6, v7
+; GFX8-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v0
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
+; GFX8-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_add_f32_e32 v3, v3, v5
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16
; GFX8-NEXT: v_alignbit_b32 v1, v1, v6, 16
@@ -30027,28 +41574,77 @@ define <4 x bfloat> @v_fmuladd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfl
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX9-NEXT: v_mul_f32_e32 v6, v7, v6
+; GFX9-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: v_and_b32_e32 v8, 0x80000000, v6
+; GFX9-NEXT: v_add3_u32 v7, v7, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX9-NEXT: v_add_f32_e32 v6, v6, v7
+; GFX9-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v8, 0x80000000, v6
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX9-NEXT: v_add3_u32 v7, v7, v6, s4
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v8
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v7, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v7, vcc
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
-; GFX9-NEXT: v_mul_f32_e32 v6, v7, v6
-; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v5
; GFX9-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX9-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX9-NEXT: v_mul_f32_e32 v3, v5, v3
-; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v7, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX9-NEXT: v_add_f32_e32 v3, v3, v5
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v7, 0x80000000, v3
+; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, s4
+; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v5, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v5
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX9-NEXT: v_add_f32_e32 v6, v6, v7
-; GFX9-NEXT: v_add_f32_e32 v3, v3, v5
; GFX9-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v4, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
; GFX9-NEXT: s_mov_b32 s4, 0x7060302
; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v6, s4
@@ -30061,62 +41657,147 @@ define <4 x bfloat> @v_fmuladd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfl
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v0
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX10-NEXT: v_mul_f32_e32 v6, v7, v6
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: v_mul_f32_e32 v1, v1, v3
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX10-NEXT: v_mul_f32_e32 v3, v8, v7
+; GFX10-NEXT: s_brev_b32 s4, 1
+; GFX10-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX10-NEXT: v_and_or_b32 v3, v6, s4, 0x400000
+; GFX10-NEXT: v_mul_f32_e32 v7, v9, v7
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX10-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v10, v10, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_and_or_b32 v6, v1, s4, 0x400000
+; GFX10-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX10-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_and_or_b32 v10, v7, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v9, v9, v7, 0x7fff
+; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX10-NEXT: v_and_or_b32 v12, v0, s4, 0x400000
+; GFX10-NEXT: v_add3_u32 v11, v11, v0, 0x7fff
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v8
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT: v_add_f32_e32 v2, v6, v2
-; GFX10-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX10-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v11, v12, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
; GFX10-NEXT: v_add_f32_e32 v1, v1, v5
-; GFX10-NEXT: v_perm_b32 v0, v0, v3, 0x7060302
-; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302
+; GFX10-NEXT: v_and_or_b32 v5, v3, s4, 0x400000
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX10-NEXT: v_and_or_b32 v9, v1, s4, 0x400000
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX10-NEXT: v_add3_u32 v4, v7, v3, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
+; GFX10-NEXT: v_add3_u32 v4, v6, v1, 0x7fff
+; GFX10-NEXT: v_add3_u32 v5, v7, v2, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v6, v2, s4, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX10-NEXT: v_and_or_b32 v8, v0, s4, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc_lo
+; GFX10-NEXT: v_perm_b32 v1, v1, v3, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fmuladd_v4bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v0
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mul_f32 v6, v7, v6 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v5
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_dual_add_f32 v2, v6, v2 :: v_dual_mul_f32 v1, v1, v3
-; GFX11-NEXT: v_mul_f32_e32 v3, v8, v7
-; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-NEXT: s_brev_b32 s0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_mul_f32 v6, v7, v6 :: v_dual_and_b32 v5, 0xffff0000, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX11-NEXT: v_dual_mul_f32 v1, v1, v3 :: v_dual_and_b32 v2, 0xffff0000, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-NEXT: v_mul_f32_e32 v7, v9, v7
+; GFX11-NEXT: v_and_or_b32 v3, v6, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add3_u32 v10, v10, v6, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v6, v1, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX11-NEXT: v_dual_cndmask_b32 v3, v10, v3 :: v_dual_mul_f32 v0, v0, v2
+; GFX11-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_and_or_b32 v10, v7, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v9, v9, v7, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX11-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v12, v0, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add3_u32 v11, v11, v0, 0x7fff
+; GFX11-NEXT: v_dual_cndmask_b32 v1, v2, v6 :: v_dual_lshlrev_b32 v6, 16, v4
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11-NEXT: v_add_f32_e32 v1, v1, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v0, v0, v3, 0x7060302
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_cndmask_b32 v2, v9, v10 :: v_dual_and_b32 v1, 0xffff0000, v1
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_dual_add_f32 v1, v1, v5 :: v_dual_and_b32 v2, 0xffff0000, v2
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v11, v12, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_and_or_b32 v9, v1, s0, 0x400000
+; GFX11-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_f32_e32 v3, v3, v8
+; GFX11-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-NEXT: v_and_or_b32 v5, v3, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v4, v7, v3, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
+; GFX11-NEXT: v_add3_u32 v4, v6, v1, 0x7fff
+; GFX11-NEXT: v_add3_u32 v5, v7, v2, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v6, v2, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v8, v0, s0, 0x400000
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc_lo
+; GFX11-NEXT: v_perm_b32 v1, v1, v3, 0x7060302
; GFX11-NEXT: s_setpc_b64 s[30:31]
%op = call <4 x bfloat> @llvm.fmuladd.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c)
ret <4 x bfloat> %op
diff --git a/llvm/test/CodeGen/AMDGPU/fmed3-cast-combine.ll b/llvm/test/CodeGen/AMDGPU/fmed3-cast-combine.ll
index a69fb35f8f0cb0..cfe1e46bf2c5ee 100644
--- a/llvm/test/CodeGen/AMDGPU/fmed3-cast-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmed3-cast-combine.ll
@@ -787,6 +787,13 @@ define bfloat @fmed3_f32_fpext_f16_fptrunc_bf16(half %arg0, half %arg1, half %ar
; GFX8-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX8-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX8-NEXT: v_med3_f32 v0, v0, v1, v2
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX8-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -796,7 +803,14 @@ define bfloat @fmed3_f32_fpext_f16_fptrunc_bf16(half %arg0, half %arg1, half %ar
; GFX9-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX9-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX9-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX9-NEXT: s_movk_i32 s4, 0x7fff
; GFX9-NEXT: v_med3_f32 v0, v0, v1, v2
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v2, 0x80000000, v0
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s4
+; GFX9-NEXT: v_or_b32_e32 v2, 0x400000, v2
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
%arg0.ext = fpext half %arg0 to float
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
index 9a8ddb5bd38312..cd1ec85eb8d0f3 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
@@ -1233,8 +1233,12 @@ define double @fneg_f64_bitcast_build_vector_v4bf16_to_f64(bfloat %elt0, bfloat
; GFX7-LABEL: fneg_f64_bitcast_build_vector_v4bf16_to_f64:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7-NEXT: v_alignbit_b32 v2, v3, v2, 16
; GFX7-NEXT: v_alignbit_b32 v0, v1, v0, 16
; GFX7-NEXT: v_xor_b32_e32 v1, 0x80000000, v2
diff --git a/llvm/test/CodeGen/AMDGPU/function-args.ll b/llvm/test/CodeGen/AMDGPU/function-args.ll
index 38bfee961dd296..db89ad66ffab0d 100644
--- a/llvm/test/CodeGen/AMDGPU/function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-args.ll
@@ -2775,9 +2775,9 @@ define void @void_func_v32i32_i1_i8_i16_bf16(<32 x i32> %arg0, i1 %arg1, i8 %arg
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_store_dwordx4 v[20:23], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:20
; CI-NEXT: buffer_store_dwordx4 v[16:19], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:20
; CI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:16
; CI-NEXT: buffer_load_ubyte v17, off, s[0:3], s32 offset:4
; CI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:8
@@ -2791,8 +2791,9 @@ define void @void_func_v32i32_i1_i8_i16_bf16(<32 x i32> %arg0, i1 %arg1, i8 %arg
; CI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_cvt_f16_f32_e32 v16, v16
+; CI-NEXT: v_mul_f32_e32 v20, 1.0, v20
; CI-NEXT: v_and_b32_e32 v0, 1, v17
-; CI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
+; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v20
; CI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_store_byte v18, off, s[4:7], 0
@@ -2801,7 +2802,7 @@ define void @void_func_v32i32_i1_i8_i16_bf16(<32 x i32> %arg0, i1 %arg1, i8 %arg
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_store_short v16, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: buffer_store_short v20, off, s[4:7], 0
+; CI-NEXT: buffer_store_short v1, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: s_setpc_b64 s[30:31]
;
@@ -3108,22 +3109,14 @@ define void @void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16(<32 x i32> %arg0, <2 x i
; CI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:20
; CI-NEXT: buffer_store_dwordx4 v[12:15], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:28
; CI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:24
; CI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:12
; CI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:16
; CI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:8
+; CI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:28
; CI-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:4
-; CI-NEXT: v_lshrrev_b32_e32 v10, 16, v16
-; CI-NEXT: v_lshrrev_b32_e32 v11, 16, v17
-; CI-NEXT: v_lshrrev_b32_e32 v16, 16, v18
-; CI-NEXT: v_lshrrev_b32_e32 v17, 16, v19
-; CI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
-; CI-NEXT: v_lshrrev_b32_e32 v9, 16, v20
-; CI-NEXT: v_cvt_f16_f32_e32 v13, v13
-; CI-NEXT: v_cvt_f16_f32_e32 v14, v14
; CI-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
@@ -3132,21 +3125,35 @@ define void @void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16(<32 x i32> %arg0, <2 x i
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_store_short v8, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: v_cvt_f16_f32_e32 v14, v14
+; CI-NEXT: v_cvt_f16_f32_e32 v13, v13
+; CI-NEXT: v_mul_f32_e32 v9, 1.0, v20
+; CI-NEXT: v_mul_f32_e32 v10, 1.0, v16
+; CI-NEXT: v_mul_f32_e32 v11, 1.0, v17
+; CI-NEXT: v_mul_f32_e32 v16, 1.0, v18
+; CI-NEXT: v_mul_f32_e32 v17, 1.0, v19
+; CI-NEXT: v_mul_f32_e32 v12, 1.0, v12
+; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v9
+; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v10
+; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v11
+; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v16
+; CI-NEXT: v_lshrrev_b32_e32 v4, 16, v17
+; CI-NEXT: v_lshrrev_b32_e32 v5, 16, v12
; CI-NEXT: buffer_store_short v14, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_store_short v13, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: buffer_store_short v12, off, s[4:7], 0
+; CI-NEXT: buffer_store_short v5, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: buffer_store_short v17, off, s[4:7], 0
+; CI-NEXT: buffer_store_short v4, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: buffer_store_short v16, off, s[4:7], 0
+; CI-NEXT: buffer_store_short v3, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: buffer_store_short v11, off, s[4:7], 0
+; CI-NEXT: buffer_store_short v2, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: buffer_store_short v10, off, s[4:7], 0
+; CI-NEXT: buffer_store_short v1, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: buffer_store_short v9, off, s[4:7], 0
+; CI-NEXT: buffer_store_short v0, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: s_setpc_b64 s[30:31]
;
@@ -4633,6 +4640,7 @@ define void @void_func_bf16(bfloat %arg0) #0 {
; CI-LABEL: void_func_bf16:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
@@ -4664,7 +4672,9 @@ define void @void_func_v2bf16(<2 x bfloat> %arg0) #0 {
; CI-LABEL: void_func_v2bf16:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; CI-NEXT: v_alignbit_b32 v0, v1, v0, 16
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
@@ -4696,9 +4706,12 @@ define void @void_func_v3bf16(<3 x bfloat> %arg0) #0 {
; CI-LABEL: void_func_v3bf16:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; CI-NEXT: v_alignbit_b32 v0, v1, v0, 16
-; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; CI-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: buffer_store_short v1, off, s[4:7], 0
@@ -4733,8 +4746,12 @@ define void @void_func_v4bf16(<4 x bfloat> %arg0) #0 {
; CI-LABEL: void_func_v4bf16:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; CI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; CI-NEXT: v_mul_f32_e32 v2, 1.0, v2
; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; CI-NEXT: v_alignbit_b32 v2, v3, v2, 16
; CI-NEXT: v_alignbit_b32 v1, v1, v0, 16
; CI-NEXT: s_mov_b32 s7, 0xf000
@@ -4767,10 +4784,18 @@ define void @void_func_v8bf16(<8 x bfloat> %arg0) #0 {
; CI-LABEL: void_func_v8bf16:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; CI-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; CI-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; CI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; CI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; CI-NEXT: v_mul_f32_e32 v6, 1.0, v6
; CI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; CI-NEXT: v_mul_f32_e32 v4, 1.0, v4
; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; CI-NEXT: v_mul_f32_e32 v2, 1.0, v2
; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; CI-NEXT: v_alignbit_b32 v6, v7, v6, 16
; CI-NEXT: v_alignbit_b32 v5, v5, v4, 16
; CI-NEXT: v_alignbit_b32 v4, v3, v2, 16
@@ -4805,21 +4830,37 @@ define void @void_func_v16bf16(<16 x bfloat> %arg0) #0 {
; CI-LABEL: void_func_v16bf16:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-NEXT: v_mul_f32_e32 v5, 1.0, v5
+; CI-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; CI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; CI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; CI-NEXT: v_mul_f32_e32 v4, 1.0, v4
; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; CI-NEXT: v_mul_f32_e32 v2, 1.0, v2
; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; CI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; CI-NEXT: v_alignbit_b32 v5, v5, v4, 16
; CI-NEXT: v_alignbit_b32 v4, v3, v2, 16
; CI-NEXT: v_alignbit_b32 v3, v1, v0, 16
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v15
-; CI-NEXT: v_alignbit_b32 v14, v0, v14, 16
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v13
-; CI-NEXT: v_alignbit_b32 v13, v0, v12, 16
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; CI-NEXT: v_alignbit_b32 v12, v0, v10, 16
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v9
+; CI-NEXT: v_mul_f32_e32 v0, 1.0, v15
+; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; CI-NEXT: v_mul_f32_e32 v1, 1.0, v14
+; CI-NEXT: v_alignbit_b32 v14, v0, v1, 16
+; CI-NEXT: v_mul_f32_e32 v0, 1.0, v13
+; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; CI-NEXT: v_mul_f32_e32 v1, 1.0, v12
+; CI-NEXT: v_alignbit_b32 v13, v0, v1, 16
+; CI-NEXT: v_mul_f32_e32 v0, 1.0, v11
+; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; CI-NEXT: v_mul_f32_e32 v1, 1.0, v10
+; CI-NEXT: v_alignbit_b32 v12, v0, v1, 16
+; CI-NEXT: v_mul_f32_e32 v0, 1.0, v9
+; CI-NEXT: v_mul_f32_e32 v7, 1.0, v7
+; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; CI-NEXT: v_mul_f32_e32 v1, 1.0, v8
; CI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; CI-NEXT: v_alignbit_b32 v11, v0, v8, 16
+; CI-NEXT: v_mul_f32_e32 v6, 1.0, v6
+; CI-NEXT: v_alignbit_b32 v11, v0, v1, 16
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: v_alignbit_b32 v6, v7, v6, 16
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll b/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll
index 490167ee3635a3..b88aa1ce33fb37 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll
@@ -1504,26 +1504,33 @@ define amdgpu_kernel void @infer_as_before_atomic(ptr addrspace(4) %arg) #0 {
define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX900: ; %bb.0:
-; GFX900-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX900-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24
; GFX900-NEXT: s_mov_b64 s[0:1], 0
+; GFX900-NEXT: s_movk_i32 s4, 0x7fff
; GFX900-NEXT: v_mov_b32_e32 v0, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: s_and_b32 s2, s4, -4
-; GFX900-NEXT: s_mov_b32 s3, s5
-; GFX900-NEXT: s_load_dword s6, s[2:3], 0x0
-; GFX900-NEXT: s_and_b32 s4, s4, 3
-; GFX900-NEXT: s_lshl_b32 s4, s4, 3
-; GFX900-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX900-NEXT: s_not_b32 s5, s5
+; GFX900-NEXT: s_and_b32 s2, s6, -4
+; GFX900-NEXT: s_mov_b32 s3, s7
+; GFX900-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX900-NEXT: s_and_b32 s5, s6, 3
+; GFX900-NEXT: s_lshl_b32 s5, s5, 3
+; GFX900-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX900-NEXT: s_not_b32 s6, s6
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v1, s6
+; GFX900-NEXT: v_mov_b32_e32 v1, s7
; GFX900-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v2, v1
-; GFX900-NEXT: v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX900-NEXT: v_lshrrev_b32_sdwa v1, s5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX900-NEXT: v_add_f32_e32 v1, 4.0, v1
-; GFX900-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX900-NEXT: v_and_or_b32 v1, v2, s5, v1
+; GFX900-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX900-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX900-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX900-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX900-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX900-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX900-NEXT: v_and_or_b32 v1, v2, s6, v1
; GFX900-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
@@ -1533,32 +1540,39 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %p
; GFX900-NEXT: s_cbranch_execnz .LBB10_1
; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX900-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX900-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX900-NEXT: v_lshrrev_b32_e32 v0, s5, v1
; GFX900-NEXT: global_store_short v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX908: ; %bb.0:
-; GFX908-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX908-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24
; GFX908-NEXT: s_mov_b64 s[0:1], 0
+; GFX908-NEXT: s_movk_i32 s4, 0x7fff
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: s_and_b32 s2, s4, -4
-; GFX908-NEXT: s_mov_b32 s3, s5
-; GFX908-NEXT: s_load_dword s6, s[2:3], 0x0
-; GFX908-NEXT: s_and_b32 s4, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, s4, 3
-; GFX908-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX908-NEXT: s_not_b32 s5, s5
+; GFX908-NEXT: s_and_b32 s2, s6, -4
+; GFX908-NEXT: s_mov_b32 s3, s7
+; GFX908-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX908-NEXT: s_and_b32 s5, s6, 3
+; GFX908-NEXT: s_lshl_b32 s5, s5, 3
+; GFX908-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX908-NEXT: s_not_b32 s6, s6
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
+; GFX908-NEXT: v_mov_b32_e32 v1, s7
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v2, v1
-; GFX908-NEXT: v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX908-NEXT: v_lshrrev_b32_sdwa v1, s5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX908-NEXT: v_add_f32_e32 v1, 4.0, v1
-; GFX908-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX908-NEXT: v_and_or_b32 v1, v2, s5, v1
+; GFX908-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX908-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX908-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX908-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX908-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX908-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX908-NEXT: v_and_or_b32 v1, v2, s6, v1
; GFX908-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
@@ -1568,32 +1582,39 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %p
; GFX908-NEXT: s_cbranch_execnz .LBB10_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX908-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, s5, v1
; GFX908-NEXT: global_store_short v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
+; GFX90A-NEXT: s_movk_i32 s4, 0x7fff
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_and_b32 s2, s4, -4
-; GFX90A-NEXT: s_mov_b32 s3, s5
-; GFX90A-NEXT: s_load_dword s6, s[2:3], 0x0
-; GFX90A-NEXT: s_and_b32 s4, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX90A-NEXT: s_not_b32 s5, s5
+; GFX90A-NEXT: s_and_b32 s2, s6, -4
+; GFX90A-NEXT: s_mov_b32 s3, s7
+; GFX90A-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX90A-NEXT: s_and_b32 s5, s6, 3
+; GFX90A-NEXT: s_lshl_b32 s5, s5, 3
+; GFX90A-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX90A-NEXT: s_not_b32 s6, s6
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-NEXT: v_mov_b32_e32 v1, s7
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v3, v1
-; GFX90A-NEXT: v_lshrrev_b32_sdwa v1, s4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX90A-NEXT: v_lshrrev_b32_sdwa v1, s5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX90A-NEXT: v_add_f32_e32 v1, 4.0, v1
-; GFX90A-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_and_or_b32 v2, v3, s5, v1
+; GFX90A-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX90A-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX90A-NEXT: v_add3_u32 v2, v2, v1, s4
+; GFX90A-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX90A-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
+; GFX90A-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX90A-NEXT: v_and_or_b32 v2, v3, s6, v1
; GFX90A-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
@@ -1603,7 +1624,7 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %p
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s5, v1
; GFX90A-NEXT: global_store_short v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
@@ -1611,6 +1632,7 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %p
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_brev_b32 s5, 1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_and_b32 s0, s2, -4
; GFX10-NEXT: s_mov_b32 s1, s3
@@ -1627,6 +1649,11 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %p
; GFX10-NEXT: v_mov_b32_e32 v2, v1
; GFX10-NEXT: v_lshrrev_b32_sdwa v1, s2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_add_f32_e32 v1, 4.0, v1
+; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v1, s5, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT: v_and_or_b32 v1, v2, s4, v1
; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
@@ -1646,6 +1673,7 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %p
; GFX11-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x24
+; GFX11-NEXT: s_brev_b32 s5, 1
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s0, s2, -4
@@ -1658,12 +1686,18 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %p
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s3
; GFX11-NEXT: s_mov_b32 s3, 0
+; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_mov_b32_e32 v2, v1
; GFX11-NEXT: v_lshrrev_b32_e32 v1, s2, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_add_f32_e32 v1, 4.0, v1
+; GFX11-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v1, s5, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v1, s2, v1
; GFX11-NEXT: v_and_or_b32 v1, v2, s4, v1
@@ -1690,26 +1724,33 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %p
define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX900: ; %bb.0:
-; GFX900-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX900-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24
; GFX900-NEXT: s_mov_b64 s[0:1], 0
+; GFX900-NEXT: s_movk_i32 s4, 0x7fff
; GFX900-NEXT: v_mov_b32_e32 v0, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: s_and_b32 s2, s4, -4
-; GFX900-NEXT: s_mov_b32 s3, s5
-; GFX900-NEXT: s_load_dword s6, s[2:3], 0x0
-; GFX900-NEXT: s_and_b32 s4, s4, 3
-; GFX900-NEXT: s_lshl_b32 s4, s4, 3
-; GFX900-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX900-NEXT: s_not_b32 s5, s5
+; GFX900-NEXT: s_and_b32 s2, s6, -4
+; GFX900-NEXT: s_mov_b32 s3, s7
+; GFX900-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX900-NEXT: s_and_b32 s5, s6, 3
+; GFX900-NEXT: s_lshl_b32 s5, s5, 3
+; GFX900-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX900-NEXT: s_not_b32 s6, s6
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v1, s6
+; GFX900-NEXT: v_mov_b32_e32 v1, s7
; GFX900-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v2, v1
-; GFX900-NEXT: v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX900-NEXT: v_lshrrev_b32_sdwa v1, s5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX900-NEXT: v_add_f32_e32 v1, 4.0, v1
-; GFX900-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX900-NEXT: v_and_or_b32 v1, v2, s5, v1
+; GFX900-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX900-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX900-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX900-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX900-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX900-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX900-NEXT: v_and_or_b32 v1, v2, s6, v1
; GFX900-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
@@ -1719,32 +1760,39 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) %
; GFX900-NEXT: s_cbranch_execnz .LBB11_1
; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX900-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX900-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX900-NEXT: v_lshrrev_b32_e32 v0, s5, v1
; GFX900-NEXT: global_store_short v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX908: ; %bb.0:
-; GFX908-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX908-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24
; GFX908-NEXT: s_mov_b64 s[0:1], 0
+; GFX908-NEXT: s_movk_i32 s4, 0x7fff
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: s_and_b32 s2, s4, -4
-; GFX908-NEXT: s_mov_b32 s3, s5
-; GFX908-NEXT: s_load_dword s6, s[2:3], 0x0
-; GFX908-NEXT: s_and_b32 s4, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, s4, 3
-; GFX908-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX908-NEXT: s_not_b32 s5, s5
+; GFX908-NEXT: s_and_b32 s2, s6, -4
+; GFX908-NEXT: s_mov_b32 s3, s7
+; GFX908-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX908-NEXT: s_and_b32 s5, s6, 3
+; GFX908-NEXT: s_lshl_b32 s5, s5, 3
+; GFX908-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX908-NEXT: s_not_b32 s6, s6
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
+; GFX908-NEXT: v_mov_b32_e32 v1, s7
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v2, v1
-; GFX908-NEXT: v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX908-NEXT: v_lshrrev_b32_sdwa v1, s5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX908-NEXT: v_add_f32_e32 v1, 4.0, v1
-; GFX908-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX908-NEXT: v_and_or_b32 v1, v2, s5, v1
+; GFX908-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX908-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX908-NEXT: v_add3_u32 v3, v3, v1, s4
+; GFX908-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX908-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX908-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX908-NEXT: v_and_or_b32 v1, v2, s6, v1
; GFX908-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
@@ -1754,32 +1802,39 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) %
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX908-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, s5, v1
; GFX908-NEXT: global_store_short v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
+; GFX90A-NEXT: s_movk_i32 s4, 0x7fff
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_and_b32 s2, s4, -4
-; GFX90A-NEXT: s_mov_b32 s3, s5
-; GFX90A-NEXT: s_load_dword s6, s[2:3], 0x0
-; GFX90A-NEXT: s_and_b32 s4, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX90A-NEXT: s_not_b32 s5, s5
+; GFX90A-NEXT: s_and_b32 s2, s6, -4
+; GFX90A-NEXT: s_mov_b32 s3, s7
+; GFX90A-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX90A-NEXT: s_and_b32 s5, s6, 3
+; GFX90A-NEXT: s_lshl_b32 s5, s5, 3
+; GFX90A-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX90A-NEXT: s_not_b32 s6, s6
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-NEXT: v_mov_b32_e32 v1, s7
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v3, v1
-; GFX90A-NEXT: v_lshrrev_b32_sdwa v1, s4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX90A-NEXT: v_lshrrev_b32_sdwa v1, s5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX90A-NEXT: v_add_f32_e32 v1, 4.0, v1
-; GFX90A-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_and_or_b32 v2, v3, s5, v1
+; GFX90A-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX90A-NEXT: v_and_b32_e32 v4, 0x80000000, v1
+; GFX90A-NEXT: v_add3_u32 v2, v2, v1, s4
+; GFX90A-NEXT: v_or_b32_e32 v4, 0x400000, v4
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX90A-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
+; GFX90A-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX90A-NEXT: v_and_or_b32 v2, v3, s6, v1
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1791,7 +1846,7 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) %
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s5, v1
; GFX90A-NEXT: global_store_short v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
@@ -1799,6 +1854,7 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) %
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_brev_b32 s5, 1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_and_b32 s0, s2, -4
; GFX10-NEXT: s_mov_b32 s1, s3
@@ -1815,6 +1871,11 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) %
; GFX10-NEXT: v_mov_b32_e32 v2, v1
; GFX10-NEXT: v_lshrrev_b32_sdwa v1, s2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_add_f32_e32 v1, 4.0, v1
+; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX10-NEXT: v_and_or_b32 v4, v1, s5, 0x400000
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT: v_and_or_b32 v1, v2, s4, v1
; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
@@ -1834,6 +1895,7 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) %
; GFX11-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x24
+; GFX11-NEXT: s_brev_b32 s5, 1
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s0, s2, -4
@@ -1846,12 +1908,18 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) %
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s3
; GFX11-NEXT: s_mov_b32 s3, 0
+; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_mov_b32_e32 v2, v1
; GFX11-NEXT: v_lshrrev_b32_e32 v1, s2, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_add_f32_e32 v1, 4.0, v1
+; GFX11-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-NEXT: v_and_or_b32 v4, v1, s5, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v1, s2, v1
; GFX11-NEXT: v_and_or_b32 v1, v2, s4, v1
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
index 78db126fb2dc49..ad788b8d55014c 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
@@ -3,10 +3,10 @@
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-wavefrontsize32,+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX11 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize32,-wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX10 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -mattr=-wavefrontsize32,+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=GISEL-GFX10 %s
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX11 %s
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-wavefrontsize32,+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX11 %s
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize32,-wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX10 %s
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1030 -mattr=-wavefrontsize32,+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX10 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX11-WF32 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-wavefrontsize32,+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX11-WF64 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize32,-wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX10-WF32 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1030 -mattr=-wavefrontsize32,+wavefrontsize64 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck -check-prefix=DAGISEL-GFX10-WF64 %s
; We only care about which physical registers the parameters are copied from;
; the function bodies are just some arbitrary uses.
@@ -64,59 +64,113 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc(<4 x i32> inre
; GISEL-GFX10-NEXT: FLAT_STORE_DWORDX4 [[COPY12]], [[REG_SEQUENCE]], 0, 0, implicit $exec, implicit $flat_scr :: (store (<4 x s32>) into `ptr poison`)
; GISEL-GFX10-NEXT: S_ENDPGM 0
;
- ; DAGISEL-GFX11-LABEL: name: amdgpu_cs_chain_preserve_cc
- ; DAGISEL-GFX11: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr8, $vgpr9, $vgpr10, $vgpr11
- ; DAGISEL-GFX11-NEXT: {{ $}}
- ; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr11
- ; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr9
- ; DAGISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
- ; DAGISEL-GFX11-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
- ; DAGISEL-GFX11-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
- ; DAGISEL-GFX11-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY]], 0, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY1]], 0, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[V_ADD_U32_e64_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY6]], [[COPY2]], 0, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[V_ADD_U32_e64_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY7]], [[COPY3]], 0, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_ADD_U32_e64_3]], %subreg.sub0, [[V_ADD_U32_e64_2]], %subreg.sub1, [[V_ADD_U32_e64_1]], %subreg.sub2, [[V_ADD_U32_e64_]], %subreg.sub3
- ; DAGISEL-GFX11-NEXT: [[DEF4:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[DEF4]]
- ; DAGISEL-GFX11-NEXT: [[COPY9:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
- ; DAGISEL-GFX11-NEXT: FLAT_STORE_DWORDX4 killed [[COPY8]], killed [[COPY9]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison`)
- ; DAGISEL-GFX11-NEXT: S_ENDPGM 0
- ;
- ; DAGISEL-GFX10-LABEL: name: amdgpu_cs_chain_preserve_cc
- ; DAGISEL-GFX10: bb.0 (%ir-block.0):
- ; DAGISEL-GFX10-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr8, $vgpr9, $vgpr10, $vgpr11
- ; DAGISEL-GFX10-NEXT: {{ $}}
- ; DAGISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr11
- ; DAGISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; DAGISEL-GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr9
- ; DAGISEL-GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX10-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
- ; DAGISEL-GFX10-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
- ; DAGISEL-GFX10-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
- ; DAGISEL-GFX10-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX10-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY]], 0, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY1]], 0, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[V_ADD_U32_e64_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY6]], [[COPY2]], 0, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[V_ADD_U32_e64_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY7]], [[COPY3]], 0, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_ADD_U32_e64_3]], %subreg.sub0, [[V_ADD_U32_e64_2]], %subreg.sub1, [[V_ADD_U32_e64_1]], %subreg.sub2, [[V_ADD_U32_e64_]], %subreg.sub3
- ; DAGISEL-GFX10-NEXT: [[DEF4:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[DEF4]]
- ; DAGISEL-GFX10-NEXT: [[COPY9:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
- ; DAGISEL-GFX10-NEXT: FLAT_STORE_DWORDX4 killed [[COPY8]], killed [[COPY9]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison`)
- ; DAGISEL-GFX10-NEXT: S_ENDPGM 0
+ ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc
+ ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF32-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr8, $vgpr9, $vgpr10, $vgpr11
+ ; DAGISEL-GFX11-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY]], 0, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY1]], 0, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD_U32_e64_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY6]], [[COPY2]], 0, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD_U32_e64_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY7]], [[COPY3]], 0, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_ADD_U32_e64_3]], %subreg.sub0, [[V_ADD_U32_e64_2]], %subreg.sub1, [[V_ADD_U32_e64_1]], %subreg.sub2, [[V_ADD_U32_e64_]], %subreg.sub3
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF4:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[DEF4]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY9:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX11-WF32-NEXT: FLAT_STORE_DWORDX4 killed [[COPY8]], killed [[COPY9]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison`)
+ ; DAGISEL-GFX11-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc
+ ; DAGISEL-GFX11-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF64-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr8, $vgpr9, $vgpr10, $vgpr11
+ ; DAGISEL-GFX11-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY]], 0, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY1]], 0, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD_U32_e64_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY6]], [[COPY2]], 0, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD_U32_e64_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY7]], [[COPY3]], 0, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_ADD_U32_e64_3]], %subreg.sub0, [[V_ADD_U32_e64_2]], %subreg.sub1, [[V_ADD_U32_e64_1]], %subreg.sub2, [[V_ADD_U32_e64_]], %subreg.sub3
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF4:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[DEF4]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY9:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX11-WF64-NEXT: FLAT_STORE_DWORDX4 killed [[COPY8]], killed [[COPY9]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison`)
+ ; DAGISEL-GFX11-WF64-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc
+ ; DAGISEL-GFX10-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF32-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr8, $vgpr9, $vgpr10, $vgpr11
+ ; DAGISEL-GFX10-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY]], 0, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY1]], 0, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_ADD_U32_e64_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY6]], [[COPY2]], 0, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_ADD_U32_e64_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY7]], [[COPY3]], 0, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_ADD_U32_e64_3]], %subreg.sub0, [[V_ADD_U32_e64_2]], %subreg.sub1, [[V_ADD_U32_e64_1]], %subreg.sub2, [[V_ADD_U32_e64_]], %subreg.sub3
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF4:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[DEF4]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY9:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX10-WF32-NEXT: FLAT_STORE_DWORDX4 killed [[COPY8]], killed [[COPY9]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison`)
+ ; DAGISEL-GFX10-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc
+ ; DAGISEL-GFX10-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF64-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr8, $vgpr9, $vgpr10, $vgpr11
+ ; DAGISEL-GFX10-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY]], 0, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY1]], 0, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_ADD_U32_e64_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY6]], [[COPY2]], 0, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_ADD_U32_e64_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY7]], [[COPY3]], 0, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_ADD_U32_e64_3]], %subreg.sub0, [[V_ADD_U32_e64_2]], %subreg.sub1, [[V_ADD_U32_e64_1]], %subreg.sub2, [[V_ADD_U32_e64_]], %subreg.sub3
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF4:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[DEF4]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY9:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX10-WF64-NEXT: FLAT_STORE_DWORDX4 killed [[COPY8]], killed [[COPY9]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison`)
+ ; DAGISEL-GFX10-WF64-NEXT: S_ENDPGM 0
%c = add <4 x i32> %a, %b
store <4 x i32> %c, ptr poison
ret void
@@ -183,81 +237,157 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_ptr(ptr inreg
; GISEL-GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY15]], [[COPY11]], $sgpr48_sgpr49_sgpr50_sgpr51, 0, 0, 0, 0, implicit $exec :: (store (p5) into %ir.b5, addrspace 5)
; GISEL-GFX10-NEXT: S_ENDPGM 0
;
- ; DAGISEL-GFX11-LABEL: name: amdgpu_cs_chain_preserve_cc_ptr
- ; DAGISEL-GFX11: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-NEXT: liveins: $sgpr0, $sgpr1, $vgpr8, $vgpr9, $sgpr2, $sgpr3, $vgpr10, $vgpr11, $sgpr4, $vgpr12, $sgpr5, $vgpr13
- ; DAGISEL-GFX11-NEXT: {{ $}}
- ; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr13
- ; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr5
- ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr12
- ; DAGISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr4
- ; DAGISEL-GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr11
- ; DAGISEL-GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; DAGISEL-GFX11-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr3
- ; DAGISEL-GFX11-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr2
- ; DAGISEL-GFX11-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr9
- ; DAGISEL-GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr1
- ; DAGISEL-GFX11-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
- ; DAGISEL-GFX11-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
- ; DAGISEL-GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY13]], %subreg.sub1
- ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
- ; DAGISEL-GFX11-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
- ; DAGISEL-GFX11-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
- ; DAGISEL-GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY14]], %subreg.sub0, [[COPY15]], %subreg.sub1
- ; DAGISEL-GFX11-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1
- ; DAGISEL-GFX11-NEXT: [[COPY16:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; DAGISEL-GFX11-NEXT: FLAT_STORE_DWORDX2 killed [[COPY16]], killed [[REG_SEQUENCE2]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s64) into %ir.b)
- ; DAGISEL-GFX11-NEXT: [[COPY17:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
- ; DAGISEL-GFX11-NEXT: GLOBAL_STORE_DWORDX2 killed [[COPY17]], killed [[REG_SEQUENCE]], 0, 0, implicit $exec :: (store (s64) into %ir.b1, addrspace 1)
- ; DAGISEL-GFX11-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
- ; DAGISEL-GFX11-NEXT: DS_WRITE_B32_gfx9 [[COPY2]], [[COPY18]], 0, 0, implicit $exec :: (store (s32) into %ir.b3, addrspace 3)
- ; DAGISEL-GFX11-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; DAGISEL-GFX11-NEXT: SCRATCH_STORE_DWORD [[COPY19]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %ir.b5, addrspace 5)
- ; DAGISEL-GFX11-NEXT: S_ENDPGM 0
- ;
- ; DAGISEL-GFX10-LABEL: name: amdgpu_cs_chain_preserve_cc_ptr
- ; DAGISEL-GFX10: bb.0 (%ir-block.0):
- ; DAGISEL-GFX10-NEXT: liveins: $sgpr0, $sgpr1, $vgpr8, $vgpr9, $sgpr2, $sgpr3, $vgpr10, $vgpr11, $sgpr4, $vgpr12, $sgpr5, $vgpr13
- ; DAGISEL-GFX10-NEXT: {{ $}}
- ; DAGISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr13
- ; DAGISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr5
- ; DAGISEL-GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr12
- ; DAGISEL-GFX10-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr4
- ; DAGISEL-GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr11
- ; DAGISEL-GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; DAGISEL-GFX10-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr3
- ; DAGISEL-GFX10-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr2
- ; DAGISEL-GFX10-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr9
- ; DAGISEL-GFX10-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX10-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr1
- ; DAGISEL-GFX10-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX10-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
- ; DAGISEL-GFX10-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
- ; DAGISEL-GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY13]], %subreg.sub1
- ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
- ; DAGISEL-GFX10-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
- ; DAGISEL-GFX10-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
- ; DAGISEL-GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY14]], %subreg.sub0, [[COPY15]], %subreg.sub1
- ; DAGISEL-GFX10-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1
- ; DAGISEL-GFX10-NEXT: [[COPY16:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
- ; DAGISEL-GFX10-NEXT: FLAT_STORE_DWORDX2 killed [[COPY16]], killed [[REG_SEQUENCE2]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s64) into %ir.b)
- ; DAGISEL-GFX10-NEXT: [[COPY17:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
- ; DAGISEL-GFX10-NEXT: GLOBAL_STORE_DWORDX2 killed [[COPY17]], killed [[REG_SEQUENCE]], 0, 0, implicit $exec :: (store (s64) into %ir.b1, addrspace 1)
- ; DAGISEL-GFX10-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
- ; DAGISEL-GFX10-NEXT: DS_WRITE_B32_gfx9 [[COPY2]], [[COPY18]], 0, 0, implicit $exec :: (store (s32) into %ir.b3, addrspace 3)
- ; DAGISEL-GFX10-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; DAGISEL-GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY19]], [[COPY]], $sgpr48_sgpr49_sgpr50_sgpr51, 0, 0, 0, 0, implicit $exec :: (store (s32) into %ir.b5, addrspace 5)
- ; DAGISEL-GFX10-NEXT: S_ENDPGM 0
+ ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc_ptr
+ ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF32-NEXT: liveins: $sgpr0, $sgpr1, $vgpr8, $vgpr9, $sgpr2, $sgpr3, $vgpr10, $vgpr11, $sgpr4, $vgpr12, $sgpr5, $vgpr13
+ ; DAGISEL-GFX11-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr13
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr5
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr12
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY13]], %subreg.sub1
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY14]], %subreg.sub0, [[COPY15]], %subreg.sub1
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY16:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
+ ; DAGISEL-GFX11-WF32-NEXT: FLAT_STORE_DWORDX2 killed [[COPY16]], killed [[REG_SEQUENCE2]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s64) into %ir.b)
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY17:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+ ; DAGISEL-GFX11-WF32-NEXT: GLOBAL_STORE_DWORDX2 killed [[COPY17]], killed [[REG_SEQUENCE]], 0, 0, implicit $exec :: (store (s64) into %ir.b1, addrspace 1)
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+ ; DAGISEL-GFX11-WF32-NEXT: DS_WRITE_B32_gfx9 [[COPY2]], [[COPY18]], 0, 0, implicit $exec :: (store (s32) into %ir.b3, addrspace 3)
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-WF32-NEXT: SCRATCH_STORE_DWORD [[COPY19]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %ir.b5, addrspace 5)
+ ; DAGISEL-GFX11-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc_ptr
+ ; DAGISEL-GFX11-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF64-NEXT: liveins: $sgpr0, $sgpr1, $vgpr8, $vgpr9, $sgpr2, $sgpr3, $vgpr10, $vgpr11, $sgpr4, $vgpr12, $sgpr5, $vgpr13
+ ; DAGISEL-GFX11-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr13
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr5
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr12
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY13]], %subreg.sub1
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY14]], %subreg.sub0, [[COPY15]], %subreg.sub1
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY16:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
+ ; DAGISEL-GFX11-WF64-NEXT: FLAT_STORE_DWORDX2 killed [[COPY16]], killed [[REG_SEQUENCE2]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s64) into %ir.b)
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY17:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+ ; DAGISEL-GFX11-WF64-NEXT: GLOBAL_STORE_DWORDX2 killed [[COPY17]], killed [[REG_SEQUENCE]], 0, 0, implicit $exec :: (store (s64) into %ir.b1, addrspace 1)
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+ ; DAGISEL-GFX11-WF64-NEXT: DS_WRITE_B32_gfx9 [[COPY2]], [[COPY18]], 0, 0, implicit $exec :: (store (s32) into %ir.b3, addrspace 3)
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-WF64-NEXT: SCRATCH_STORE_DWORD [[COPY19]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %ir.b5, addrspace 5)
+ ; DAGISEL-GFX11-WF64-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc_ptr
+ ; DAGISEL-GFX10-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF32-NEXT: liveins: $sgpr0, $sgpr1, $vgpr8, $vgpr9, $sgpr2, $sgpr3, $vgpr10, $vgpr11, $sgpr4, $vgpr12, $sgpr5, $vgpr13
+ ; DAGISEL-GFX10-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr13
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr5
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr12
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY13]], %subreg.sub1
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY14]], %subreg.sub0, [[COPY15]], %subreg.sub1
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY16:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
+ ; DAGISEL-GFX10-WF32-NEXT: FLAT_STORE_DWORDX2 killed [[COPY16]], killed [[REG_SEQUENCE2]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s64) into %ir.b)
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY17:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+ ; DAGISEL-GFX10-WF32-NEXT: GLOBAL_STORE_DWORDX2 killed [[COPY17]], killed [[REG_SEQUENCE]], 0, 0, implicit $exec :: (store (s64) into %ir.b1, addrspace 1)
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+ ; DAGISEL-GFX10-WF32-NEXT: DS_WRITE_B32_gfx9 [[COPY2]], [[COPY18]], 0, 0, implicit $exec :: (store (s32) into %ir.b3, addrspace 3)
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+ ; DAGISEL-GFX10-WF32-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY19]], [[COPY]], $sgpr48_sgpr49_sgpr50_sgpr51, 0, 0, 0, 0, implicit $exec :: (store (s32) into %ir.b5, addrspace 5)
+ ; DAGISEL-GFX10-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc_ptr
+ ; DAGISEL-GFX10-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF64-NEXT: liveins: $sgpr0, $sgpr1, $vgpr8, $vgpr9, $sgpr2, $sgpr3, $vgpr10, $vgpr11, $sgpr4, $vgpr12, $sgpr5, $vgpr13
+ ; DAGISEL-GFX10-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr13
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr5
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr12
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY13]], %subreg.sub1
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY14]], %subreg.sub0, [[COPY15]], %subreg.sub1
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY16:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE3]]
+ ; DAGISEL-GFX10-WF64-NEXT: FLAT_STORE_DWORDX2 killed [[COPY16]], killed [[REG_SEQUENCE2]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s64) into %ir.b)
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY17:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+ ; DAGISEL-GFX10-WF64-NEXT: GLOBAL_STORE_DWORDX2 killed [[COPY17]], killed [[REG_SEQUENCE]], 0, 0, implicit $exec :: (store (s64) into %ir.b1, addrspace 1)
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
+ ; DAGISEL-GFX10-WF64-NEXT: DS_WRITE_B32_gfx9 [[COPY2]], [[COPY18]], 0, 0, implicit $exec :: (store (s32) into %ir.b3, addrspace 3)
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+ ; DAGISEL-GFX10-WF64-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY19]], [[COPY]], $sgpr48_sgpr49_sgpr50_sgpr51, 0, 0, 0, 0, implicit $exec :: (store (s32) into %ir.b5, addrspace 5)
+ ; DAGISEL-GFX10-WF64-NEXT: S_ENDPGM 0
store ptr %a, ptr %b
store ptr addrspace(1) %a1, ptr addrspace(1) %b1
store ptr addrspace(3) %a3, ptr addrspace(3) %b3
@@ -346,119 +476,233 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_struct( {ptr,
; GISEL-GFX10-NEXT: GLOBAL_STORE_DWORDX4 [[COPY22]], [[REG_SEQUENCE3]], 16, 0, implicit $exec :: (store (<4 x s32>) into `ptr addrspace(1) poison` + 16, addrspace 1)
; GISEL-GFX10-NEXT: S_ENDPGM 0
;
- ; DAGISEL-GFX11-LABEL: name: amdgpu_cs_chain_preserve_cc_struct
- ; DAGISEL-GFX11: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14
- ; DAGISEL-GFX11-NEXT: {{ $}}
- ; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr14
- ; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr13
- ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr12
- ; DAGISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr11
- ; DAGISEL-GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; DAGISEL-GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr9
- ; DAGISEL-GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr6
- ; DAGISEL-GFX11-NEXT: [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr5
- ; DAGISEL-GFX11-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr4
- ; DAGISEL-GFX11-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr3
- ; DAGISEL-GFX11-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr2
- ; DAGISEL-GFX11-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr1
- ; DAGISEL-GFX11-NEXT: [[COPY13:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
- ; DAGISEL-GFX11-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
- ; DAGISEL-GFX11-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
- ; DAGISEL-GFX11-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
- ; DAGISEL-GFX11-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
- ; DAGISEL-GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY14]], %subreg.sub0, [[COPY15]], %subreg.sub1, [[COPY16]], %subreg.sub2, [[COPY17]], %subreg.sub3
- ; DAGISEL-GFX11-NEXT: [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; DAGISEL-GFX11-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY13]]
- ; DAGISEL-GFX11-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY12]]
- ; DAGISEL-GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY18]], %subreg.sub0, [[COPY19]], %subreg.sub1
- ; DAGISEL-GFX11-NEXT: [[DEF6:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY20:%[0-9]+]]:vreg_64 = COPY [[DEF6]]
- ; DAGISEL-GFX11-NEXT: GLOBAL_STORE_DWORDX2 [[COPY20]], killed [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, addrspace 1)
- ; DAGISEL-GFX11-NEXT: [[DEF7:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY21:%[0-9]+]]:vreg_64 = COPY [[DEF7]]
- ; DAGISEL-GFX11-NEXT: [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
- ; DAGISEL-GFX11-NEXT: GLOBAL_STORE_DWORD [[COPY21]], [[COPY22]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison`, addrspace 1)
- ; DAGISEL-GFX11-NEXT: [[DEF8:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY23:%[0-9]+]]:vreg_64 = COPY [[DEF8]]
- ; DAGISEL-GFX11-NEXT: GLOBAL_STORE_DWORDX4 [[COPY23]], killed [[REG_SEQUENCE1]], 0, 0, implicit $exec :: (store (s128) into `ptr addrspace(1) poison`, addrspace 1)
- ; DAGISEL-GFX11-NEXT: [[DEF9:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY24:%[0-9]+]]:vreg_64 = COPY [[DEF9]]
- ; DAGISEL-GFX11-NEXT: [[COPY25:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
- ; DAGISEL-GFX11-NEXT: GLOBAL_STORE_DWORDX4 [[COPY24]], killed [[COPY25]], 0, 0, implicit $exec :: (store (s128) into `ptr addrspace(1) poison` + 16, addrspace 1)
- ; DAGISEL-GFX11-NEXT: [[DEF10:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY26:%[0-9]+]]:vreg_64 = COPY [[DEF10]]
- ; DAGISEL-GFX11-NEXT: GLOBAL_STORE_DWORD [[COPY26]], [[COPY4]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison` + 8, align 8, basealign 16, addrspace 1)
- ; DAGISEL-GFX11-NEXT: [[DEF11:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY27:%[0-9]+]]:vreg_64 = COPY [[DEF11]]
- ; DAGISEL-GFX11-NEXT: [[COPY28:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]]
- ; DAGISEL-GFX11-NEXT: GLOBAL_STORE_DWORDX2 [[COPY27]], killed [[COPY28]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, align 16, addrspace 1)
- ; DAGISEL-GFX11-NEXT: S_ENDPGM 0
- ;
- ; DAGISEL-GFX10-LABEL: name: amdgpu_cs_chain_preserve_cc_struct
- ; DAGISEL-GFX10: bb.0 (%ir-block.0):
- ; DAGISEL-GFX10-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14
- ; DAGISEL-GFX10-NEXT: {{ $}}
- ; DAGISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr14
- ; DAGISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr13
- ; DAGISEL-GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr12
- ; DAGISEL-GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr11
- ; DAGISEL-GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; DAGISEL-GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr9
- ; DAGISEL-GFX10-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX10-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr6
- ; DAGISEL-GFX10-NEXT: [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr5
- ; DAGISEL-GFX10-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr4
- ; DAGISEL-GFX10-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr3
- ; DAGISEL-GFX10-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr2
- ; DAGISEL-GFX10-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr1
- ; DAGISEL-GFX10-NEXT: [[COPY13:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
- ; DAGISEL-GFX10-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
- ; DAGISEL-GFX10-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
- ; DAGISEL-GFX10-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
- ; DAGISEL-GFX10-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
- ; DAGISEL-GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY14]], %subreg.sub0, [[COPY15]], %subreg.sub1, [[COPY16]], %subreg.sub2, [[COPY17]], %subreg.sub3
- ; DAGISEL-GFX10-NEXT: [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; DAGISEL-GFX10-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY13]]
- ; DAGISEL-GFX10-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY12]]
- ; DAGISEL-GFX10-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY18]], %subreg.sub0, [[COPY19]], %subreg.sub1
- ; DAGISEL-GFX10-NEXT: [[DEF6:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY20:%[0-9]+]]:vreg_64 = COPY [[DEF6]]
- ; DAGISEL-GFX10-NEXT: GLOBAL_STORE_DWORDX2 [[COPY20]], killed [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, addrspace 1)
- ; DAGISEL-GFX10-NEXT: [[DEF7:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY21:%[0-9]+]]:vreg_64 = COPY [[DEF7]]
- ; DAGISEL-GFX10-NEXT: [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
- ; DAGISEL-GFX10-NEXT: GLOBAL_STORE_DWORD [[COPY21]], [[COPY22]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison`, addrspace 1)
- ; DAGISEL-GFX10-NEXT: [[DEF8:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY23:%[0-9]+]]:vreg_64 = COPY [[DEF8]]
- ; DAGISEL-GFX10-NEXT: GLOBAL_STORE_DWORDX4 [[COPY23]], killed [[REG_SEQUENCE1]], 0, 0, implicit $exec :: (store (s128) into `ptr addrspace(1) poison`, addrspace 1)
- ; DAGISEL-GFX10-NEXT: [[DEF9:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY24:%[0-9]+]]:vreg_64 = COPY [[DEF9]]
- ; DAGISEL-GFX10-NEXT: [[COPY25:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
- ; DAGISEL-GFX10-NEXT: GLOBAL_STORE_DWORDX4 [[COPY24]], killed [[COPY25]], 0, 0, implicit $exec :: (store (s128) into `ptr addrspace(1) poison` + 16, addrspace 1)
- ; DAGISEL-GFX10-NEXT: [[DEF10:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY26:%[0-9]+]]:vreg_64 = COPY [[DEF10]]
- ; DAGISEL-GFX10-NEXT: GLOBAL_STORE_DWORD [[COPY26]], [[COPY4]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison` + 8, align 8, basealign 16, addrspace 1)
- ; DAGISEL-GFX10-NEXT: [[DEF11:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY27:%[0-9]+]]:vreg_64 = COPY [[DEF11]]
- ; DAGISEL-GFX10-NEXT: [[COPY28:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]]
- ; DAGISEL-GFX10-NEXT: GLOBAL_STORE_DWORDX2 [[COPY27]], killed [[COPY28]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, align 16, addrspace 1)
- ; DAGISEL-GFX10-NEXT: S_ENDPGM 0
+ ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc_struct
+ ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF32-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14
+ ; DAGISEL-GFX11-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr14
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr13
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr12
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr6
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr5
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY13:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY14]], %subreg.sub0, [[COPY15]], %subreg.sub1, [[COPY16]], %subreg.sub2, [[COPY17]], %subreg.sub3
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY13]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY12]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY18]], %subreg.sub0, [[COPY19]], %subreg.sub1
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF6:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY20:%[0-9]+]]:vreg_64 = COPY [[DEF6]]
+ ; DAGISEL-GFX11-WF32-NEXT: GLOBAL_STORE_DWORDX2 [[COPY20]], killed [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF7:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY21:%[0-9]+]]:vreg_64 = COPY [[DEF7]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+ ; DAGISEL-GFX11-WF32-NEXT: GLOBAL_STORE_DWORD [[COPY21]], [[COPY22]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF8:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY23:%[0-9]+]]:vreg_64 = COPY [[DEF8]]
+ ; DAGISEL-GFX11-WF32-NEXT: GLOBAL_STORE_DWORDX4 [[COPY23]], killed [[REG_SEQUENCE1]], 0, 0, implicit $exec :: (store (s128) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF9:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY24:%[0-9]+]]:vreg_64 = COPY [[DEF9]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY25:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX11-WF32-NEXT: GLOBAL_STORE_DWORDX4 [[COPY24]], killed [[COPY25]], 0, 0, implicit $exec :: (store (s128) into `ptr addrspace(1) poison` + 16, addrspace 1)
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF10:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY26:%[0-9]+]]:vreg_64 = COPY [[DEF10]]
+ ; DAGISEL-GFX11-WF32-NEXT: GLOBAL_STORE_DWORD [[COPY26]], [[COPY4]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison` + 8, align 8, basealign 16, addrspace 1)
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF11:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY27:%[0-9]+]]:vreg_64 = COPY [[DEF11]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY28:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]]
+ ; DAGISEL-GFX11-WF32-NEXT: GLOBAL_STORE_DWORDX2 [[COPY27]], killed [[COPY28]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, align 16, addrspace 1)
+ ; DAGISEL-GFX11-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc_struct
+ ; DAGISEL-GFX11-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF64-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14
+ ; DAGISEL-GFX11-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr14
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr13
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr12
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr6
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr5
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY13:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY14]], %subreg.sub0, [[COPY15]], %subreg.sub1, [[COPY16]], %subreg.sub2, [[COPY17]], %subreg.sub3
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY13]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY12]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY18]], %subreg.sub0, [[COPY19]], %subreg.sub1
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF6:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY20:%[0-9]+]]:vreg_64 = COPY [[DEF6]]
+ ; DAGISEL-GFX11-WF64-NEXT: GLOBAL_STORE_DWORDX2 [[COPY20]], killed [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF7:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY21:%[0-9]+]]:vreg_64 = COPY [[DEF7]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+ ; DAGISEL-GFX11-WF64-NEXT: GLOBAL_STORE_DWORD [[COPY21]], [[COPY22]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF8:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY23:%[0-9]+]]:vreg_64 = COPY [[DEF8]]
+ ; DAGISEL-GFX11-WF64-NEXT: GLOBAL_STORE_DWORDX4 [[COPY23]], killed [[REG_SEQUENCE1]], 0, 0, implicit $exec :: (store (s128) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF9:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY24:%[0-9]+]]:vreg_64 = COPY [[DEF9]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY25:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX11-WF64-NEXT: GLOBAL_STORE_DWORDX4 [[COPY24]], killed [[COPY25]], 0, 0, implicit $exec :: (store (s128) into `ptr addrspace(1) poison` + 16, addrspace 1)
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF10:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY26:%[0-9]+]]:vreg_64 = COPY [[DEF10]]
+ ; DAGISEL-GFX11-WF64-NEXT: GLOBAL_STORE_DWORD [[COPY26]], [[COPY4]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison` + 8, align 8, basealign 16, addrspace 1)
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF11:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY27:%[0-9]+]]:vreg_64 = COPY [[DEF11]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY28:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]]
+ ; DAGISEL-GFX11-WF64-NEXT: GLOBAL_STORE_DWORDX2 [[COPY27]], killed [[COPY28]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, align 16, addrspace 1)
+ ; DAGISEL-GFX11-WF64-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc_struct
+ ; DAGISEL-GFX10-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF32-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14
+ ; DAGISEL-GFX10-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr14
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr13
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr12
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr6
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr5
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY13:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY14]], %subreg.sub0, [[COPY15]], %subreg.sub1, [[COPY16]], %subreg.sub2, [[COPY17]], %subreg.sub3
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY13]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY12]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY18]], %subreg.sub0, [[COPY19]], %subreg.sub1
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF6:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY20:%[0-9]+]]:vreg_64 = COPY [[DEF6]]
+ ; DAGISEL-GFX10-WF32-NEXT: GLOBAL_STORE_DWORDX2 [[COPY20]], killed [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF7:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY21:%[0-9]+]]:vreg_64 = COPY [[DEF7]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+ ; DAGISEL-GFX10-WF32-NEXT: GLOBAL_STORE_DWORD [[COPY21]], [[COPY22]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF8:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY23:%[0-9]+]]:vreg_64 = COPY [[DEF8]]
+ ; DAGISEL-GFX10-WF32-NEXT: GLOBAL_STORE_DWORDX4 [[COPY23]], killed [[REG_SEQUENCE1]], 0, 0, implicit $exec :: (store (s128) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF9:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY24:%[0-9]+]]:vreg_64 = COPY [[DEF9]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY25:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX10-WF32-NEXT: GLOBAL_STORE_DWORDX4 [[COPY24]], killed [[COPY25]], 0, 0, implicit $exec :: (store (s128) into `ptr addrspace(1) poison` + 16, addrspace 1)
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF10:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY26:%[0-9]+]]:vreg_64 = COPY [[DEF10]]
+ ; DAGISEL-GFX10-WF32-NEXT: GLOBAL_STORE_DWORD [[COPY26]], [[COPY4]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison` + 8, align 8, basealign 16, addrspace 1)
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF11:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY27:%[0-9]+]]:vreg_64 = COPY [[DEF11]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY28:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]]
+ ; DAGISEL-GFX10-WF32-NEXT: GLOBAL_STORE_DWORDX2 [[COPY27]], killed [[COPY28]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, align 16, addrspace 1)
+ ; DAGISEL-GFX10-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc_struct
+ ; DAGISEL-GFX10-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF64-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14
+ ; DAGISEL-GFX10-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr14
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr13
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr12
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr6
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr5
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY13:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY10]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY14]], %subreg.sub0, [[COPY15]], %subreg.sub1, [[COPY16]], %subreg.sub2, [[COPY17]], %subreg.sub3
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY13]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY12]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY18]], %subreg.sub0, [[COPY19]], %subreg.sub1
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF6:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY20:%[0-9]+]]:vreg_64 = COPY [[DEF6]]
+ ; DAGISEL-GFX10-WF64-NEXT: GLOBAL_STORE_DWORDX2 [[COPY20]], killed [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF7:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY21:%[0-9]+]]:vreg_64 = COPY [[DEF7]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[COPY11]]
+ ; DAGISEL-GFX10-WF64-NEXT: GLOBAL_STORE_DWORD [[COPY21]], [[COPY22]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF8:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY23:%[0-9]+]]:vreg_64 = COPY [[DEF8]]
+ ; DAGISEL-GFX10-WF64-NEXT: GLOBAL_STORE_DWORDX4 [[COPY23]], killed [[REG_SEQUENCE1]], 0, 0, implicit $exec :: (store (s128) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF9:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY24:%[0-9]+]]:vreg_64 = COPY [[DEF9]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY25:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX10-WF64-NEXT: GLOBAL_STORE_DWORDX4 [[COPY24]], killed [[COPY25]], 0, 0, implicit $exec :: (store (s128) into `ptr addrspace(1) poison` + 16, addrspace 1)
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF10:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY26:%[0-9]+]]:vreg_64 = COPY [[DEF10]]
+ ; DAGISEL-GFX10-WF64-NEXT: GLOBAL_STORE_DWORD [[COPY26]], [[COPY4]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison` + 8, align 8, basealign 16, addrspace 1)
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF11:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY27:%[0-9]+]]:vreg_64 = COPY [[DEF11]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY28:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]]
+ ; DAGISEL-GFX10-WF64-NEXT: GLOBAL_STORE_DWORDX2 [[COPY27]], killed [[COPY28]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, align 16, addrspace 1)
+ ; DAGISEL-GFX10-WF64-NEXT: S_ENDPGM 0
%p = extractvalue {ptr, i32, <4 x i32>} %a, 0
%i = extractvalue {ptr, i32, <4 x i32>} %a, 1
%v = extractvalue {ptr, i32, <4 x i32>} %a, 2
@@ -497,29 +741,53 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_float(float in
; GISEL-GFX10-NEXT: FLAT_STORE_DWORD [[COPY3]], [[V_ADD_F32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
; GISEL-GFX10-NEXT: S_ENDPGM 0
;
- ; DAGISEL-GFX11-LABEL: name: amdgpu_cs_chain_preserve_cc_float
- ; DAGISEL-GFX11: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX11-NEXT: {{ $}}
- ; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX11-NEXT: FLAT_STORE_DWORD killed [[COPY2]], killed [[V_ADD_F32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
- ; DAGISEL-GFX11-NEXT: S_ENDPGM 0
- ;
- ; DAGISEL-GFX10-LABEL: name: amdgpu_cs_chain_preserve_cc_float
- ; DAGISEL-GFX10: bb.0 (%ir-block.0):
- ; DAGISEL-GFX10-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX10-NEXT: {{ $}}
- ; DAGISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX10-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX10-NEXT: FLAT_STORE_DWORD killed [[COPY2]], killed [[V_ADD_F32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
- ; DAGISEL-GFX10-NEXT: S_ENDPGM 0
+ ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc_float
+ ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF32-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF32-NEXT: FLAT_STORE_DWORD killed [[COPY2]], killed [[V_ADD_F32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+ ; DAGISEL-GFX11-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc_float
+ ; DAGISEL-GFX11-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF64-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF64-NEXT: FLAT_STORE_DWORD killed [[COPY2]], killed [[V_ADD_F32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+ ; DAGISEL-GFX11-WF64-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc_float
+ ; DAGISEL-GFX10-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF32-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX10-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX10-WF32-NEXT: FLAT_STORE_DWORD killed [[COPY2]], killed [[V_ADD_F32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+ ; DAGISEL-GFX10-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc_float
+ ; DAGISEL-GFX10-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF64-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX10-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX10-WF64-NEXT: FLAT_STORE_DWORD killed [[COPY2]], killed [[V_ADD_F32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
+ ; DAGISEL-GFX10-WF64-NEXT: S_ENDPGM 0
%c = fadd float %a, %b
store float %c, ptr poison
ret void
@@ -552,29 +820,53 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_half(half inre
; GISEL-GFX10-NEXT: FLAT_STORE_SHORT [[COPY3]], [[V_ADD_F16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
; GISEL-GFX10-NEXT: S_ENDPGM 0
;
- ; DAGISEL-GFX11-LABEL: name: amdgpu_cs_chain_preserve_cc_half
- ; DAGISEL-GFX11: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX11-NEXT: {{ $}}
- ; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-NEXT: [[V_ADD_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F16_fake16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX11-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_F16_fake16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
- ; DAGISEL-GFX11-NEXT: S_ENDPGM 0
- ;
- ; DAGISEL-GFX10-LABEL: name: amdgpu_cs_chain_preserve_cc_half
- ; DAGISEL-GFX10: bb.0 (%ir-block.0):
- ; DAGISEL-GFX10-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX10-NEXT: {{ $}}
- ; DAGISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX10-NEXT: [[V_ADD_F16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX10-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_F16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
- ; DAGISEL-GFX10-NEXT: S_ENDPGM 0
+ ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc_half
+ ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF32-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F16_fake16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF32-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_F16_fake16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX11-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc_half
+ ; DAGISEL-GFX11-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF64-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F16_fake16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF64-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_F16_fake16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX11-WF64-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc_half
+ ; DAGISEL-GFX10-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF32-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX10-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_ADD_F16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX10-WF32-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_F16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX10-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc_half
+ ; DAGISEL-GFX10-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF64-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX10-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_ADD_F16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX10-WF64-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_F16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX10-WF64-NEXT: S_ENDPGM 0
%c = fadd half %a, %b
store half %c, ptr poison
ret void
@@ -607,33 +899,93 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a,
; GISEL-GFX10-NEXT: FLAT_STORE_SHORT [[COPY3]], [[V_ADD_F16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
; GISEL-GFX10-NEXT: S_ENDPGM 0
;
- ; DAGISEL-GFX11-LABEL: name: amdgpu_cs_chain_cc_bfloat
- ; DAGISEL-GFX11: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX11-NEXT: {{ $}}
- ; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
- ; DAGISEL-GFX11-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
- ; DAGISEL-GFX11-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX11-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_ADD_F32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
- ; DAGISEL-GFX11-NEXT: S_ENDPGM 0
- ;
- ; DAGISEL-GFX10-LABEL: name: amdgpu_cs_chain_cc_bfloat
- ; DAGISEL-GFX10: bb.0 (%ir-block.0):
- ; DAGISEL-GFX10-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX10-NEXT: {{ $}}
- ; DAGISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX10-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
- ; DAGISEL-GFX10-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
- ; DAGISEL-GFX10-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX10-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_ADD_F32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
- ; DAGISEL-GFX10-NEXT: S_ENDPGM 0
+ ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_cc_bfloat
+ ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF32-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
+ ; DAGISEL-GFX11-WF32-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_AND_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_2]], killed [[S_MOV_B32_1]], implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_AND_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF32-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX11-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_cc_bfloat
+ ; DAGISEL-GFX11-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF64-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
+ ; DAGISEL-GFX11-WF64-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_AND_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_2]], killed [[S_MOV_B32_1]], implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_AND_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF64-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX11-WF64-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_cc_bfloat
+ ; DAGISEL-GFX10-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF32-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX10-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
+ ; DAGISEL-GFX10-WF32-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_AND_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_2]], killed [[S_MOV_B32_1]], implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_AND_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX10-WF32-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX10-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF64-LABEL: name: amdgpu_cs_chain_cc_bfloat
+ ; DAGISEL-GFX10-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF64-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX10-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
+ ; DAGISEL-GFX10-WF64-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_AND_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_2]], killed [[S_MOV_B32_1]], implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_AND_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX10-WF64-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX10-WF64-NEXT: S_ENDPGM 0
%c = fadd bfloat %a, %b
store bfloat %c, ptr poison
ret void
@@ -666,29 +1018,53 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_i16(i16 inreg
; GISEL-GFX10-NEXT: FLAT_STORE_SHORT [[COPY3]], [[V_ADD_NC_U16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
; GISEL-GFX10-NEXT: S_ENDPGM 0
;
- ; DAGISEL-GFX11-LABEL: name: amdgpu_cs_chain_preserve_cc_i16
- ; DAGISEL-GFX11: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX11-NEXT: {{ $}}
- ; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-NEXT: [[V_ADD_NC_U16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_NC_U16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX11-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_NC_U16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
- ; DAGISEL-GFX11-NEXT: S_ENDPGM 0
- ;
- ; DAGISEL-GFX10-LABEL: name: amdgpu_cs_chain_preserve_cc_i16
- ; DAGISEL-GFX10: bb.0 (%ir-block.0):
- ; DAGISEL-GFX10-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX10-NEXT: {{ $}}
- ; DAGISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX10-NEXT: [[V_ADD_NC_U16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_NC_U16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX10-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_NC_U16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
- ; DAGISEL-GFX10-NEXT: S_ENDPGM 0
+ ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc_i16
+ ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF32-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD_NC_U16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_NC_U16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF32-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_NC_U16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX11-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc_i16
+ ; DAGISEL-GFX11-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF64-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD_NC_U16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_NC_U16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF64-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_NC_U16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX11-WF64-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc_i16
+ ; DAGISEL-GFX10-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF32-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX10-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_ADD_NC_U16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_NC_U16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX10-WF32-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_NC_U16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX10-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc_i16
+ ; DAGISEL-GFX10-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF64-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX10-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_ADD_NC_U16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_NC_U16_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX10-WF64-NEXT: FLAT_STORE_SHORT killed [[COPY2]], killed [[V_ADD_NC_U16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX10-WF64-NEXT: S_ENDPGM 0
%c = add i16 %a, %b
store i16 %c, ptr poison
ret void
@@ -787,101 +1163,197 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_v16i16(<16 x i
; GISEL-GFX10-NEXT: FLAT_STORE_DWORDX4 [[COPY27]], [[COPY25]], 16, 0, implicit $exec, implicit $flat_scr :: (store (<4 x s32>) into `ptr poison` + 16, basealign 32)
; GISEL-GFX10-NEXT: S_ENDPGM 0
;
- ; DAGISEL-GFX11-LABEL: name: amdgpu_cs_chain_preserve_cc_v16i16
- ; DAGISEL-GFX11: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15
- ; DAGISEL-GFX11-NEXT: {{ $}}
- ; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr15
- ; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr14
- ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr13
- ; DAGISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr12
- ; DAGISEL-GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr11
- ; DAGISEL-GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; DAGISEL-GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr9
- ; DAGISEL-GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-NEXT: [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr7
- ; DAGISEL-GFX11-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr6
- ; DAGISEL-GFX11-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr5
- ; DAGISEL-GFX11-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr4
- ; DAGISEL-GFX11-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr3
- ; DAGISEL-GFX11-NEXT: [[COPY13:%[0-9]+]]:sgpr_32 = COPY $sgpr2
- ; DAGISEL-GFX11-NEXT: [[COPY14:%[0-9]+]]:sgpr_32 = COPY $sgpr1
- ; DAGISEL-GFX11-NEXT: [[COPY15:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-NEXT: [[V_PK_ADD_U16_:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY12]], 8, [[COPY4]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[V_PK_ADD_U16_1:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY13]], 8, [[COPY5]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[V_PK_ADD_U16_2:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY14]], 8, [[COPY6]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[V_PK_ADD_U16_3:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY15]], 8, [[COPY7]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[V_PK_ADD_U16_4:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY8]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[V_PK_ADD_U16_5:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY9]], 8, [[COPY1]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[V_PK_ADD_U16_6:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY10]], 8, [[COPY2]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[V_PK_ADD_U16_7:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY11]], 8, [[COPY3]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_PK_ADD_U16_7]], %subreg.sub0, [[V_PK_ADD_U16_6]], %subreg.sub1, [[V_PK_ADD_U16_5]], %subreg.sub2, [[V_PK_ADD_U16_4]], %subreg.sub3
- ; DAGISEL-GFX11-NEXT: [[DEF4:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY16:%[0-9]+]]:vreg_64 = COPY [[DEF4]]
- ; DAGISEL-GFX11-NEXT: [[COPY17:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
- ; DAGISEL-GFX11-NEXT: FLAT_STORE_DWORDX4 [[COPY16]], killed [[COPY17]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison` + 16)
- ; DAGISEL-GFX11-NEXT: [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF7:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF8:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_PK_ADD_U16_3]], %subreg.sub0, [[V_PK_ADD_U16_2]], %subreg.sub1, [[V_PK_ADD_U16_1]], %subreg.sub2, [[V_PK_ADD_U16_]], %subreg.sub3
- ; DAGISEL-GFX11-NEXT: [[DEF9:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY18:%[0-9]+]]:vreg_64 = COPY [[DEF9]]
- ; DAGISEL-GFX11-NEXT: [[COPY19:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE1]]
- ; DAGISEL-GFX11-NEXT: FLAT_STORE_DWORDX4 [[COPY18]], killed [[COPY19]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison`, align 32)
- ; DAGISEL-GFX11-NEXT: S_ENDPGM 0
- ;
- ; DAGISEL-GFX10-LABEL: name: amdgpu_cs_chain_preserve_cc_v16i16
- ; DAGISEL-GFX10: bb.0 (%ir-block.0):
- ; DAGISEL-GFX10-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15
- ; DAGISEL-GFX10-NEXT: {{ $}}
- ; DAGISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr15
- ; DAGISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr14
- ; DAGISEL-GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr13
- ; DAGISEL-GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr12
- ; DAGISEL-GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr11
- ; DAGISEL-GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
- ; DAGISEL-GFX10-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr9
- ; DAGISEL-GFX10-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX10-NEXT: [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr7
- ; DAGISEL-GFX10-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr6
- ; DAGISEL-GFX10-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr5
- ; DAGISEL-GFX10-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr4
- ; DAGISEL-GFX10-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr3
- ; DAGISEL-GFX10-NEXT: [[COPY13:%[0-9]+]]:sgpr_32 = COPY $sgpr2
- ; DAGISEL-GFX10-NEXT: [[COPY14:%[0-9]+]]:sgpr_32 = COPY $sgpr1
- ; DAGISEL-GFX10-NEXT: [[COPY15:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX10-NEXT: [[V_PK_ADD_U16_:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY12]], 8, [[COPY4]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[V_PK_ADD_U16_1:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY13]], 8, [[COPY5]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[V_PK_ADD_U16_2:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY14]], 8, [[COPY6]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[V_PK_ADD_U16_3:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY15]], 8, [[COPY7]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[V_PK_ADD_U16_4:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY8]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[V_PK_ADD_U16_5:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY9]], 8, [[COPY1]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[V_PK_ADD_U16_6:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY10]], 8, [[COPY2]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[V_PK_ADD_U16_7:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY11]], 8, [[COPY3]], 0, 0, 0, 0, 0, implicit $exec
- ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_PK_ADD_U16_7]], %subreg.sub0, [[V_PK_ADD_U16_6]], %subreg.sub1, [[V_PK_ADD_U16_5]], %subreg.sub2, [[V_PK_ADD_U16_4]], %subreg.sub3
- ; DAGISEL-GFX10-NEXT: [[DEF4:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY16:%[0-9]+]]:vreg_64 = COPY [[DEF4]]
- ; DAGISEL-GFX10-NEXT: [[COPY17:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
- ; DAGISEL-GFX10-NEXT: FLAT_STORE_DWORDX4 [[COPY16]], killed [[COPY17]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison` + 16)
- ; DAGISEL-GFX10-NEXT: [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF7:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF8:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_PK_ADD_U16_3]], %subreg.sub0, [[V_PK_ADD_U16_2]], %subreg.sub1, [[V_PK_ADD_U16_1]], %subreg.sub2, [[V_PK_ADD_U16_]], %subreg.sub3
- ; DAGISEL-GFX10-NEXT: [[DEF9:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY18:%[0-9]+]]:vreg_64 = COPY [[DEF9]]
- ; DAGISEL-GFX10-NEXT: [[COPY19:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE1]]
- ; DAGISEL-GFX10-NEXT: FLAT_STORE_DWORDX4 [[COPY18]], killed [[COPY19]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison`, align 32)
- ; DAGISEL-GFX10-NEXT: S_ENDPGM 0
+ ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc_v16i16
+ ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF32-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15
+ ; DAGISEL-GFX11-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr15
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr14
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr13
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr12
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr7
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr6
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr5
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY13:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY14:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY15:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_PK_ADD_U16_:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY12]], 8, [[COPY4]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_PK_ADD_U16_1:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY13]], 8, [[COPY5]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_PK_ADD_U16_2:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY14]], 8, [[COPY6]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_PK_ADD_U16_3:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY15]], 8, [[COPY7]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_PK_ADD_U16_4:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY8]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_PK_ADD_U16_5:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY9]], 8, [[COPY1]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_PK_ADD_U16_6:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY10]], 8, [[COPY2]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_PK_ADD_U16_7:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY11]], 8, [[COPY3]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_PK_ADD_U16_7]], %subreg.sub0, [[V_PK_ADD_U16_6]], %subreg.sub1, [[V_PK_ADD_U16_5]], %subreg.sub2, [[V_PK_ADD_U16_4]], %subreg.sub3
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF4:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY16:%[0-9]+]]:vreg_64 = COPY [[DEF4]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY17:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX11-WF32-NEXT: FLAT_STORE_DWORDX4 [[COPY16]], killed [[COPY17]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison` + 16)
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF7:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF8:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_PK_ADD_U16_3]], %subreg.sub0, [[V_PK_ADD_U16_2]], %subreg.sub1, [[V_PK_ADD_U16_1]], %subreg.sub2, [[V_PK_ADD_U16_]], %subreg.sub3
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF9:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY18:%[0-9]+]]:vreg_64 = COPY [[DEF9]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY19:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE1]]
+ ; DAGISEL-GFX11-WF32-NEXT: FLAT_STORE_DWORDX4 [[COPY18]], killed [[COPY19]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison`, align 32)
+ ; DAGISEL-GFX11-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc_v16i16
+ ; DAGISEL-GFX11-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF64-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15
+ ; DAGISEL-GFX11-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr15
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr14
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr13
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr12
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr7
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr6
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr5
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY13:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY14:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY15:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_PK_ADD_U16_:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY12]], 8, [[COPY4]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_PK_ADD_U16_1:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY13]], 8, [[COPY5]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_PK_ADD_U16_2:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY14]], 8, [[COPY6]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_PK_ADD_U16_3:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY15]], 8, [[COPY7]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_PK_ADD_U16_4:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY8]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_PK_ADD_U16_5:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY9]], 8, [[COPY1]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_PK_ADD_U16_6:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY10]], 8, [[COPY2]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_PK_ADD_U16_7:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY11]], 8, [[COPY3]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_PK_ADD_U16_7]], %subreg.sub0, [[V_PK_ADD_U16_6]], %subreg.sub1, [[V_PK_ADD_U16_5]], %subreg.sub2, [[V_PK_ADD_U16_4]], %subreg.sub3
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF4:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY16:%[0-9]+]]:vreg_64 = COPY [[DEF4]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY17:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX11-WF64-NEXT: FLAT_STORE_DWORDX4 [[COPY16]], killed [[COPY17]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison` + 16)
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF7:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF8:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_PK_ADD_U16_3]], %subreg.sub0, [[V_PK_ADD_U16_2]], %subreg.sub1, [[V_PK_ADD_U16_1]], %subreg.sub2, [[V_PK_ADD_U16_]], %subreg.sub3
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF9:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY18:%[0-9]+]]:vreg_64 = COPY [[DEF9]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY19:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE1]]
+ ; DAGISEL-GFX11-WF64-NEXT: FLAT_STORE_DWORDX4 [[COPY18]], killed [[COPY19]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison`, align 32)
+ ; DAGISEL-GFX11-WF64-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_preserve_cc_v16i16
+ ; DAGISEL-GFX10-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF32-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15
+ ; DAGISEL-GFX10-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr15
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr14
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr13
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr12
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr7
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr6
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr5
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY13:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY14:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY15:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_PK_ADD_U16_:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY12]], 8, [[COPY4]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_PK_ADD_U16_1:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY13]], 8, [[COPY5]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_PK_ADD_U16_2:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY14]], 8, [[COPY6]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_PK_ADD_U16_3:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY15]], 8, [[COPY7]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_PK_ADD_U16_4:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY8]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_PK_ADD_U16_5:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY9]], 8, [[COPY1]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_PK_ADD_U16_6:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY10]], 8, [[COPY2]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[V_PK_ADD_U16_7:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY11]], 8, [[COPY3]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_PK_ADD_U16_7]], %subreg.sub0, [[V_PK_ADD_U16_6]], %subreg.sub1, [[V_PK_ADD_U16_5]], %subreg.sub2, [[V_PK_ADD_U16_4]], %subreg.sub3
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF4:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY16:%[0-9]+]]:vreg_64 = COPY [[DEF4]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY17:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX10-WF32-NEXT: FLAT_STORE_DWORDX4 [[COPY16]], killed [[COPY17]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison` + 16)
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF7:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF8:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_PK_ADD_U16_3]], %subreg.sub0, [[V_PK_ADD_U16_2]], %subreg.sub1, [[V_PK_ADD_U16_1]], %subreg.sub2, [[V_PK_ADD_U16_]], %subreg.sub3
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF9:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY18:%[0-9]+]]:vreg_64 = COPY [[DEF9]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY19:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE1]]
+ ; DAGISEL-GFX10-WF32-NEXT: FLAT_STORE_DWORDX4 [[COPY18]], killed [[COPY19]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison`, align 32)
+ ; DAGISEL-GFX10-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF64-LABEL: name: amdgpu_cs_chain_preserve_cc_v16i16
+ ; DAGISEL-GFX10-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF64-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15
+ ; DAGISEL-GFX10-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr15
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr14
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr13
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr12
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr11
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr10
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY8:%[0-9]+]]:sgpr_32 = COPY $sgpr7
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr6
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr5
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY13:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY14:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY15:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_PK_ADD_U16_:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY12]], 8, [[COPY4]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_PK_ADD_U16_1:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY13]], 8, [[COPY5]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_PK_ADD_U16_2:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY14]], 8, [[COPY6]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_PK_ADD_U16_3:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY15]], 8, [[COPY7]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_PK_ADD_U16_4:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY8]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_PK_ADD_U16_5:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY9]], 8, [[COPY1]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_PK_ADD_U16_6:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY10]], 8, [[COPY2]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[V_PK_ADD_U16_7:%[0-9]+]]:vgpr_32 = V_PK_ADD_U16 8, [[COPY11]], 8, [[COPY3]], 0, 0, 0, 0, 0, implicit $exec
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_PK_ADD_U16_7]], %subreg.sub0, [[V_PK_ADD_U16_6]], %subreg.sub1, [[V_PK_ADD_U16_5]], %subreg.sub2, [[V_PK_ADD_U16_4]], %subreg.sub3
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF4:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY16:%[0-9]+]]:vreg_64 = COPY [[DEF4]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY17:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX10-WF64-NEXT: FLAT_STORE_DWORDX4 [[COPY16]], killed [[COPY17]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison` + 16)
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF7:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF8:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[V_PK_ADD_U16_3]], %subreg.sub0, [[V_PK_ADD_U16_2]], %subreg.sub1, [[V_PK_ADD_U16_1]], %subreg.sub2, [[V_PK_ADD_U16_]], %subreg.sub3
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF9:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY18:%[0-9]+]]:vreg_64 = COPY [[DEF9]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY19:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE1]]
+ ; DAGISEL-GFX10-WF64-NEXT: FLAT_STORE_DWORDX4 [[COPY18]], killed [[COPY19]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr poison`, align 32)
+ ; DAGISEL-GFX10-WF64-NEXT: S_ENDPGM 0
%c = add <16 x i16> %a, %b
store <16 x i16> %c, ptr poison
ret void
@@ -922,45 +1394,85 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_many_regs(<36 x i
; GISEL-GFX10-NEXT: GLOBAL_STORE_DWORDX2 [[COPY5]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (store (<2 x s32>) into `ptr addrspace(1) poison`, addrspace 1)
; GISEL-GFX10-NEXT: S_ENDPGM 0
;
- ; DAGISEL-GFX11-LABEL: name: amdgpu_cs_chain_preserve_many_regs
- ; DAGISEL-GFX11: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-NEXT: liveins: $sgpr35, $vgpr8, $vgpr135
- ; DAGISEL-GFX11-NEXT: {{ $}}
- ; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr135
- ; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr35
- ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
- ; DAGISEL-GFX11-NEXT: GLOBAL_STORE_DWORD [[COPY3]], [[COPY4]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison`, addrspace 1)
- ; DAGISEL-GFX11-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
- ; DAGISEL-GFX11-NEXT: [[DEF3:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[DEF3]]
- ; DAGISEL-GFX11-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
- ; DAGISEL-GFX11-NEXT: GLOBAL_STORE_DWORDX2 [[COPY5]], killed [[COPY6]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, addrspace 1)
- ; DAGISEL-GFX11-NEXT: S_ENDPGM 0
- ;
- ; DAGISEL-GFX10-LABEL: name: amdgpu_cs_chain_preserve_many_regs
- ; DAGISEL-GFX10: bb.0 (%ir-block.0):
- ; DAGISEL-GFX10-NEXT: liveins: $sgpr35, $vgpr8, $vgpr135
- ; DAGISEL-GFX10-NEXT: {{ $}}
- ; DAGISEL-GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr135
- ; DAGISEL-GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX10-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr35
- ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
- ; DAGISEL-GFX10-NEXT: GLOBAL_STORE_DWORD [[COPY3]], [[COPY4]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison`, addrspace 1)
- ; DAGISEL-GFX10-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
- ; DAGISEL-GFX10-NEXT: [[DEF3:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX10-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[DEF3]]
- ; DAGISEL-GFX10-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
- ; DAGISEL-GFX10-NEXT: GLOBAL_STORE_DWORDX2 [[COPY5]], killed [[COPY6]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, addrspace 1)
- ; DAGISEL-GFX10-NEXT: S_ENDPGM 0
+ ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_preserve_many_regs
+ ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF32-NEXT: liveins: $sgpr35, $vgpr8, $vgpr135
+ ; DAGISEL-GFX11-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr135
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr35
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+ ; DAGISEL-GFX11-WF32-NEXT: GLOBAL_STORE_DWORD [[COPY3]], [[COPY4]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF3:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[DEF3]]
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX11-WF32-NEXT: GLOBAL_STORE_DWORDX2 [[COPY5]], killed [[COPY6]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX11-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_preserve_many_regs
+ ; DAGISEL-GFX11-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF64-NEXT: liveins: $sgpr35, $vgpr8, $vgpr135
+ ; DAGISEL-GFX11-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr135
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr35
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+ ; DAGISEL-GFX11-WF64-NEXT: GLOBAL_STORE_DWORD [[COPY3]], [[COPY4]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF3:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[DEF3]]
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX11-WF64-NEXT: GLOBAL_STORE_DWORDX2 [[COPY5]], killed [[COPY6]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX11-WF64-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_preserve_many_regs
+ ; DAGISEL-GFX10-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF32-NEXT: liveins: $sgpr35, $vgpr8, $vgpr135
+ ; DAGISEL-GFX10-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr135
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr35
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+ ; DAGISEL-GFX10-WF32-NEXT: GLOBAL_STORE_DWORD [[COPY3]], [[COPY4]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+ ; DAGISEL-GFX10-WF32-NEXT: [[DEF3:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[DEF3]]
+ ; DAGISEL-GFX10-WF32-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX10-WF32-NEXT: GLOBAL_STORE_DWORDX2 [[COPY5]], killed [[COPY6]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX10-WF32-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX10-WF64-LABEL: name: amdgpu_cs_chain_preserve_many_regs
+ ; DAGISEL-GFX10-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX10-WF64-NEXT: liveins: $sgpr35, $vgpr8, $vgpr135
+ ; DAGISEL-GFX10-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr135
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr35
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+ ; DAGISEL-GFX10-WF64-NEXT: GLOBAL_STORE_DWORD [[COPY3]], [[COPY4]], 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+ ; DAGISEL-GFX10-WF64-NEXT: [[DEF3:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[DEF3]]
+ ; DAGISEL-GFX10-WF64-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+ ; DAGISEL-GFX10-WF64-NEXT: GLOBAL_STORE_DWORDX2 [[COPY5]], killed [[COPY6]], 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(1) poison`, addrspace 1)
+ ; DAGISEL-GFX10-WF64-NEXT: S_ENDPGM 0
%c = extractelement <36 x i32> %a, i32 35
store i32 %c, ptr addrspace(1) poison
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
index 04bf2120b78cfb..ea823f30f26c22 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
@@ -161,6 +161,7 @@ define i1 @snan_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: snan_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7fc0
; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
@@ -221,6 +222,7 @@ define i1 @qnan_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: qnan_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7fbf
; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
@@ -268,6 +270,7 @@ define i1 @posinf_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: posinf_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
@@ -311,6 +314,7 @@ define i1 @neginf_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: neginf_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7CHECK-NEXT: s_mov_b32 s4, 0xff80
; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
@@ -354,6 +358,7 @@ define i1 @posnormal_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: posnormal_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_ashrrev_i32_e32 v1, 16, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xffffff80, v0
@@ -418,6 +423,7 @@ define i1 @negnormal_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: negnormal_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_ashrrev_i32_e32 v1, 16, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xffffff80, v0
@@ -482,6 +488,7 @@ define i1 @possubnormal_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: possubnormal_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, -1, v0
; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -531,6 +538,7 @@ define i1 @negsubnormal_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: negsubnormal_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_ashrrev_i32_e32 v1, 16, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: v_add_i32_e64 v0, s[4:5], -1, v0
@@ -594,6 +602,7 @@ define i1 @poszero_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: poszero_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
@@ -634,6 +643,7 @@ define i1 @negzero_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: negzero_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7CHECK-NEXT: s_mov_b32 s4, 0x8000
; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
@@ -677,6 +687,7 @@ define i1 @posfinite_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: posfinite_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
@@ -720,6 +731,7 @@ define i1 @negfinite_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: negfinite_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_ashrrev_i32_e32 v1, 16, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
@@ -778,6 +790,7 @@ define i1 @isnan_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: isnan_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
@@ -825,6 +838,7 @@ define i1 @not_isnan_bf16(bfloat %x) {
; GFX7CHECK-LABEL: not_isnan_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f81
; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
@@ -872,6 +886,8 @@ define <2 x i1> @isnan_v2bf16(<2 x bfloat> %x) nounwind {
; GFX7CHECK-LABEL: isnan_v2bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7CHECK-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: v_bfe_u32 v1, v1, 16, 15
@@ -933,8 +949,11 @@ define <3 x i1> @isnan_v3bf16(<3 x bfloat> %x) nounwind {
; GFX7CHECK-LABEL: isnan_v3bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7CHECK-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
+; GFX7CHECK-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7CHECK-NEXT: v_bfe_u32 v1, v1, 16, 15
; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
; GFX7CHECK-NEXT: v_bfe_u32 v2, v2, 16, 15
@@ -1009,10 +1028,14 @@ define <4 x i1> @isnan_v4bf16(<4 x bfloat> %x) nounwind {
; GFX7CHECK-LABEL: isnan_v4bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7CHECK-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
+; GFX7CHECK-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7CHECK-NEXT: v_bfe_u32 v1, v1, 16, 15
; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
+; GFX7CHECK-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7CHECK-NEXT: v_bfe_u32 v2, v2, 16, 15
; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v1
@@ -1104,6 +1127,7 @@ define i1 @isinf_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: isinf_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
@@ -1151,6 +1175,7 @@ define i1 @isfinite_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: isfinite_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
@@ -1198,6 +1223,7 @@ define i1 @issubnormal_or_zero_bf16(bfloat %x) {
; GFX7CHECK-LABEL: issubnormal_or_zero_bf16:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7f80, v0
; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1244,6 +1270,7 @@ define i1 @not_issubnormal_or_zero_bf16(bfloat %x) {
; GFX7CHECK-LABEL: not_issubnormal_or_zero_bf16:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0x7f80, v0
; GFX7CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
@@ -1290,6 +1317,7 @@ define i1 @isnormal_bf16(bfloat %x) {
; GFX7CHECK-LABEL: isnormal_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xffffff80, v0
; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -1343,6 +1371,7 @@ define i1 @not_isnormal_bf16(bfloat %x) {
; GFX7CHECK-LABEL: not_isnormal_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xffffff80, v0
; GFX7CHECK-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -1396,6 +1425,7 @@ define i1 @not_is_plus_normal_bf16(bfloat %x) {
; GFX7CHECK-LABEL: not_is_plus_normal_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_ashrrev_i32_e32 v1, 16, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xffffff80, v0
@@ -1460,6 +1490,7 @@ define i1 @not_is_neg_normal_bf16(bfloat %x) {
; GFX7CHECK-LABEL: not_is_neg_normal_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_ashrrev_i32_e32 v1, 16, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, 0xffffff80, v0
@@ -1524,6 +1555,7 @@ define i1 @issubnormal_bf16(bfloat %x) {
; GFX7CHECK-LABEL: issubnormal_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, -1, v0
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f
@@ -1576,6 +1608,7 @@ define i1 @not_issubnormal_bf16(bfloat %x) {
; GFX7CHECK-LABEL: not_issubnormal_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: v_add_i32_e32 v0, vcc, -1, v0
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7e
@@ -1628,6 +1661,7 @@ define i1 @iszero_bf16(bfloat %x) {
; GFX7CHECK-LABEL: iszero_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
@@ -1672,6 +1706,7 @@ define i1 @not_iszero_bf16(bfloat %x) {
; GFX7CHECK-LABEL: not_iszero_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
@@ -1716,6 +1751,7 @@ define i1 @ispositive_bf16(bfloat %x) {
; GFX7CHECK-LABEL: ispositive_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f81
; GFX7CHECK-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0
@@ -1759,6 +1795,7 @@ define i1 @not_ispositive_bf16(bfloat %x) {
; GFX7CHECK-LABEL: not_ispositive_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX7CHECK-NEXT: v_ashrrev_i32_e32 v2, 16, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
@@ -1841,6 +1878,7 @@ define i1 @isnegative_bf16(bfloat %x) {
; GFX7CHECK-LABEL: isnegative_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX7CHECK-NEXT: v_ashrrev_i32_e32 v2, 16, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
@@ -1913,6 +1951,7 @@ define i1 @not_isnegative_bf16(bfloat %x) {
; GFX7CHECK-LABEL: not_isnegative_bf16:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
@@ -1974,6 +2013,7 @@ define i1 @iszero_or_nan_bf16(bfloat %x) {
; GFX7CHECK-LABEL: iszero_or_nan_bf16:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
@@ -2032,6 +2072,7 @@ define i1 @iszero_or_nan_f_daz(bfloat %x) #0 {
; GFX7CHECK-LABEL: iszero_or_nan_f_daz:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
@@ -2090,6 +2131,7 @@ define i1 @iszero_or_nan_f_maybe_daz(bfloat %x) #1 {
; GFX7CHECK-LABEL: iszero_or_nan_f_maybe_daz:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
@@ -2148,6 +2190,7 @@ define i1 @not_iszero_or_nan_bf16(bfloat %x) {
; GFX7CHECK-LABEL: not_iszero_or_nan_bf16:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f81
; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
@@ -2206,6 +2249,7 @@ define i1 @not_iszero_or_nan_f_daz(bfloat %x) #0 {
; GFX7CHECK-LABEL: not_iszero_or_nan_f_daz:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f81
; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
@@ -2264,6 +2308,7 @@ define i1 @not_iszero_or_nan_f_maybe_daz(bfloat %x) #1 {
; GFX7CHECK-LABEL: not_iszero_or_nan_f_maybe_daz:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f81
; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
@@ -2322,6 +2367,7 @@ define i1 @iszero_or_qnan_bf16(bfloat %x) {
; GFX7CHECK-LABEL: iszero_or_qnan_bf16:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7fbf
; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
@@ -2380,6 +2426,7 @@ define i1 @iszero_or_snan_bf16(bfloat %x) {
; GFX7CHECK-LABEL: iszero_or_snan_bf16:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7fc0
; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
@@ -2451,6 +2498,7 @@ define i1 @not_iszero_or_qnan_bf16(bfloat %x) {
; GFX7CHECK-LABEL: not_iszero_or_qnan_bf16:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7fc0
; GFX7CHECK-NEXT: s_movk_i32 s8, 0x7f80
@@ -2559,6 +2607,7 @@ define i1 @not_iszero_or_snan_bf16(bfloat %x) {
; GFX7CHECK-LABEL: not_iszero_or_snan_bf16:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
@@ -2657,6 +2706,7 @@ define i1 @isinf_or_nan_bf16(bfloat %x) {
; GFX7CHECK-LABEL: isinf_or_nan_bf16:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f7f
; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
@@ -2705,6 +2755,7 @@ define i1 @not_isinf_or_nan_bf16(bfloat %x) {
; GFX7CHECK-LABEL: not_isinf_or_nan_bf16:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
@@ -2753,6 +2804,7 @@ define i1 @isfinite_or_nan_f(bfloat %x) {
; GFX7CHECK-LABEL: isfinite_or_nan_f:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_cmp_ne_u32_e32 vcc, s4, v0
@@ -2801,6 +2853,7 @@ define i1 @not_isfinite_or_nan_f(bfloat %x) {
; GFX7CHECK-LABEL: not_isfinite_or_nan_f:
; GFX7CHECK: ; %bb.0: ; %entry
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7CHECK-NEXT: v_mul_f32_e32 v0, 1.0, v0
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll b/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
index 3be4665cf3a005..e906b5327c3623 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
@@ -1411,6 +1411,13 @@ define bfloat @lds_atomic_fadd_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; VI-NEXT: v_mov_b32_e32 v4, v3
; VI-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f32_e32 v3, 4.0, v3
+; VI-NEXT: v_bfe_u32 v6, v3, 16, 1
+; VI-NEXT: v_add_u32_e32 v6, vcc, v6, v3
+; VI-NEXT: v_and_b32_e32 v7, 0x80000000, v3
+; VI-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; VI-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; VI-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; VI-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc
; VI-NEXT: v_and_b32_e32 v5, v4, v2
; VI-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; VI-NEXT: v_or_b32_e32 v3, v5, v3
@@ -1429,30 +1436,37 @@ define bfloat @lds_atomic_fadd_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v1, -4, v0
-; GFX9-NEXT: ds_read_b32 v2, v1
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v0
+; GFX9-NEXT: ds_read_b32 v3, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX9-NEXT: s_mov_b32 s4, 0xffff
-; GFX9-NEXT: v_and_b32_e32 v0, 24, v3
-; GFX9-NEXT: v_lshlrev_b32_e64 v3, v3, s4
-; GFX9-NEXT: v_not_b32_e32 v3, v3
+; GFX9-NEXT: v_and_b32_e32 v0, 24, v2
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, v2, s4
+; GFX9-NEXT: v_not_b32_e32 v2, v2
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_movk_i32 s6, 0x7fff
; GFX9-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, v2
-; GFX9-NEXT: v_lshrrev_b32_sdwa v2, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_add_f32_e32 v2, 4.0, v2
-; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_or_b32 v2, v4, v3, v2
-; GFX9-NEXT: ds_cmpst_rtn_b32 v2, v1, v4, v2
+; GFX9-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_f32_e32 v3, 4.0, v3
+; GFX9-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v3, s6
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX9-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB10_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, v0, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, v0, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: lds_atomic_fadd_ret_bf16:
@@ -1544,6 +1558,13 @@ define void @lds_atomic_fadd_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f32_e32 v4, 4.0, v4
+; VI-NEXT: v_bfe_u32 v6, v4, 16, 1
+; VI-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; VI-NEXT: v_and_b32_e32 v7, 0x80000000, v4
+; VI-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; VI-NEXT: v_or_b32_e32 v7, 0x400000, v7
+; VI-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; VI-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc
; VI-NEXT: v_and_b32_e32 v5, v3, v2
; VI-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; VI-NEXT: v_or_b32_e32 v4, v5, v4
@@ -1569,11 +1590,18 @@ define void @lds_atomic_fadd_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; GFX9-NEXT: v_lshlrev_b32_e64 v2, v2, s4
; GFX9-NEXT: v_not_b32_e32 v2, v2
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_movk_i32 s6, 0x7fff
; GFX9-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_f32_e32 v4, 4.0, v4
+; GFX9-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v5, v5, v4, s6
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
; GFX9-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_and_or_b32 v4, v3, v2, v4
; GFX9-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index d76bb48b4a82a3..590b40960faab9 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -4237,57 +4237,107 @@ define <6 x bfloat> @shuffle_v6bf16_452367(ptr addrspace(1) %arg0, ptr addrspace
define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonly %A, ptr addrspace(1) nocapture readonly %B, ptr addrspace(1) nocapture %C) {
; GFX9-LABEL: fma_shuffle_v2bf16:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX9-NEXT: v_lshlrev_b32_e32 v6, 3, v0
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10
+; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX9-NEXT: s_movk_i32 s2, 0x7fff
+; GFX9-NEXT: s_mov_b32 s3, 0x7060302
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[0:1], v6, s[6:7]
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
-; GFX9-NEXT: global_load_dwordx2 v[4:5], v6, s[2:3]
-; GFX9-NEXT: s_mov_b32 s0, 0x7060302
+; GFX9-NEXT: global_load_dwordx2 v[1:2], v0, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[3:4], v0, s[8:9]
+; GFX9-NEXT: global_load_dwordx2 v[5:6], v0, s[10:11]
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
+; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v3
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
-; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v3
+; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_fma_f32 v7, v8, v9, v7
-; GFX9-NEXT: v_fma_f32 v0, v8, v4, v0
+; GFX9-NEXT: v_fma_f32 v1, v8, v5, v1
; GFX9-NEXT: v_fma_f32 v8, v12, v9, v11
-; GFX9-NEXT: v_fma_f32 v1, v12, v4, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX9-NEXT: v_fma_f32 v2, v12, v5, v2
+; GFX9-NEXT: v_bfe_u32 v5, v7, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v9, 0x80000000, v7
+; GFX9-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v12, 0x80000000, v1
+; GFX9-NEXT: v_add3_u32 v5, v5, v7, s2
+; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v9
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
+; GFX9-NEXT: v_bfe_u32 v13, v8, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v14, 0x80000000, v8
+; GFX9-NEXT: v_add3_u32 v11, v11, v1, s2
+; GFX9-NEXT: v_or_b32_e32 v12, 0x400000, v12
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_bfe_u32 v15, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v16, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v13, v13, v8, s2
+; GFX9-NEXT: v_or_b32_e32 v14, 0x400000, v14
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v11, v12, vcc
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
+; GFX9-NEXT: v_add3_u32 v15, v15, v2, s2
+; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v16
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v15, v16, vcc
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v8
-; GFX9-NEXT: v_fma_f32 v0, v2, v10, v0
-; GFX9-NEXT: v_fma_f32 v2, v2, v5, v4
+; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX9-NEXT: v_fma_f32 v1, v3, v10, v1
-; GFX9-NEXT: v_fma_f32 v3, v3, v5, v7
-; GFX9-NEXT: v_perm_b32 v1, v3, v1, s0
-; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0
-; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[6:7]
+; GFX9-NEXT: v_fma_f32 v3, v3, v6, v5
+; GFX9-NEXT: v_fma_f32 v2, v4, v10, v2
+; GFX9-NEXT: v_fma_f32 v4, v4, v6, v7
+; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v6, 0x80000000, v1
+; GFX9-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v8, 0x80000000, v3
+; GFX9-NEXT: v_add3_u32 v5, v5, v1, s2
+; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v6
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: v_bfe_u32 v9, v2, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v10, 0x80000000, v2
+; GFX9-NEXT: v_add3_u32 v7, v7, v3, s2
+; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT: v_bfe_u32 v11, v4, 16, 1
+; GFX9-NEXT: v_and_b32_e32 v12, 0x80000000, v4
+; GFX9-NEXT: v_add3_u32 v9, v9, v2, s2
+; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT: v_add3_u32 v11, v11, v4, s2
+; GFX9-NEXT: v_or_b32_e32 v12, 0x400000, v12
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc
+; GFX9-NEXT: v_perm_b32 v2, v4, v2, s3
+; GFX9-NEXT: v_perm_b32 v1, v3, v1, s3
+; GFX9-NEXT: global_store_dwordx2 v0, v[1:2], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: fma_shuffle_v2bf16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10
+; GFX10-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x0
; GFX10-NEXT: v_lshlrev_b32_e32 v6, 3, v0
+; GFX10-NEXT: s_brev_b32 s2, 1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: global_load_dwordx2 v[0:1], v6, s[6:7]
-; GFX10-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
-; GFX10-NEXT: global_load_dwordx2 v[4:5], v6, s[2:3]
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1]
+; GFX10-NEXT: global_load_dwordx2 v[2:3], v6, s[8:9]
+; GFX10-NEXT: global_load_dwordx2 v[4:5], v6, s[10:11]
; GFX10-NEXT: s_waitcnt vmcnt(2)
; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
@@ -4296,73 +4346,162 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_fmac_f32_e32 v7, v8, v9
; GFX10-NEXT: v_fmac_f32_e32 v0, v8, v4
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_fmac_f32_e32 v11, v10, v4
-; GFX10-NEXT: v_fmac_f32_e32 v1, v10, v9
-; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX10-NEXT: v_fmac_f32_e32 v11, v12, v9
+; GFX10-NEXT: v_fmac_f32_e32 v1, v12, v4
+; GFX10-NEXT: v_bfe_u32 v4, v7, 16, 1
+; GFX10-NEXT: v_and_or_b32 v8, v7, s2, 0x400000
+; GFX10-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX10-NEXT: v_and_or_b32 v12, v0, s2, 0x400000
+; GFX10-NEXT: v_add3_u32 v4, v4, v7, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v15, v1, 16, 1
+; GFX10-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX10-NEXT: v_and_or_b32 v16, v1, s2, 0x400000
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v15, v15, v1, 0x7fff
+; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX10-NEXT: v_and_or_b32 v14, v11, s2, 0x400000
+; GFX10-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v11
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v15, v16, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX10-NEXT: v_fmac_f32_e32 v4, v2, v5
+; GFX10-NEXT: v_fmac_f32_e32 v0, v2, v10
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX10-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v8, v4, s2, 0x400000
+; GFX10-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX10-NEXT: v_fmac_f32_e32 v1, v3, v10
; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX10-NEXT: v_fmac_f32_e32 v0, v2, v12
-; GFX10-NEXT: v_fmac_f32_e32 v4, v3, v12
-; GFX10-NEXT: v_fmac_f32_e32 v1, v3, v5
-; GFX10-NEXT: v_fmac_f32_e32 v7, v2, v5
-; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
-; GFX10-NEXT: v_perm_b32 v0, v7, v0, 0x7060302
-; GFX10-NEXT: global_store_dwordx2 v6, v[0:1], s[6:7]
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX10-NEXT: v_fmac_f32_e32 v7, v3, v5
+; GFX10-NEXT: v_and_or_b32 v3, v0, s2, 0x400000
+; GFX10-NEXT: v_and_or_b32 v10, v1, s2, 0x400000
+; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX10-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v11, v7, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_and_or_b32 v12, v7, s2, 0x400000
+; GFX10-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX10-NEXT: v_add3_u32 v11, v11, v7, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_perm_b32 v1, v2, v1, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v8, vcc_lo
+; GFX10-NEXT: v_perm_b32 v0, v3, v0, 0x7060302
+; GFX10-NEXT: global_store_dwordx2 v6, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: fma_shuffle_v2bf16:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x10
-; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
+; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x10
+; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x0
; GFX11-NEXT: v_lshlrev_b32_e32 v6, 3, v0
+; GFX11-NEXT: s_brev_b32 s0, 1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: global_load_b64 v[0:1], v6, s[4:5]
-; GFX11-NEXT: global_load_b64 v[2:3], v6, s[0:1]
-; GFX11-NEXT: global_load_b64 v[4:5], v6, s[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v1
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
+; GFX11-NEXT: global_load_b64 v[0:1], v6, s[2:3]
+; GFX11-NEXT: global_load_b64 v[2:3], v6, s[4:5]
+; GFX11-NEXT: global_load_b64 v[4:5], v6, s[6:7]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
+; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v3
; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_fmac_f32_e32 v1, v10, v9
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_fmac_f32 v1, v3, v5 :: v_dual_lshlrev_b32 v4, 16, v4
-; GFX11-NEXT: v_dual_fmac_f32 v11, v10, v4 :: v_dual_lshlrev_b32 v8, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_fmac_f32 v11, v12, v9 :: v_dual_and_b32 v2, 0xffff0000, v2
+; GFX11-NEXT: v_fmac_f32_e32 v1, v12, v4
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX11-NEXT: v_and_or_b32 v14, v11, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v15, v1, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_fmac_f32_e32 v7, v8, v9
+; GFX11-NEXT: v_and_or_b32 v16, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_fmac_f32 v7, v8, v9 :: v_dual_and_b32 v2, 0xffff0000, v2
+; GFX11-NEXT: v_add3_u32 v15, v15, v1, 0x7fff
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_fmac_f32_e32 v0, v8, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v11
+; GFX11-NEXT: v_bfe_u32 v4, v7, 16, 1
+; GFX11-NEXT: v_and_or_b32 v8, v7, s0, 0x400000
+; GFX11-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add3_u32 v4, v4, v7, 0x7fff
+; GFX11-NEXT: v_and_or_b32 v12, v0, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_dual_fmac_f32 v4, v2, v5 :: v_dual_cndmask_b32 v1, v15, v16
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_and_or_b32 v8, v4, s0, 0x400000
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_fmac_f32_e32 v1, v3, v10
; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-NEXT: v_dual_fmac_f32 v4, v3, v12 :: v_dual_fmac_f32 v7, v2, v5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_fmac_f32_e32 v0, v2, v12
-; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v0, v7, v0, 0x7060302
-; GFX11-NEXT: global_store_b64 v6, v[0:1], s[4:5]
+; GFX11-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-NEXT: v_fmac_f32_e32 v7, v3, v5
+; GFX11-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: v_bfe_u32 v11, v7, 16, 1
+; GFX11-NEXT: v_and_or_b32 v12, v7, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_fmac_f32_e32 v0, v2, v10
+; GFX11-NEXT: v_and_or_b32 v10, v1, s0, 0x400000
+; GFX11-NEXT: v_add3_u32 v11, v11, v7, 0x7fff
+; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT: v_and_or_b32 v3, v0, s0, 0x400000
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v1, v2, v1, 0x7060302
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v5, v8, vcc_lo
+; GFX11-NEXT: v_perm_b32 v0, v3, v0, 0x7060302
+; GFX11-NEXT: global_store_b64 v6, v[0:1], s[2:3]
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
index a9faa130d6379f..884860712632d7 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | FileCheck --check-prefixes=CHECK,SM80 %s
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | FileCheck --check-prefixes=CHECK,SM80 %s
; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck --check-prefixes=CHECK,SM90 %s
; RUN: %if ptxas-11.8 %{ llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | %ptxas-verify -arch=sm_80 %}
; RUN: %if ptxas-11.8 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}
More information about the llvm-commits
mailing list