[llvm] [NVPTX] fold movs into loads and stores (PR #144581)
Princeton Ferro via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 18 10:25:16 PDT 2025
https://github.com/Prince781 updated https://github.com/llvm/llvm-project/pull/144581
>From 1ccea451ffa2258751a3f12cb449b63ec74b35cd Mon Sep 17 00:00:00 2001
From: Princeton Ferro <pferro at nvidia.com>
Date: Thu, 12 Jun 2025 13:45:27 -0400
Subject: [PATCH] [NVPTX] fold movs into loads and stores
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 285 +++++-
llvm/test/CodeGen/NVPTX/bf16-instructions.ll | 517 ++++++-----
.../NVPTX/bf16x2-instructions-approx.ll | 34 +-
.../test/CodeGen/NVPTX/bf16x2-instructions.ll | 388 ++++-----
llvm/test/CodeGen/NVPTX/f16x2-instructions.ll | 104 +--
llvm/test/CodeGen/NVPTX/fexp2.ll | 111 ++-
llvm/test/CodeGen/NVPTX/flog2.ll | 66 +-
llvm/test/CodeGen/NVPTX/fma-relu-contract.ll | 532 ++++++------
.../CodeGen/NVPTX/fma-relu-fma-intrinsic.ll | 403 +++++----
.../NVPTX/fma-relu-instruction-flag.ll | 822 +++++++++---------
llvm/test/CodeGen/NVPTX/i16x2-instructions.ll | 185 ++--
llvm/test/CodeGen/NVPTX/i8x2-instructions.ll | 4 +-
llvm/test/CodeGen/NVPTX/i8x4-instructions.ll | 48 +-
llvm/test/CodeGen/NVPTX/ldg-invariant.ll | 13 +-
llvm/test/CodeGen/NVPTX/load-store-vectors.ll | 56 +-
.../NVPTX/load-with-non-coherent-cache.ll | 654 ++++++++++++--
llvm/test/CodeGen/NVPTX/math-intrins.ll | 356 ++++----
llvm/test/CodeGen/NVPTX/param-load-store.ll | 8 +-
.../CodeGen/NVPTX/reduction-intrinsics.ll | 288 +++---
llvm/test/CodeGen/NVPTX/sext-setcc.ll | 7 +-
llvm/test/CodeGen/NVPTX/shift-opt.ll | 15 +-
...unfold-masked-merge-vector-variablemask.ll | 74 +-
llvm/test/CodeGen/NVPTX/vector-loads.ll | 315 +++++--
23 files changed, 2970 insertions(+), 2315 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 492f4ab76fdbb..0db9391732a5a 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -238,18 +238,11 @@ getVectorLoweringShape(EVT VectorEVT, bool CanLowerTo256Bit) {
return std::nullopt;
LLVM_FALLTHROUGH;
case MVT::v2i8:
- case MVT::v2i16:
case MVT::v2i32:
case MVT::v2i64:
- case MVT::v2f16:
- case MVT::v2bf16:
case MVT::v2f32:
case MVT::v2f64:
- case MVT::v4i8:
- case MVT::v4i16:
case MVT::v4i32:
- case MVT::v4f16:
- case MVT::v4bf16:
case MVT::v4f32:
// This is a "native" vector type
return std::pair(NumElts, EltVT);
@@ -262,6 +255,13 @@ getVectorLoweringShape(EVT VectorEVT, bool CanLowerTo256Bit) {
if (!CanLowerTo256Bit)
return std::nullopt;
LLVM_FALLTHROUGH;
+ case MVT::v2i16: // <1 x i16x2>
+ case MVT::v2f16: // <1 x f16x2>
+ case MVT::v2bf16: // <1 x bf16x2>
+ case MVT::v4i8: // <1 x i8x4>
+ case MVT::v4i16: // <2 x i16x2>
+ case MVT::v4f16: // <2 x f16x2>
+ case MVT::v4bf16: // <2 x bf16x2>
case MVT::v8i8: // <2 x i8x4>
case MVT::v8f16: // <4 x f16x2>
case MVT::v8bf16: // <4 x bf16x2>
@@ -845,7 +845,8 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
// We have some custom DAG combine patterns for these nodes
setTargetDAGCombine({ISD::ADD, ISD::AND, ISD::EXTRACT_VECTOR_ELT, ISD::FADD,
ISD::MUL, ISD::SHL, ISD::SREM, ISD::UREM, ISD::VSELECT,
- ISD::BUILD_VECTOR, ISD::ADDRSPACECAST});
+ ISD::BUILD_VECTOR, ISD::ADDRSPACECAST, ISD::LOAD,
+ ISD::STORE});
// setcc for f16x2 and bf16x2 needs special handling to prevent
// legalizer's attempt to scalarize it due to v2i1 not being legal.
@@ -3464,19 +3465,16 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
unsigned I = 0;
for (const unsigned NumElts : VectorInfo) {
const EVT EltVT = VTs[I];
- const EVT LoadVT = [&]() -> EVT {
- // i1 is loaded/stored as i8.
- if (EltVT == MVT::i1)
- return MVT::i8;
- // getLoad needs a vector type, but it can't handle
- // vectors which contain v2f16 or v2bf16 elements. So we must load
- // using i32 here and then bitcast back.
- if (EltVT.isVector())
- return MVT::getIntegerVT(EltVT.getFixedSizeInBits());
- return EltVT;
- }();
+ // i1 is loaded/stored as i8
+ const EVT LoadVT = EltVT == MVT::i1 ? MVT::i8 : EltVT;
+ // If the element is a packed type (ex. v2f16, v4i8, etc) holding
+ // multiple elements.
+ const unsigned PackingAmt =
+ LoadVT.isVector() ? LoadVT.getVectorNumElements() : 1;
+
+ const EVT VecVT = EVT::getVectorVT(
+ F->getContext(), LoadVT.getScalarType(), NumElts * PackingAmt);
- const EVT VecVT = EVT::getVectorVT(F->getContext(), LoadVT, NumElts);
SDValue VecAddr = DAG.getObjectPtrOffset(
dl, ArgSymbol, TypeSize::getFixed(Offsets[I]));
@@ -3496,8 +3494,10 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
if (P.getNode())
P.getNode()->setIROrder(Arg.getArgNo() + 1);
for (const unsigned J : llvm::seq(NumElts)) {
- SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, LoadVT, P,
- DAG.getIntPtrConstant(J, dl));
+ SDValue Elt = DAG.getNode(LoadVT.isVector() ? ISD::EXTRACT_SUBVECTOR
+ : ISD::EXTRACT_VECTOR_ELT,
+ dl, LoadVT, P,
+ DAG.getIntPtrConstant(J * PackingAmt, dl));
// Extend or truncate the element if necessary (e.g. an i8 is loaded
// into an i16 register)
@@ -3511,9 +3511,6 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
Elt);
} else if (ExpactedVT.bitsLT(Elt.getValueType())) {
Elt = DAG.getNode(ISD::TRUNCATE, dl, ExpactedVT, Elt);
- } else {
- // v2f16 was loaded as an i32. Now we must bitcast it back.
- Elt = DAG.getBitcast(EltVT, Elt);
}
InVals.push_back(Elt);
}
@@ -5047,26 +5044,237 @@ PerformFADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
return SDValue();
}
-static SDValue PerformStoreCombineHelper(SDNode *N, std::size_t Front,
- std::size_t Back) {
+/// Combine extractelts into a load by increasing the number of return values.
+static SDValue
+combineUnpackingMovIntoLoad(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
+ // Don't run this optimization before the legalizer
+ if (!DCI.isAfterLegalizeDAG())
+ return SDValue();
+
+ EVT ElemVT = N->getValueType(0);
+ if (!Isv2x16VT(ElemVT))
+ return SDValue();
+
+ // Check whether all outputs are either used by an extractelt or are
+ // glue/chain nodes
+ if (!all_of(N->uses(), [&](SDUse &U) {
+ // Skip glue, chain nodes
+ if (U.getValueType() == MVT::Glue || U.getValueType() == MVT::Other)
+ return true;
+ if (U.getUser()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
+ if (N->getOpcode() != ISD::LOAD)
+ return true;
+ // Since this is an ISD::LOAD, check all extractelts are used. If
+ // any are not used, we don't want to defeat another optimization that
+ // will narrow the load.
+ //
+ // For example:
+ //
+ // L: v2f16,ch = load <p>
+ // e0: f16 = extractelt L:0, 0
+ // e1: f16 = extractelt L:0, 1 <-- unused
+ // store e0
+ //
+ // Can be optimized by DAGCombiner to:
+ //
+ // L: f16,ch = load <p>
+ // store L:0
+ return !U.getUser()->use_empty();
+ }
+
+ // Otherwise, this use prevents us from splitting a value.
+ return false;
+ }))
+ return SDValue();
+
+ auto *LD = cast<MemSDNode>(N);
+ EVT MemVT = LD->getMemoryVT();
+ SDLoc DL(LD);
+
+ // the new opcode after we double the number of operands
+ NVPTXISD::NodeType Opcode;
+ SmallVector<SDValue> Operands(LD->ops());
+ unsigned OldNumValues; // non-glue, non-chain outputs
+ switch (LD->getOpcode()) {
+ case ISD::LOAD:
+ OldNumValues = 1;
+ // Any packed type is legal, so the legalizer will not have lowered
+ // ISD::LOAD -> NVPTXISD::Load (unless it's under-aligned). We have to do it
+ // here.
+ Opcode = NVPTXISD::LoadV2;
+ Operands.push_back(DCI.DAG.getIntPtrConstant(
+ cast<LoadSDNode>(LD)->getExtensionType(), DL));
+ break;
+ case NVPTXISD::LoadParamV2:
+ OldNumValues = 2;
+ Opcode = NVPTXISD::LoadParamV4;
+ break;
+ case NVPTXISD::LoadV2:
+ OldNumValues = 2;
+ Opcode = NVPTXISD::LoadV4;
+ break;
+ case NVPTXISD::LoadV4:
+ // PTX doesn't support v8 for 16-bit values
+ case NVPTXISD::LoadV8:
+ // PTX doesn't support the next doubling of outputs
+ return SDValue();
+ }
+
+ SmallVector<EVT> NewVTs(OldNumValues * 2, ElemVT.getVectorElementType());
+ // add remaining chain and glue values
+ NewVTs.append(LD->value_begin() + OldNumValues, LD->value_end());
+
+ // Create the new load
+ SDValue NewLoad =
+ DCI.DAG.getMemIntrinsicNode(Opcode, DL, DCI.DAG.getVTList(NewVTs),
+ Operands, MemVT, LD->getMemOperand());
+
+ // Now we use a combination of BUILD_VECTORs and a MERGE_VALUES node to keep
+ // the outputs the same. These nodes will be optimized away in later
+ // DAGCombiner iterations.
+ SmallVector<SDValue> Results;
+ for (unsigned I = 0; I < NewLoad->getNumValues();) {
+ if (I < OldNumValues * 2) {
+ Results.push_back(DCI.DAG.getBuildVector(
+ ElemVT, DL, {NewLoad.getValue(I), NewLoad.getValue(I + 1)}));
+ I += 2;
+ } else {
+ Results.push_back(NewLoad.getValue(I));
+ I += 1;
+ }
+ }
+
+ return DCI.DAG.getMergeValues(Results, DL);
+}
+
+/// Fold a packing mov into a store. This may help lower register pressure.
+///
+/// ex:
+/// v: v2f16 = build_vector a:f16, b:f16
+/// StoreRetval v
+///
+/// ...is turned into...
+///
+/// StoreRetvalV2 a:f16, b:f16
+static SDValue combinePackingMovIntoStore(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI,
+ unsigned Front, unsigned Back) {
+ // We want to run this as late as possible since other optimizations may
+ // eliminate the BUILD_VECTORs.
+ if (!DCI.isAfterLegalizeDAG())
+ return SDValue();
+
+ // Get the type of the operands being stored.
+ EVT ElementVT = N->getOperand(Front).getValueType();
+
+ if (!Isv2x16VT(ElementVT))
+ return SDValue();
+
+ auto *ST = cast<MemSDNode>(N);
+ EVT MemVT = ElementVT.getVectorElementType();
+
+ // The new opcode after we double the number of operands.
+ NVPTXISD::NodeType Opcode;
+ switch (N->getOpcode()) {
+ case ISD::STORE:
+ // Any packed type is legal, so the legalizer will not have lowered
+ // ISD::STORE -> NVPTXISD::Store (unless it's under-aligned). We have to do
+ // it here.
+ MemVT = ST->getMemoryVT();
+ Opcode = NVPTXISD::StoreV2;
+ break;
+ case NVPTXISD::StoreParam:
+ Opcode = NVPTXISD::StoreParamV2;
+ break;
+ case NVPTXISD::StoreParamV2:
+ Opcode = NVPTXISD::StoreParamV4;
+ break;
+ case NVPTXISD::StoreRetval:
+ Opcode = NVPTXISD::StoreRetvalV2;
+ break;
+ case NVPTXISD::StoreRetvalV2:
+ Opcode = NVPTXISD::StoreRetvalV4;
+ break;
+ case NVPTXISD::StoreV2:
+ MemVT = ST->getMemoryVT();
+ Opcode = NVPTXISD::StoreV4;
+ break;
+ case NVPTXISD::StoreV4:
+ // PTX doesn't support v8 for 16-bit values
+ case NVPTXISD::StoreParamV4:
+ case NVPTXISD::StoreRetvalV4:
+ case NVPTXISD::StoreV8:
+ // PTX doesn't support the next doubling of operands for these opcodes.
+ return SDValue();
+ default:
+ llvm_unreachable("Unhandled store opcode");
+ }
+
+ // Scan the operands and if they're all BUILD_VECTORs, we'll have gathered
+ // their elements.
+ SmallVector<SDValue, 4> Operands(N->ops().take_front(Front));
+ for (SDValue BV : N->ops().drop_front(Front).drop_back(Back)) {
+ if (BV.getOpcode() != ISD::BUILD_VECTOR)
+ return SDValue();
+
+ // If the operand has multiple uses, this optimization can increase register
+ // pressure.
+ if (!BV.hasOneUse())
+ return SDValue();
+
+ // DAGCombiner visits nodes bottom-up. Check the BUILD_VECTOR operands for
+ // any signs they may be folded by some other pattern or rule.
+ for (SDValue Op : BV->ops()) {
+ // Peek through bitcasts
+ if (Op.getOpcode() == ISD::BITCAST)
+ Op = Op.getOperand(0);
+
+ // This may be folded into a PRMT.
+ if (Op.getValueType() == MVT::i16 && Op.getOpcode() == ISD::TRUNCATE &&
+ Op->getOperand(0).getValueType() == MVT::i32)
+ return SDValue();
+
+ // This may be folded into cvt.bf16x2
+ if (Op.getOpcode() == ISD::FP_ROUND)
+ return SDValue();
+ }
+ Operands.append({BV.getOperand(0), BV.getOperand(1)});
+ }
+ Operands.append(N->op_end() - Back, N->op_end());
+
+ // Now we replace the store
+ return DCI.DAG.getMemIntrinsicNode(Opcode, SDLoc(N), N->getVTList(), Operands,
+ MemVT, ST->getMemOperand());
+}
+
+static SDValue PerformStoreCombineHelper(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI,
+ unsigned Front, unsigned Back) {
if (all_of(N->ops().drop_front(Front).drop_back(Back),
[](const SDUse &U) { return U.get()->isUndef(); }))
// Operand 0 is the previous value in the chain. Cannot return EntryToken
// as the previous value will become unused and eliminated later.
return N->getOperand(0);
- return SDValue();
+ return combinePackingMovIntoStore(N, DCI, Front, Back);
+}
+
+static SDValue PerformStoreCombine(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI) {
+ return combinePackingMovIntoStore(N, DCI, 1, 2);
}
-static SDValue PerformStoreParamCombine(SDNode *N) {
+static SDValue PerformStoreParamCombine(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI) {
// Operands from the 3rd to the 2nd last one are the values to be stored.
// {Chain, ArgID, Offset, Val, Glue}
- return PerformStoreCombineHelper(N, 3, 1);
+ return PerformStoreCombineHelper(N, DCI, 3, 1);
}
-static SDValue PerformStoreRetvalCombine(SDNode *N) {
+static SDValue PerformStoreRetvalCombine(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI) {
// Operands from the 2nd to the last one are the values to be stored
- return PerformStoreCombineHelper(N, 2, 0);
+ return PerformStoreCombineHelper(N, DCI, 2, 0);
}
/// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
@@ -5697,14 +5905,23 @@ SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
return PerformREMCombine(N, DCI, OptLevel);
case ISD::SETCC:
return PerformSETCCCombine(N, DCI, STI.getSmVersion());
+ case ISD::LOAD:
+ case NVPTXISD::LoadParamV2:
+ case NVPTXISD::LoadV2:
+ case NVPTXISD::LoadV4:
+ return combineUnpackingMovIntoLoad(N, DCI);
case NVPTXISD::StoreRetval:
case NVPTXISD::StoreRetvalV2:
case NVPTXISD::StoreRetvalV4:
- return PerformStoreRetvalCombine(N);
+ return PerformStoreRetvalCombine(N, DCI);
case NVPTXISD::StoreParam:
case NVPTXISD::StoreParamV2:
case NVPTXISD::StoreParamV4:
- return PerformStoreParamCombine(N);
+ return PerformStoreParamCombine(N, DCI);
+ case ISD::STORE:
+ case NVPTXISD::StoreV2:
+ case NVPTXISD::StoreV4:
+ return PerformStoreCombine(N, DCI);
case ISD::EXTRACT_VECTOR_ELT:
return PerformEXTRACTCombine(N, DCI);
case ISD::VSELECT:
diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
index 32225ed04e2d9..95af9c64a73ac 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
@@ -146,37 +146,35 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM70: {
; SM70-NEXT: .reg .pred %p<3>;
; SM70-NEXT: .reg .b16 %rs<5>;
-; SM70-NEXT: .reg .b32 %r<24>;
+; SM70-NEXT: .reg .b32 %r<22>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r1, [test_faddx2_param_0];
-; SM70-NEXT: ld.param.b32 %r2, [test_faddx2_param_1];
-; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_faddx2_param_0];
+; SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_faddx2_param_1];
+; SM70-NEXT: cvt.u32.u16 %r1, %rs4;
+; SM70-NEXT: shl.b32 %r2, %r1, 16;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
-; SM70-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NEXT: add.rn.f32 %r7, %r6, %r4;
-; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; SM70-NEXT: add.s32 %r9, %r8, %r7;
-; SM70-NEXT: add.s32 %r10, %r9, 32767;
-; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: add.rn.f32 %r5, %r4, %r2;
+; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT: add.s32 %r7, %r6, %r5;
+; SM70-NEXT: add.s32 %r8, %r7, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r5, %r5;
+; SM70-NEXT: or.b32 %r9, %r5, 4194304;
+; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
-; SM70-NEXT: shl.b32 %r16, %r15, 16;
-; SM70-NEXT: add.rn.f32 %r17, %r16, %r14;
-; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
-; SM70-NEXT: add.s32 %r19, %r18, %r17;
-; SM70-NEXT: add.s32 %r20, %r19, 32767;
-; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
-; SM70-NEXT: or.b32 %r21, %r17, 4194304;
-; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
-; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
-; SM70-NEXT: st.param.b32 [func_retval0], %r23;
+; SM70-NEXT: add.rn.f32 %r15, %r14, %r12;
+; SM70-NEXT: bfe.u32 %r16, %r15, 16, 1;
+; SM70-NEXT: add.s32 %r17, %r16, %r15;
+; SM70-NEXT: add.s32 %r18, %r17, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
+; SM70-NEXT: or.b32 %r19, %r15, 4194304;
+; SM70-NEXT: selp.b32 %r20, %r19, %r18, %p2;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r10, 0x7632U;
+; SM70-NEXT: st.param.b32 [func_retval0], %r21;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_faddx2(
@@ -184,31 +182,29 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: .reg .b32 %r<5>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
-; SM80-NEXT: ld.param.b32 %r1, [test_faddx2_param_1];
-; SM80-NEXT: ld.param.b32 %r2, [test_faddx2_param_0];
+; SM80-NEXT: ld.param.b32 %r1, [test_faddx2_param_0];
+; SM80-NEXT: ld.param.b32 %r2, [test_faddx2_param_1];
; SM80-NEXT: mov.b32 %r3, 1065369472;
-; SM80-NEXT: fma.rn.bf16x2 %r4, %r2, %r3, %r1;
+; SM80-NEXT: fma.rn.bf16x2 %r4, %r1, %r3, %r2;
; SM80-NEXT: st.param.b32 [func_retval0], %r4;
; SM80-NEXT: ret;
;
; SM80-FTZ-LABEL: test_faddx2(
; SM80-FTZ: {
; SM80-FTZ-NEXT: .reg .b16 %rs<5>;
-; SM80-FTZ-NEXT: .reg .b32 %r<10>;
+; SM80-FTZ-NEXT: .reg .b32 %r<8>;
; SM80-FTZ-EMPTY:
; SM80-FTZ-NEXT: // %bb.0:
-; SM80-FTZ-NEXT: ld.param.b32 %r1, [test_faddx2_param_0];
-; SM80-FTZ-NEXT: ld.param.b32 %r2, [test_faddx2_param_1];
-; SM80-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r3, %rs1;
-; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r4, %rs3;
-; SM80-FTZ-NEXT: add.rn.ftz.f32 %r5, %r4, %r3;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r6, %rs2;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r7, %rs4;
-; SM80-FTZ-NEXT: add.rn.ftz.f32 %r8, %r7, %r6;
-; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r9;
+; SM80-FTZ-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_faddx2_param_0];
+; SM80-FTZ-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_faddx2_param_1];
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs3;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; SM80-FTZ-NEXT: add.rn.ftz.f32 %r3, %r2, %r1;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r4, %rs4;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r5, %rs2;
+; SM80-FTZ-NEXT: add.rn.ftz.f32 %r6, %r5, %r4;
+; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r7;
; SM80-FTZ-NEXT: ret;
;
; SM90-LABEL: test_faddx2(
@@ -216,9 +212,9 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM90-NEXT: .reg .b32 %r<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_faddx2_param_1];
-; SM90-NEXT: ld.param.b32 %r2, [test_faddx2_param_0];
-; SM90-NEXT: add.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT: ld.param.b32 %r1, [test_faddx2_param_0];
+; SM90-NEXT: ld.param.b32 %r2, [test_faddx2_param_1];
+; SM90-NEXT: add.rn.bf16x2 %r3, %r1, %r2;
; SM90-NEXT: st.param.b32 [func_retval0], %r3;
; SM90-NEXT: ret;
%r = fadd <2 x bfloat> %a, %b
@@ -230,37 +226,35 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM70: {
; SM70-NEXT: .reg .pred %p<3>;
; SM70-NEXT: .reg .b16 %rs<5>;
-; SM70-NEXT: .reg .b32 %r<24>;
+; SM70-NEXT: .reg .b32 %r<22>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r1, [test_fsubx2_param_0];
-; SM70-NEXT: ld.param.b32 %r2, [test_fsubx2_param_1];
-; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_fsubx2_param_0];
+; SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_fsubx2_param_1];
+; SM70-NEXT: cvt.u32.u16 %r1, %rs4;
+; SM70-NEXT: shl.b32 %r2, %r1, 16;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
-; SM70-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NEXT: sub.rn.f32 %r7, %r6, %r4;
-; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; SM70-NEXT: add.s32 %r9, %r8, %r7;
-; SM70-NEXT: add.s32 %r10, %r9, 32767;
-; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: sub.rn.f32 %r5, %r4, %r2;
+; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT: add.s32 %r7, %r6, %r5;
+; SM70-NEXT: add.s32 %r8, %r7, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r5, %r5;
+; SM70-NEXT: or.b32 %r9, %r5, 4194304;
+; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
-; SM70-NEXT: shl.b32 %r16, %r15, 16;
-; SM70-NEXT: sub.rn.f32 %r17, %r16, %r14;
-; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
-; SM70-NEXT: add.s32 %r19, %r18, %r17;
-; SM70-NEXT: add.s32 %r20, %r19, 32767;
-; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
-; SM70-NEXT: or.b32 %r21, %r17, 4194304;
-; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
-; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
-; SM70-NEXT: st.param.b32 [func_retval0], %r23;
+; SM70-NEXT: sub.rn.f32 %r15, %r14, %r12;
+; SM70-NEXT: bfe.u32 %r16, %r15, 16, 1;
+; SM70-NEXT: add.s32 %r17, %r16, %r15;
+; SM70-NEXT: add.s32 %r18, %r17, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
+; SM70-NEXT: or.b32 %r19, %r15, 4194304;
+; SM70-NEXT: selp.b32 %r20, %r19, %r18, %p2;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r10, 0x7632U;
+; SM70-NEXT: st.param.b32 [func_retval0], %r21;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_fsubx2(
@@ -269,30 +263,28 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
; SM80-NEXT: ld.param.b32 %r1, [test_fsubx2_param_0];
-; SM80-NEXT: ld.param.b32 %r2, [test_fsubx2_param_1];
-; SM80-NEXT: mov.b32 %r3, -1082081408;
-; SM80-NEXT: fma.rn.bf16x2 %r4, %r2, %r3, %r1;
+; SM80-NEXT: mov.b32 %r2, -1082081408;
+; SM80-NEXT: ld.param.b32 %r3, [test_fsubx2_param_1];
+; SM80-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
; SM80-NEXT: st.param.b32 [func_retval0], %r4;
; SM80-NEXT: ret;
;
; SM80-FTZ-LABEL: test_fsubx2(
; SM80-FTZ: {
; SM80-FTZ-NEXT: .reg .b16 %rs<5>;
-; SM80-FTZ-NEXT: .reg .b32 %r<10>;
+; SM80-FTZ-NEXT: .reg .b32 %r<8>;
; SM80-FTZ-EMPTY:
; SM80-FTZ-NEXT: // %bb.0:
-; SM80-FTZ-NEXT: ld.param.b32 %r1, [test_fsubx2_param_0];
-; SM80-FTZ-NEXT: ld.param.b32 %r2, [test_fsubx2_param_1];
-; SM80-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r3, %rs1;
-; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r4, %rs3;
-; SM80-FTZ-NEXT: sub.rn.ftz.f32 %r5, %r4, %r3;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r6, %rs2;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r7, %rs4;
-; SM80-FTZ-NEXT: sub.rn.ftz.f32 %r8, %r7, %r6;
-; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r9;
+; SM80-FTZ-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_fsubx2_param_0];
+; SM80-FTZ-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_fsubx2_param_1];
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs3;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; SM80-FTZ-NEXT: sub.rn.ftz.f32 %r3, %r2, %r1;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r4, %rs4;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r5, %rs2;
+; SM80-FTZ-NEXT: sub.rn.ftz.f32 %r6, %r5, %r4;
+; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r7;
; SM80-FTZ-NEXT: ret;
;
; SM90-LABEL: test_fsubx2(
@@ -300,9 +292,9 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM90-NEXT: .reg .b32 %r<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_fsubx2_param_1];
-; SM90-NEXT: ld.param.b32 %r2, [test_fsubx2_param_0];
-; SM90-NEXT: sub.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT: ld.param.b32 %r1, [test_fsubx2_param_0];
+; SM90-NEXT: ld.param.b32 %r2, [test_fsubx2_param_1];
+; SM90-NEXT: sub.rn.bf16x2 %r3, %r1, %r2;
; SM90-NEXT: st.param.b32 [func_retval0], %r3;
; SM90-NEXT: ret;
%r = fsub <2 x bfloat> %a, %b
@@ -314,37 +306,35 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM70: {
; SM70-NEXT: .reg .pred %p<3>;
; SM70-NEXT: .reg .b16 %rs<5>;
-; SM70-NEXT: .reg .b32 %r<24>;
+; SM70-NEXT: .reg .b32 %r<22>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r1, [test_fmulx2_param_0];
-; SM70-NEXT: ld.param.b32 %r2, [test_fmulx2_param_1];
-; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_fmulx2_param_0];
+; SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_fmulx2_param_1];
+; SM70-NEXT: cvt.u32.u16 %r1, %rs4;
+; SM70-NEXT: shl.b32 %r2, %r1, 16;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
-; SM70-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NEXT: mul.rn.f32 %r7, %r6, %r4;
-; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; SM70-NEXT: add.s32 %r9, %r8, %r7;
-; SM70-NEXT: add.s32 %r10, %r9, 32767;
-; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: mul.rn.f32 %r5, %r4, %r2;
+; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT: add.s32 %r7, %r6, %r5;
+; SM70-NEXT: add.s32 %r8, %r7, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r5, %r5;
+; SM70-NEXT: or.b32 %r9, %r5, 4194304;
+; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
-; SM70-NEXT: shl.b32 %r16, %r15, 16;
-; SM70-NEXT: mul.rn.f32 %r17, %r16, %r14;
-; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
-; SM70-NEXT: add.s32 %r19, %r18, %r17;
-; SM70-NEXT: add.s32 %r20, %r19, 32767;
-; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
-; SM70-NEXT: or.b32 %r21, %r17, 4194304;
-; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
-; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
-; SM70-NEXT: st.param.b32 [func_retval0], %r23;
+; SM70-NEXT: mul.rn.f32 %r15, %r14, %r12;
+; SM70-NEXT: bfe.u32 %r16, %r15, 16, 1;
+; SM70-NEXT: add.s32 %r17, %r16, %r15;
+; SM70-NEXT: add.s32 %r18, %r17, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
+; SM70-NEXT: or.b32 %r19, %r15, 4194304;
+; SM70-NEXT: selp.b32 %r20, %r19, %r18, %p2;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r10, 0x7632U;
+; SM70-NEXT: st.param.b32 [func_retval0], %r21;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_fmulx2(
@@ -352,31 +342,29 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: .reg .b32 %r<5>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
-; SM80-NEXT: ld.param.b32 %r1, [test_fmulx2_param_1];
-; SM80-NEXT: ld.param.b32 %r2, [test_fmulx2_param_0];
-; SM80-NEXT: mov.b32 %r3, -2147450880;
-; SM80-NEXT: fma.rn.bf16x2 %r4, %r2, %r1, %r3;
+; SM80-NEXT: ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM80-NEXT: mov.b32 %r2, -2147450880;
+; SM80-NEXT: ld.param.b32 %r3, [test_fmulx2_param_1];
+; SM80-NEXT: fma.rn.bf16x2 %r4, %r1, %r3, %r2;
; SM80-NEXT: st.param.b32 [func_retval0], %r4;
; SM80-NEXT: ret;
;
; SM80-FTZ-LABEL: test_fmulx2(
; SM80-FTZ: {
; SM80-FTZ-NEXT: .reg .b16 %rs<5>;
-; SM80-FTZ-NEXT: .reg .b32 %r<10>;
+; SM80-FTZ-NEXT: .reg .b32 %r<8>;
; SM80-FTZ-EMPTY:
; SM80-FTZ-NEXT: // %bb.0:
-; SM80-FTZ-NEXT: ld.param.b32 %r1, [test_fmulx2_param_0];
-; SM80-FTZ-NEXT: ld.param.b32 %r2, [test_fmulx2_param_1];
-; SM80-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r3, %rs1;
-; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r4, %rs3;
-; SM80-FTZ-NEXT: mul.rn.ftz.f32 %r5, %r4, %r3;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r6, %rs2;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r7, %rs4;
-; SM80-FTZ-NEXT: mul.rn.ftz.f32 %r8, %r7, %r6;
-; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r9;
+; SM80-FTZ-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_fmulx2_param_0];
+; SM80-FTZ-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_fmulx2_param_1];
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs3;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; SM80-FTZ-NEXT: mul.rn.ftz.f32 %r3, %r2, %r1;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r4, %rs4;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r5, %rs2;
+; SM80-FTZ-NEXT: mul.rn.ftz.f32 %r6, %r5, %r4;
+; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r7;
; SM80-FTZ-NEXT: ret;
;
; SM90-LABEL: test_fmulx2(
@@ -384,9 +372,9 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM90-NEXT: .reg .b32 %r<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_fmulx2_param_1];
-; SM90-NEXT: ld.param.b32 %r2, [test_fmulx2_param_0];
-; SM90-NEXT: mul.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT: ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM90-NEXT: ld.param.b32 %r2, [test_fmulx2_param_1];
+; SM90-NEXT: mul.rn.bf16x2 %r3, %r1, %r2;
; SM90-NEXT: st.param.b32 [func_retval0], %r3;
; SM90-NEXT: ret;
%r = fmul <2 x bfloat> %a, %b
@@ -398,97 +386,89 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM70: {
; SM70-NEXT: .reg .pred %p<3>;
; SM70-NEXT: .reg .b16 %rs<5>;
-; SM70-NEXT: .reg .b32 %r<24>;
+; SM70-NEXT: .reg .b32 %r<22>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r1, [test_fdiv_param_0];
-; SM70-NEXT: ld.param.b32 %r2, [test_fdiv_param_1];
-; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_fdiv_param_0];
+; SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_fdiv_param_1];
+; SM70-NEXT: cvt.u32.u16 %r1, %rs4;
+; SM70-NEXT: shl.b32 %r2, %r1, 16;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
-; SM70-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NEXT: div.rn.f32 %r7, %r6, %r4;
-; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; SM70-NEXT: add.s32 %r9, %r8, %r7;
-; SM70-NEXT: add.s32 %r10, %r9, 32767;
-; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: div.rn.f32 %r5, %r4, %r2;
+; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT: add.s32 %r7, %r6, %r5;
+; SM70-NEXT: add.s32 %r8, %r7, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r5, %r5;
+; SM70-NEXT: or.b32 %r9, %r5, 4194304;
+; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
-; SM70-NEXT: shl.b32 %r16, %r15, 16;
-; SM70-NEXT: div.rn.f32 %r17, %r16, %r14;
-; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
-; SM70-NEXT: add.s32 %r19, %r18, %r17;
-; SM70-NEXT: add.s32 %r20, %r19, 32767;
-; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
-; SM70-NEXT: or.b32 %r21, %r17, 4194304;
-; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
-; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
-; SM70-NEXT: st.param.b32 [func_retval0], %r23;
+; SM70-NEXT: div.rn.f32 %r15, %r14, %r12;
+; SM70-NEXT: bfe.u32 %r16, %r15, 16, 1;
+; SM70-NEXT: add.s32 %r17, %r16, %r15;
+; SM70-NEXT: add.s32 %r18, %r17, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
+; SM70-NEXT: or.b32 %r19, %r15, 4194304;
+; SM70-NEXT: selp.b32 %r20, %r19, %r18, %p2;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r10, 0x7632U;
+; SM70-NEXT: st.param.b32 [func_retval0], %r21;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_fdiv(
; SM80: {
; SM80-NEXT: .reg .b16 %rs<5>;
-; SM80-NEXT: .reg .b32 %r<10>;
+; SM80-NEXT: .reg .b32 %r<8>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
-; SM80-NEXT: ld.param.b32 %r1, [test_fdiv_param_0];
-; SM80-NEXT: ld.param.b32 %r2, [test_fdiv_param_1];
-; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT: cvt.f32.bf16 %r3, %rs1;
-; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %r4, %rs3;
-; SM80-NEXT: div.rn.f32 %r5, %r4, %r3;
-; SM80-NEXT: cvt.f32.bf16 %r6, %rs2;
-; SM80-NEXT: cvt.f32.bf16 %r7, %rs4;
-; SM80-NEXT: div.rn.f32 %r8, %r7, %r6;
-; SM80-NEXT: cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; SM80-NEXT: st.param.b32 [func_retval0], %r9;
+; SM80-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_fdiv_param_0];
+; SM80-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_fdiv_param_1];
+; SM80-NEXT: cvt.f32.bf16 %r1, %rs3;
+; SM80-NEXT: cvt.f32.bf16 %r2, %rs1;
+; SM80-NEXT: div.rn.f32 %r3, %r2, %r1;
+; SM80-NEXT: cvt.f32.bf16 %r4, %rs4;
+; SM80-NEXT: cvt.f32.bf16 %r5, %rs2;
+; SM80-NEXT: div.rn.f32 %r6, %r5, %r4;
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; SM80-NEXT: st.param.b32 [func_retval0], %r7;
; SM80-NEXT: ret;
;
; SM80-FTZ-LABEL: test_fdiv(
; SM80-FTZ: {
; SM80-FTZ-NEXT: .reg .b16 %rs<5>;
-; SM80-FTZ-NEXT: .reg .b32 %r<10>;
+; SM80-FTZ-NEXT: .reg .b32 %r<8>;
; SM80-FTZ-EMPTY:
; SM80-FTZ-NEXT: // %bb.0:
-; SM80-FTZ-NEXT: ld.param.b32 %r1, [test_fdiv_param_0];
-; SM80-FTZ-NEXT: ld.param.b32 %r2, [test_fdiv_param_1];
-; SM80-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r3, %rs1;
-; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r4, %rs3;
-; SM80-FTZ-NEXT: div.rn.ftz.f32 %r5, %r4, %r3;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r6, %rs2;
-; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r7, %rs4;
-; SM80-FTZ-NEXT: div.rn.ftz.f32 %r8, %r7, %r6;
-; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r9;
+; SM80-FTZ-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_fdiv_param_0];
+; SM80-FTZ-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_fdiv_param_1];
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs3;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; SM80-FTZ-NEXT: div.rn.ftz.f32 %r3, %r2, %r1;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r4, %rs4;
+; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %r5, %rs2;
+; SM80-FTZ-NEXT: div.rn.ftz.f32 %r6, %r5, %r4;
+; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r7;
; SM80-FTZ-NEXT: ret;
;
; SM90-LABEL: test_fdiv(
; SM90: {
; SM90-NEXT: .reg .b16 %rs<5>;
-; SM90-NEXT: .reg .b32 %r<10>;
+; SM90-NEXT: .reg .b32 %r<8>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_fdiv_param_0];
-; SM90-NEXT: ld.param.b32 %r2, [test_fdiv_param_1];
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NEXT: cvt.f32.bf16 %r3, %rs1;
-; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM90-NEXT: cvt.f32.bf16 %r4, %rs3;
-; SM90-NEXT: div.rn.f32 %r5, %r4, %r3;
-; SM90-NEXT: cvt.f32.bf16 %r6, %rs2;
-; SM90-NEXT: cvt.f32.bf16 %r7, %rs4;
-; SM90-NEXT: div.rn.f32 %r8, %r7, %r6;
-; SM90-NEXT: cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; SM90-NEXT: st.param.b32 [func_retval0], %r9;
+; SM90-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_fdiv_param_0];
+; SM90-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_fdiv_param_1];
+; SM90-NEXT: cvt.f32.bf16 %r1, %rs3;
+; SM90-NEXT: cvt.f32.bf16 %r2, %rs1;
+; SM90-NEXT: div.rn.f32 %r3, %r2, %r1;
+; SM90-NEXT: cvt.f32.bf16 %r4, %rs4;
+; SM90-NEXT: cvt.f32.bf16 %r5, %rs2;
+; SM90-NEXT: div.rn.f32 %r6, %r5, %r4;
+; SM90-NEXT: cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; SM90-NEXT: st.param.b32 [func_retval0], %r7;
; SM90-NEXT: ret;
%r = fdiv <2 x bfloat> %a, %b
ret <2 x bfloat> %r
@@ -1477,47 +1457,44 @@ define <2 x bfloat> @test_maximum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
; SM70: {
; SM70-NEXT: .reg .pred %p<11>;
; SM70-NEXT: .reg .b16 %rs<15>;
-; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b32 %r<13>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r1, [test_maximum_v2_param_0];
-; SM70-NEXT: ld.param.b32 %r2, [test_maximum_v2_param_1];
-; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_maximum_v2_param_0];
+; SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_maximum_v2_param_1];
+; SM70-NEXT: cvt.u32.u16 %r1, %rs4;
+; SM70-NEXT: shl.b32 %r2, %r1, 16;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
-; SM70-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NEXT: setp.gt.f32 %p1, %r6, %r4;
-; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM70-NEXT: setp.gt.f32 %p1, %r4, %r2;
+; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r4, %r2;
; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM70-NEXT: setp.eq.s16 %p3, %rs4, 0;
-; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT: setp.eq.s16 %p4, %rs2, 0;
-; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT: cvt.u32.u16 %r7, %rs6;
-; SM70-NEXT: shl.b32 %r8, %r7, 16;
-; SM70-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM70-NEXT: setp.eq.s16 %p3, %rs2, 0;
+; SM70-NEXT: selp.b16 %rs7, %rs2, %rs6, %p3;
+; SM70-NEXT: setp.eq.s16 %p4, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs8, %rs4, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs6;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r6, 0f00000000;
; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs3;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
; SM70-NEXT: shl.b32 %r10, %r9, 16;
-; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM70-NEXT: shl.b32 %r12, %r11, 16;
-; SM70-NEXT: setp.gt.f32 %p6, %r12, %r10;
-; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT: setp.gt.f32 %p6, %r10, %r8;
+; SM70-NEXT: selp.b16 %rs10, %rs1, %rs3, %p6;
+; SM70-NEXT: setp.nan.f32 %p7, %r10, %r8;
; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM70-NEXT: setp.eq.s16 %p8, %rs3, 0;
-; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM70-NEXT: setp.eq.s16 %p9, %rs1, 0;
-; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs11;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM70-NEXT: setp.eq.s16 %p8, %rs1, 0;
+; SM70-NEXT: selp.b16 %rs12, %rs1, %rs11, %p8;
+; SM70-NEXT: setp.eq.s16 %p9, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p9;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs11;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NEXT: setp.eq.f32 %p10, %r12, 0f00000000;
; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM70-NEXT: mov.b32 %r15, {%rs14, %rs9};
-; SM70-NEXT: st.param.b32 [func_retval0], %r15;
+; SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs14, %rs9};
; SM70-NEXT: ret;
;
; SM80-LABEL: test_maximum_v2(
@@ -1525,9 +1502,9 @@ define <2 x bfloat> @test_maximum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
; SM80-NEXT: .reg .b32 %r<4>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
-; SM80-NEXT: ld.param.b32 %r1, [test_maximum_v2_param_1];
-; SM80-NEXT: ld.param.b32 %r2, [test_maximum_v2_param_0];
-; SM80-NEXT: max.NaN.bf16x2 %r3, %r2, %r1;
+; SM80-NEXT: ld.param.b32 %r1, [test_maximum_v2_param_0];
+; SM80-NEXT: ld.param.b32 %r2, [test_maximum_v2_param_1];
+; SM80-NEXT: max.NaN.bf16x2 %r3, %r1, %r2;
; SM80-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-NEXT: ret;
;
@@ -1536,9 +1513,9 @@ define <2 x bfloat> @test_maximum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
; SM80-FTZ-NEXT: .reg .b32 %r<4>;
; SM80-FTZ-EMPTY:
; SM80-FTZ-NEXT: // %bb.0:
-; SM80-FTZ-NEXT: ld.param.b32 %r1, [test_maximum_v2_param_1];
-; SM80-FTZ-NEXT: ld.param.b32 %r2, [test_maximum_v2_param_0];
-; SM80-FTZ-NEXT: max.NaN.bf16x2 %r3, %r2, %r1;
+; SM80-FTZ-NEXT: ld.param.b32 %r1, [test_maximum_v2_param_0];
+; SM80-FTZ-NEXT: ld.param.b32 %r2, [test_maximum_v2_param_1];
+; SM80-FTZ-NEXT: max.NaN.bf16x2 %r3, %r1, %r2;
; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-FTZ-NEXT: ret;
;
@@ -1547,9 +1524,9 @@ define <2 x bfloat> @test_maximum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
; SM90-NEXT: .reg .b32 %r<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_maximum_v2_param_1];
-; SM90-NEXT: ld.param.b32 %r2, [test_maximum_v2_param_0];
-; SM90-NEXT: max.NaN.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT: ld.param.b32 %r1, [test_maximum_v2_param_0];
+; SM90-NEXT: ld.param.b32 %r2, [test_maximum_v2_param_1];
+; SM90-NEXT: max.NaN.bf16x2 %r3, %r1, %r2;
; SM90-NEXT: st.param.b32 [func_retval0], %r3;
; SM90-NEXT: ret;
%r = call <2 x bfloat> @llvm.maximum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
@@ -1561,37 +1538,35 @@ define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
; SM70: {
; SM70-NEXT: .reg .pred %p<3>;
; SM70-NEXT: .reg .b16 %rs<5>;
-; SM70-NEXT: .reg .b32 %r<24>;
+; SM70-NEXT: .reg .b32 %r<22>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r1, [test_maxnum_v2_param_0];
-; SM70-NEXT: ld.param.b32 %r2, [test_maxnum_v2_param_1];
-; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_maxnum_v2_param_0];
+; SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_maxnum_v2_param_1];
+; SM70-NEXT: cvt.u32.u16 %r1, %rs4;
+; SM70-NEXT: shl.b32 %r2, %r1, 16;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
-; SM70-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NEXT: max.f32 %r7, %r6, %r4;
-; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; SM70-NEXT: add.s32 %r9, %r8, %r7;
-; SM70-NEXT: add.s32 %r10, %r9, 32767;
-; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: max.f32 %r5, %r4, %r2;
+; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT: add.s32 %r7, %r6, %r5;
+; SM70-NEXT: add.s32 %r8, %r7, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r5, %r5;
+; SM70-NEXT: or.b32 %r9, %r5, 4194304;
+; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
-; SM70-NEXT: shl.b32 %r16, %r15, 16;
-; SM70-NEXT: max.f32 %r17, %r16, %r14;
-; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
-; SM70-NEXT: add.s32 %r19, %r18, %r17;
-; SM70-NEXT: add.s32 %r20, %r19, 32767;
-; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
-; SM70-NEXT: or.b32 %r21, %r17, 4194304;
-; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
-; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
-; SM70-NEXT: st.param.b32 [func_retval0], %r23;
+; SM70-NEXT: max.f32 %r15, %r14, %r12;
+; SM70-NEXT: bfe.u32 %r16, %r15, 16, 1;
+; SM70-NEXT: add.s32 %r17, %r16, %r15;
+; SM70-NEXT: add.s32 %r18, %r17, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
+; SM70-NEXT: or.b32 %r19, %r15, 4194304;
+; SM70-NEXT: selp.b32 %r20, %r19, %r18, %p2;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r10, 0x7632U;
+; SM70-NEXT: st.param.b32 [func_retval0], %r21;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_maxnum_v2(
@@ -1599,9 +1574,9 @@ define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
; SM80-NEXT: .reg .b32 %r<4>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
-; SM80-NEXT: ld.param.b32 %r1, [test_maxnum_v2_param_1];
-; SM80-NEXT: ld.param.b32 %r2, [test_maxnum_v2_param_0];
-; SM80-NEXT: max.bf16x2 %r3, %r2, %r1;
+; SM80-NEXT: ld.param.b32 %r1, [test_maxnum_v2_param_0];
+; SM80-NEXT: ld.param.b32 %r2, [test_maxnum_v2_param_1];
+; SM80-NEXT: max.bf16x2 %r3, %r1, %r2;
; SM80-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-NEXT: ret;
;
@@ -1610,9 +1585,9 @@ define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
; SM80-FTZ-NEXT: .reg .b32 %r<4>;
; SM80-FTZ-EMPTY:
; SM80-FTZ-NEXT: // %bb.0:
-; SM80-FTZ-NEXT: ld.param.b32 %r1, [test_maxnum_v2_param_1];
-; SM80-FTZ-NEXT: ld.param.b32 %r2, [test_maxnum_v2_param_0];
-; SM80-FTZ-NEXT: max.bf16x2 %r3, %r2, %r1;
+; SM80-FTZ-NEXT: ld.param.b32 %r1, [test_maxnum_v2_param_0];
+; SM80-FTZ-NEXT: ld.param.b32 %r2, [test_maxnum_v2_param_1];
+; SM80-FTZ-NEXT: max.bf16x2 %r3, %r1, %r2;
; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3;
; SM80-FTZ-NEXT: ret;
;
@@ -1621,9 +1596,9 @@ define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
; SM90-NEXT: .reg .b32 %r<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_maxnum_v2_param_1];
-; SM90-NEXT: ld.param.b32 %r2, [test_maxnum_v2_param_0];
-; SM90-NEXT: max.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT: ld.param.b32 %r1, [test_maxnum_v2_param_0];
+; SM90-NEXT: ld.param.b32 %r2, [test_maxnum_v2_param_1];
+; SM90-NEXT: max.bf16x2 %r3, %r1, %r2;
; SM90-NEXT: st.param.b32 [func_retval0], %r3;
; SM90-NEXT: ret;
%r = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
index 0e90b254225eb..80627a03354a0 100644
--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
@@ -11,17 +11,16 @@ define <2 x bfloat> @test_sin(<2 x bfloat> %a) #0 #1 {
; CHECK-LABEL: test_sin(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b32 %r<6>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [test_sin_param_0];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.f32.bf16 %r2, %rs1;
-; CHECK-NEXT: sin.approx.f32 %r3, %r2;
-; CHECK-NEXT: cvt.f32.bf16 %r4, %rs2;
-; CHECK-NEXT: sin.approx.f32 %r5, %r4;
-; CHECK-NEXT: cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_sin_param_0];
+; CHECK-NEXT: cvt.f32.bf16 %r1, %rs1;
+; CHECK-NEXT: sin.approx.f32 %r2, %r1;
+; CHECK-NEXT: cvt.f32.bf16 %r3, %rs2;
+; CHECK-NEXT: sin.approx.f32 %r4, %r3;
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r5;
; CHECK-NEXT: ret;
%r = call <2 x bfloat> @llvm.sin.f16(<2 x bfloat> %a)
ret <2 x bfloat> %r
@@ -31,17 +30,16 @@ define <2 x bfloat> @test_cos(<2 x bfloat> %a) #0 #1 {
; CHECK-LABEL: test_cos(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b32 %r<6>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [test_cos_param_0];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.f32.bf16 %r2, %rs1;
-; CHECK-NEXT: cos.approx.f32 %r3, %r2;
-; CHECK-NEXT: cvt.f32.bf16 %r4, %rs2;
-; CHECK-NEXT: cos.approx.f32 %r5, %r4;
-; CHECK-NEXT: cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_cos_param_0];
+; CHECK-NEXT: cvt.f32.bf16 %r1, %rs1;
+; CHECK-NEXT: cos.approx.f32 %r2, %r1;
+; CHECK-NEXT: cvt.f32.bf16 %r3, %rs2;
+; CHECK-NEXT: cos.approx.f32 %r4, %r3;
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r5;
; CHECK-NEXT: ret;
%r = call <2 x bfloat> @llvm.cos.f16(<2 x bfloat> %a)
ret <2 x bfloat> %r
diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
index ec993aa15a85a..6f115756a8ae7 100644
--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
@@ -79,9 +79,9 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
; SM80-NEXT: ld.param.b32 %r1, [test_fsubx2_param_0];
-; SM80-NEXT: ld.param.b32 %r2, [test_fsubx2_param_1];
-; SM80-NEXT: mov.b32 %r3, -1082081408;
-; SM80-NEXT: fma.rn.bf16x2 %r4, %r2, %r3, %r1;
+; SM80-NEXT: mov.b32 %r2, -1082081408;
+; SM80-NEXT: ld.param.b32 %r3, [test_fsubx2_param_1];
+; SM80-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
; SM80-NEXT: st.param.b32 [func_retval0], %r4;
; SM80-NEXT: ret;
;
@@ -90,9 +90,9 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM90-NEXT: .reg .b32 %r<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_fsubx2_param_1];
-; SM90-NEXT: ld.param.b32 %r2, [test_fsubx2_param_0];
-; SM90-NEXT: sub.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT: ld.param.b32 %r1, [test_fsubx2_param_0];
+; SM90-NEXT: ld.param.b32 %r2, [test_fsubx2_param_1];
+; SM90-NEXT: sub.rn.bf16x2 %r3, %r1, %r2;
; SM90-NEXT: st.param.b32 [func_retval0], %r3;
; SM90-NEXT: ret;
%r = fsub <2 x bfloat> %a, %b
@@ -105,10 +105,10 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: .reg .b32 %r<5>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
-; SM80-NEXT: ld.param.b32 %r1, [test_fmulx2_param_1];
-; SM80-NEXT: ld.param.b32 %r2, [test_fmulx2_param_0];
-; SM80-NEXT: mov.b32 %r3, -2147450880;
-; SM80-NEXT: fma.rn.bf16x2 %r4, %r2, %r1, %r3;
+; SM80-NEXT: ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM80-NEXT: mov.b32 %r2, -2147450880;
+; SM80-NEXT: ld.param.b32 %r3, [test_fmulx2_param_1];
+; SM80-NEXT: fma.rn.bf16x2 %r4, %r1, %r3, %r2;
; SM80-NEXT: st.param.b32 [func_retval0], %r4;
; SM80-NEXT: ret;
;
@@ -117,9 +117,9 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM90-NEXT: .reg .b32 %r<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_fmulx2_param_1];
-; SM90-NEXT: ld.param.b32 %r2, [test_fmulx2_param_0];
-; SM90-NEXT: mul.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT: ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM90-NEXT: ld.param.b32 %r2, [test_fmulx2_param_1];
+; SM90-NEXT: mul.rn.bf16x2 %r3, %r1, %r2;
; SM90-NEXT: st.param.b32 [func_retval0], %r3;
; SM90-NEXT: ret;
%r = fmul <2 x bfloat> %a, %b
@@ -130,21 +130,19 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; CHECK-LABEL: test_fdiv(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<10>;
+; CHECK-NEXT: .reg .b32 %r<8>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [test_fdiv_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [test_fdiv_param_1];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NEXT: cvt.f32.bf16 %r3, %rs1;
-; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NEXT: cvt.f32.bf16 %r4, %rs3;
-; CHECK-NEXT: div.rn.f32 %r5, %r4, %r3;
-; CHECK-NEXT: cvt.f32.bf16 %r6, %rs2;
-; CHECK-NEXT: cvt.f32.bf16 %r7, %rs4;
-; CHECK-NEXT: div.rn.f32 %r8, %r7, %r6;
-; CHECK-NEXT: cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_fdiv_param_0];
+; CHECK-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_fdiv_param_1];
+; CHECK-NEXT: cvt.f32.bf16 %r1, %rs3;
+; CHECK-NEXT: cvt.f32.bf16 %r2, %rs1;
+; CHECK-NEXT: div.rn.f32 %r3, %r2, %r1;
+; CHECK-NEXT: cvt.f32.bf16 %r4, %rs4;
+; CHECK-NEXT: cvt.f32.bf16 %r5, %rs2;
+; CHECK-NEXT: div.rn.f32 %r6, %r5, %r4;
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r7;
; CHECK-NEXT: ret;
%r = fdiv <2 x bfloat> %a, %b
ret <2 x bfloat> %r
@@ -157,7 +155,7 @@ define <2 x bfloat> @test_fneg(<2 x bfloat> %a) #0 {
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_fneg_param_0];
-; CHECK-NEXT: xor.b32 %r2, %r1, -2147450880;
+; CHECK-NEXT: neg.bf16x2 %r2, %r1;
; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
; CHECK-NEXT: ret;
%r = fneg <2 x bfloat> %a
@@ -243,9 +241,9 @@ define <2 x bfloat> @test_select(<2 x bfloat> %a, <2 x bfloat> %b, i1 zeroext %c
; CHECK-NEXT: ld.param.b8 %rs1, [test_select_param_2];
; CHECK-NEXT: and.b16 %rs2, %rs1, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT: ld.param.b32 %r1, [test_select_param_1];
-; CHECK-NEXT: ld.param.b32 %r2, [test_select_param_0];
-; CHECK-NEXT: selp.b32 %r3, %r2, %r1, %p1;
+; CHECK-NEXT: ld.param.b32 %r1, [test_select_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [test_select_param_1];
+; CHECK-NEXT: selp.b32 %r3, %r1, %r2, %p1;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%r = select i1 %c, <2 x bfloat> %a, <2 x bfloat> %b
@@ -257,47 +255,39 @@ define <2 x bfloat> @test_select_cc(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloa
; SM80: {
; SM80-NEXT: .reg .pred %p<3>;
; SM80-NEXT: .reg .b16 %rs<11>;
-; SM80-NEXT: .reg .b32 %r<10>;
+; SM80-NEXT: .reg .b32 %r<5>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
-; SM80-NEXT: ld.param.b32 %r1, [test_select_cc_param_0];
-; SM80-NEXT: ld.param.b32 %r2, [test_select_cc_param_1];
-; SM80-NEXT: ld.param.b32 %r3, [test_select_cc_param_2];
-; SM80-NEXT: ld.param.b32 %r4, [test_select_cc_param_3];
-; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM80-NEXT: cvt.f32.bf16 %r5, %rs1;
-; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM80-NEXT: cvt.f32.bf16 %r6, %rs3;
-; SM80-NEXT: setp.neu.f32 %p1, %r6, %r5;
-; SM80-NEXT: cvt.f32.bf16 %r7, %rs2;
-; SM80-NEXT: cvt.f32.bf16 %r8, %rs4;
-; SM80-NEXT: setp.neu.f32 %p2, %r8, %r7;
-; SM80-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM80-NEXT: mov.b32 {%rs7, %rs8}, %r1;
-; SM80-NEXT: selp.b16 %rs9, %rs8, %rs6, %p2;
-; SM80-NEXT: selp.b16 %rs10, %rs7, %rs5, %p1;
-; SM80-NEXT: mov.b32 %r9, {%rs10, %rs9};
-; SM80-NEXT: st.param.b32 [func_retval0], %r9;
+; SM80-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_select_cc_param_0];
+; SM80-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_select_cc_param_3];
+; SM80-NEXT: cvt.f32.bf16 %r1, %rs3;
+; SM80-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [test_select_cc_param_2];
+; SM80-NEXT: cvt.f32.bf16 %r2, %rs5;
+; SM80-NEXT: setp.neu.f32 %p1, %r2, %r1;
+; SM80-NEXT: cvt.f32.bf16 %r3, %rs4;
+; SM80-NEXT: cvt.f32.bf16 %r4, %rs6;
+; SM80-NEXT: setp.neu.f32 %p2, %r4, %r3;
+; SM80-NEXT: ld.param.v2.b16 {%rs7, %rs8}, [test_select_cc_param_1];
+; SM80-NEXT: selp.b16 %rs9, %rs2, %rs8, %p2;
+; SM80-NEXT: selp.b16 %rs10, %rs1, %rs7, %p1;
+; SM80-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs9};
; SM80-NEXT: ret;
;
; SM90-LABEL: test_select_cc(
; SM90: {
; SM90-NEXT: .reg .pred %p<3>;
; SM90-NEXT: .reg .b16 %rs<7>;
-; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b32 %r<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_select_cc_param_0];
-; SM90-NEXT: ld.param.b32 %r2, [test_select_cc_param_1];
-; SM90-NEXT: ld.param.b32 %r3, [test_select_cc_param_3];
-; SM90-NEXT: ld.param.b32 %r4, [test_select_cc_param_2];
-; SM90-NEXT: setp.neu.bf16x2 %p1|%p2, %r4, %r3;
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; SM90-NEXT: selp.b16 %rs5, %rs4, %rs2, %p2;
-; SM90-NEXT: selp.b16 %rs6, %rs3, %rs1, %p1;
-; SM90-NEXT: mov.b32 %r5, {%rs6, %rs5};
-; SM90-NEXT: st.param.b32 [func_retval0], %r5;
+; SM90-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_select_cc_param_0];
+; SM90-NEXT: ld.param.b32 %r1, [test_select_cc_param_2];
+; SM90-NEXT: ld.param.b32 %r2, [test_select_cc_param_3];
+; SM90-NEXT: setp.neu.bf16x2 %p1|%p2, %r1, %r2;
+; SM90-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_select_cc_param_1];
+; SM90-NEXT: selp.b16 %rs5, %rs2, %rs4, %p2;
+; SM90-NEXT: selp.b16 %rs6, %rs1, %rs3, %p1;
+; SM90-NEXT: st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; SM90-NEXT: ret;
%cc = fcmp une <2 x bfloat> %c, %d
%r = select <2 x i1> %cc, <2 x bfloat> %a, <2 x bfloat> %b
@@ -309,24 +299,22 @@ define <2 x float> @test_select_cc_f32_bf16(<2 x float> %a, <2 x float> %b,
; SM80: {
; SM80-NEXT: .reg .pred %p<3>;
; SM80-NEXT: .reg .b16 %rs<5>;
-; SM80-NEXT: .reg .b32 %r<13>;
+; SM80-NEXT: .reg .b32 %r<11>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
; SM80-NEXT: ld.param.v2.b32 {%r1, %r2}, [test_select_cc_f32_bf16_param_0];
-; SM80-NEXT: ld.param.b32 %r3, [test_select_cc_f32_bf16_param_2];
-; SM80-NEXT: ld.param.b32 %r4, [test_select_cc_f32_bf16_param_3];
-; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r4;
-; SM80-NEXT: cvt.f32.bf16 %r5, %rs1;
-; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r3;
-; SM80-NEXT: cvt.f32.bf16 %r6, %rs3;
-; SM80-NEXT: setp.neu.f32 %p1, %r6, %r5;
-; SM80-NEXT: cvt.f32.bf16 %r7, %rs2;
-; SM80-NEXT: cvt.f32.bf16 %r8, %rs4;
-; SM80-NEXT: setp.neu.f32 %p2, %r8, %r7;
-; SM80-NEXT: ld.param.v2.b32 {%r9, %r10}, [test_select_cc_f32_bf16_param_1];
-; SM80-NEXT: selp.f32 %r11, %r2, %r10, %p2;
-; SM80-NEXT: selp.f32 %r12, %r1, %r9, %p1;
-; SM80-NEXT: st.param.v2.b32 [func_retval0], {%r12, %r11};
+; SM80-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_select_cc_f32_bf16_param_3];
+; SM80-NEXT: cvt.f32.bf16 %r3, %rs1;
+; SM80-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_select_cc_f32_bf16_param_2];
+; SM80-NEXT: cvt.f32.bf16 %r4, %rs3;
+; SM80-NEXT: setp.neu.f32 %p1, %r4, %r3;
+; SM80-NEXT: cvt.f32.bf16 %r5, %rs2;
+; SM80-NEXT: cvt.f32.bf16 %r6, %rs4;
+; SM80-NEXT: setp.neu.f32 %p2, %r6, %r5;
+; SM80-NEXT: ld.param.v2.b32 {%r7, %r8}, [test_select_cc_f32_bf16_param_1];
+; SM80-NEXT: selp.f32 %r9, %r2, %r8, %p2;
+; SM80-NEXT: selp.f32 %r10, %r1, %r7, %p1;
+; SM80-NEXT: st.param.v2.b32 [func_retval0], {%r10, %r9};
; SM80-NEXT: ret;
;
; SM90-LABEL: test_select_cc_f32_bf16(
@@ -336,9 +324,9 @@ define <2 x float> @test_select_cc_f32_bf16(<2 x float> %a, <2 x float> %b,
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [test_select_cc_f32_bf16_param_0];
-; SM90-NEXT: ld.param.b32 %r3, [test_select_cc_f32_bf16_param_3];
-; SM90-NEXT: ld.param.b32 %r4, [test_select_cc_f32_bf16_param_2];
-; SM90-NEXT: setp.neu.bf16x2 %p1|%p2, %r4, %r3;
+; SM90-NEXT: ld.param.b32 %r3, [test_select_cc_f32_bf16_param_2];
+; SM90-NEXT: ld.param.b32 %r4, [test_select_cc_f32_bf16_param_3];
+; SM90-NEXT: setp.neu.bf16x2 %p1|%p2, %r3, %r4;
; SM90-NEXT: ld.param.v2.b32 {%r5, %r6}, [test_select_cc_f32_bf16_param_1];
; SM90-NEXT: selp.f32 %r7, %r2, %r6, %p2;
; SM90-NEXT: selp.f32 %r8, %r1, %r5, %p1;
@@ -355,21 +343,18 @@ define <2 x bfloat> @test_select_cc_bf16_f32(<2 x bfloat> %a, <2 x bfloat> %b,
; CHECK: {
; CHECK-NEXT: .reg .pred %p<3>;
; CHECK-NEXT: .reg .b16 %rs<7>;
-; CHECK-NEXT: .reg .b32 %r<8>;
+; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [test_select_cc_bf16_f32_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [test_select_cc_bf16_f32_param_1];
-; CHECK-NEXT: ld.param.v2.b32 {%r3, %r4}, [test_select_cc_bf16_f32_param_2];
-; CHECK-NEXT: ld.param.v2.b32 {%r5, %r6}, [test_select_cc_bf16_f32_param_3];
-; CHECK-NEXT: setp.neu.f32 %p1, %r3, %r5;
-; CHECK-NEXT: setp.neu.f32 %p2, %r4, %r6;
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NEXT: selp.b16 %rs5, %rs4, %rs2, %p2;
-; CHECK-NEXT: selp.b16 %rs6, %rs3, %rs1, %p1;
-; CHECK-NEXT: mov.b32 %r7, {%rs6, %rs5};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r7;
+; CHECK-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_select_cc_bf16_f32_param_0];
+; CHECK-NEXT: ld.param.v2.b32 {%r1, %r2}, [test_select_cc_bf16_f32_param_2];
+; CHECK-NEXT: ld.param.v2.b32 {%r3, %r4}, [test_select_cc_bf16_f32_param_3];
+; CHECK-NEXT: setp.neu.f32 %p1, %r1, %r3;
+; CHECK-NEXT: setp.neu.f32 %p2, %r2, %r4;
+; CHECK-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [test_select_cc_bf16_f32_param_1];
+; CHECK-NEXT: selp.b16 %rs5, %rs2, %rs4, %p2;
+; CHECK-NEXT: selp.b16 %rs6, %rs1, %rs3, %p1;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; CHECK-NEXT: ret;
<2 x float> %c, <2 x float> %d) #0 {
%cc = fcmp une <2 x float> %c, %d
@@ -395,14 +380,13 @@ define <2 x float> @test_fpext_2xfloat(<2 x bfloat> %a) #0 {
; CHECK-LABEL: test_fpext_2xfloat(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [test_fpext_2xfloat_param_0];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.f32.bf16 %r2, %rs2;
-; CHECK-NEXT: cvt.f32.bf16 %r3, %rs1;
-; CHECK-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r2};
+; CHECK-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_fpext_2xfloat_param_0];
+; CHECK-NEXT: cvt.f32.bf16 %r1, %rs2;
+; CHECK-NEXT: cvt.f32.bf16 %r2, %rs1;
+; CHECK-NEXT: st.param.v2.b32 [func_retval0], {%r2, %r1};
; CHECK-NEXT: ret;
%r = fpext <2 x bfloat> %a to <2 x float>
ret <2 x float> %r
@@ -461,17 +445,16 @@ define <2 x bfloat> @test_sqrt(<2 x bfloat> %a) #0 {
; CHECK-LABEL: test_sqrt(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b32 %r<6>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [test_sqrt_param_0];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.f32.bf16 %r2, %rs1;
-; CHECK-NEXT: sqrt.rn.f32 %r3, %r2;
-; CHECK-NEXT: cvt.f32.bf16 %r4, %rs2;
-; CHECK-NEXT: sqrt.rn.f32 %r5, %r4;
-; CHECK-NEXT: cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_sqrt_param_0];
+; CHECK-NEXT: cvt.f32.bf16 %r1, %rs1;
+; CHECK-NEXT: sqrt.rn.f32 %r2, %r1;
+; CHECK-NEXT: cvt.f32.bf16 %r3, %rs2;
+; CHECK-NEXT: sqrt.rn.f32 %r4, %r3;
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r5;
; CHECK-NEXT: ret;
%r = call <2 x bfloat> @llvm.sqrt.f16(<2 x bfloat> %a)
ret <2 x bfloat> %r
@@ -483,10 +466,10 @@ define <2 x bfloat> @test_fmuladd(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat>
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [test_fmuladd_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [test_fmuladd_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [test_fmuladd_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [test_fmuladd_param_0];
-; CHECK-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [test_fmuladd_param_2];
+; CHECK-NEXT: fma.rn.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
%r = call <2 x bfloat> @llvm.fmuladd.f16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
@@ -500,7 +483,7 @@ define <2 x bfloat> @test_fabs(<2 x bfloat> %a) #0 {
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_fabs_param_0];
-; CHECK-NEXT: and.b32 %r2, %r1, 2147450879;
+; CHECK-NEXT: abs.bf16x2 %r2, %r1;
; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
; CHECK-NEXT: ret;
%r = call <2 x bfloat> @llvm.fabs.f16(<2 x bfloat> %a)
@@ -513,12 +496,12 @@ define <2 x bfloat> @test_fabs_add(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-NEXT: .reg .b32 %r<7>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
-; SM80-NEXT: ld.param.b32 %r1, [test_fabs_add_param_1];
-; SM80-NEXT: ld.param.b32 %r2, [test_fabs_add_param_0];
-; SM80-NEXT: mov.b32 %r3, 1065369472;
-; SM80-NEXT: fma.rn.bf16x2 %r4, %r2, %r3, %r2;
-; SM80-NEXT: abs.bf16x2 %r5, %r4;
-; SM80-NEXT: fma.rn.bf16x2 %r6, %r5, %r3, %r1;
+; SM80-NEXT: ld.param.b32 %r1, [test_fabs_add_param_0];
+; SM80-NEXT: mov.b32 %r2, 1065369472;
+; SM80-NEXT: fma.rn.bf16x2 %r3, %r1, %r2, %r1;
+; SM80-NEXT: ld.param.b32 %r4, [test_fabs_add_param_1];
+; SM80-NEXT: abs.bf16x2 %r5, %r3;
+; SM80-NEXT: fma.rn.bf16x2 %r6, %r5, %r2, %r4;
; SM80-NEXT: st.param.b32 [func_retval0], %r6;
; SM80-NEXT: ret;
;
@@ -527,11 +510,11 @@ define <2 x bfloat> @test_fabs_add(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM90-NEXT: .reg .b32 %r<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_fabs_add_param_1];
-; SM90-NEXT: ld.param.b32 %r2, [test_fabs_add_param_0];
-; SM90-NEXT: add.rn.bf16x2 %r3, %r2, %r2;
-; SM90-NEXT: abs.bf16x2 %r4, %r3;
-; SM90-NEXT: add.rn.bf16x2 %r5, %r4, %r1;
+; SM90-NEXT: ld.param.b32 %r1, [test_fabs_add_param_0];
+; SM90-NEXT: add.rn.bf16x2 %r2, %r1, %r1;
+; SM90-NEXT: ld.param.b32 %r3, [test_fabs_add_param_1];
+; SM90-NEXT: abs.bf16x2 %r4, %r2;
+; SM90-NEXT: add.rn.bf16x2 %r5, %r4, %r3;
; SM90-NEXT: st.param.b32 [func_retval0], %r5;
; SM90-NEXT: ret;
%s = fadd <2 x bfloat> %a, %a
@@ -546,9 +529,9 @@ define <2 x bfloat> @test_minnum(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; CHECK-NEXT: .reg .b32 %r<4>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [test_minnum_param_1];
-; CHECK-NEXT: ld.param.b32 %r2, [test_minnum_param_0];
-; CHECK-NEXT: min.bf16x2 %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r1, [test_minnum_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [test_minnum_param_1];
+; CHECK-NEXT: min.bf16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%r = call <2 x bfloat> @llvm.minnum.f16(<2 x bfloat> %a, <2 x bfloat> %b)
@@ -561,9 +544,9 @@ define <2 x bfloat> @test_maxnum(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; CHECK-NEXT: .reg .b32 %r<4>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [test_maxnum_param_1];
-; CHECK-NEXT: ld.param.b32 %r2, [test_maxnum_param_0];
-; CHECK-NEXT: max.bf16x2 %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r1, [test_maxnum_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [test_maxnum_param_1];
+; CHECK-NEXT: max.bf16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%r = call <2 x bfloat> @llvm.maxnum.f16(<2 x bfloat> %a, <2 x bfloat> %b)
@@ -574,31 +557,27 @@ define <2 x bfloat> @test_floor(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_floor(
; SM80: {
; SM80-NEXT: .reg .b16 %rs<3>;
-; SM80-NEXT: .reg .b32 %r<7>;
+; SM80-NEXT: .reg .b32 %r<6>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
-; SM80-NEXT: ld.param.b32 %r1, [test_floor_param_0];
-; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %r2, %rs1;
-; SM80-NEXT: cvt.rmi.f32.f32 %r3, %r2;
-; SM80-NEXT: cvt.f32.bf16 %r4, %rs2;
-; SM80-NEXT: cvt.rmi.f32.f32 %r5, %r4;
-; SM80-NEXT: cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; SM80-NEXT: st.param.b32 [func_retval0], %r6;
+; SM80-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_floor_param_0];
+; SM80-NEXT: cvt.f32.bf16 %r1, %rs1;
+; SM80-NEXT: cvt.rmi.f32.f32 %r2, %r1;
+; SM80-NEXT: cvt.f32.bf16 %r3, %rs2;
+; SM80-NEXT: cvt.rmi.f32.f32 %r4, %r3;
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; SM80-NEXT: st.param.b32 [func_retval0], %r5;
; SM80-NEXT: ret;
;
; SM90-LABEL: test_floor(
; SM90: {
; SM90-NEXT: .reg .b16 %rs<5>;
-; SM90-NEXT: .reg .b32 %r<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_floor_param_0];
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_floor_param_0];
; SM90-NEXT: cvt.rmi.bf16.bf16 %rs3, %rs2;
; SM90-NEXT: cvt.rmi.bf16.bf16 %rs4, %rs1;
-; SM90-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; SM90-NEXT: ret;
%r = call <2 x bfloat> @llvm.floor.f16(<2 x bfloat> %a)
ret <2 x bfloat> %r
@@ -608,31 +587,27 @@ define <2 x bfloat> @test_ceil(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_ceil(
; SM80: {
; SM80-NEXT: .reg .b16 %rs<3>;
-; SM80-NEXT: .reg .b32 %r<7>;
+; SM80-NEXT: .reg .b32 %r<6>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
-; SM80-NEXT: ld.param.b32 %r1, [test_ceil_param_0];
-; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %r2, %rs1;
-; SM80-NEXT: cvt.rpi.f32.f32 %r3, %r2;
-; SM80-NEXT: cvt.f32.bf16 %r4, %rs2;
-; SM80-NEXT: cvt.rpi.f32.f32 %r5, %r4;
-; SM80-NEXT: cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; SM80-NEXT: st.param.b32 [func_retval0], %r6;
+; SM80-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_ceil_param_0];
+; SM80-NEXT: cvt.f32.bf16 %r1, %rs1;
+; SM80-NEXT: cvt.rpi.f32.f32 %r2, %r1;
+; SM80-NEXT: cvt.f32.bf16 %r3, %rs2;
+; SM80-NEXT: cvt.rpi.f32.f32 %r4, %r3;
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; SM80-NEXT: st.param.b32 [func_retval0], %r5;
; SM80-NEXT: ret;
;
; SM90-LABEL: test_ceil(
; SM90: {
; SM90-NEXT: .reg .b16 %rs<5>;
-; SM90-NEXT: .reg .b32 %r<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_ceil_param_0];
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_ceil_param_0];
; SM90-NEXT: cvt.rpi.bf16.bf16 %rs3, %rs2;
; SM90-NEXT: cvt.rpi.bf16.bf16 %rs4, %rs1;
-; SM90-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; SM90-NEXT: ret;
%r = call <2 x bfloat> @llvm.ceil.f16(<2 x bfloat> %a)
ret <2 x bfloat> %r
@@ -642,31 +617,27 @@ define <2 x bfloat> @test_trunc(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_trunc(
; SM80: {
; SM80-NEXT: .reg .b16 %rs<3>;
-; SM80-NEXT: .reg .b32 %r<7>;
+; SM80-NEXT: .reg .b32 %r<6>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
-; SM80-NEXT: ld.param.b32 %r1, [test_trunc_param_0];
-; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %r2, %rs1;
-; SM80-NEXT: cvt.rzi.f32.f32 %r3, %r2;
-; SM80-NEXT: cvt.f32.bf16 %r4, %rs2;
-; SM80-NEXT: cvt.rzi.f32.f32 %r5, %r4;
-; SM80-NEXT: cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; SM80-NEXT: st.param.b32 [func_retval0], %r6;
+; SM80-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_trunc_param_0];
+; SM80-NEXT: cvt.f32.bf16 %r1, %rs1;
+; SM80-NEXT: cvt.rzi.f32.f32 %r2, %r1;
+; SM80-NEXT: cvt.f32.bf16 %r3, %rs2;
+; SM80-NEXT: cvt.rzi.f32.f32 %r4, %r3;
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; SM80-NEXT: st.param.b32 [func_retval0], %r5;
; SM80-NEXT: ret;
;
; SM90-LABEL: test_trunc(
; SM90: {
; SM90-NEXT: .reg .b16 %rs<5>;
-; SM90-NEXT: .reg .b32 %r<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_trunc_param_0];
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_trunc_param_0];
; SM90-NEXT: cvt.rzi.bf16.bf16 %rs3, %rs2;
; SM90-NEXT: cvt.rzi.bf16.bf16 %rs4, %rs1;
-; SM90-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; SM90-NEXT: ret;
%r = call <2 x bfloat> @llvm.trunc.f16(<2 x bfloat> %a)
ret <2 x bfloat> %r
@@ -676,31 +647,27 @@ define <2 x bfloat> @test_rint(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_rint(
; SM80: {
; SM80-NEXT: .reg .b16 %rs<3>;
-; SM80-NEXT: .reg .b32 %r<7>;
+; SM80-NEXT: .reg .b32 %r<6>;
; SM80-EMPTY:
; SM80-NEXT: // %bb.0:
-; SM80-NEXT: ld.param.b32 %r1, [test_rint_param_0];
-; SM80-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT: cvt.f32.bf16 %r2, %rs1;
-; SM80-NEXT: cvt.rni.f32.f32 %r3, %r2;
-; SM80-NEXT: cvt.f32.bf16 %r4, %rs2;
-; SM80-NEXT: cvt.rni.f32.f32 %r5, %r4;
-; SM80-NEXT: cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; SM80-NEXT: st.param.b32 [func_retval0], %r6;
+; SM80-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_rint_param_0];
+; SM80-NEXT: cvt.f32.bf16 %r1, %rs1;
+; SM80-NEXT: cvt.rni.f32.f32 %r2, %r1;
+; SM80-NEXT: cvt.f32.bf16 %r3, %rs2;
+; SM80-NEXT: cvt.rni.f32.f32 %r4, %r3;
+; SM80-NEXT: cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; SM80-NEXT: st.param.b32 [func_retval0], %r5;
; SM80-NEXT: ret;
;
; SM90-LABEL: test_rint(
; SM90: {
; SM90-NEXT: .reg .b16 %rs<5>;
-; SM90-NEXT: .reg .b32 %r<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r1, [test_rint_param_0];
-; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_rint_param_0];
; SM90-NEXT: cvt.rni.bf16.bf16 %rs3, %rs2;
; SM90-NEXT: cvt.rni.bf16.bf16 %rs4, %rs1;
-; SM90-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; SM90-NEXT: ret;
%r = call <2 x bfloat> @llvm.rint.f16(<2 x bfloat> %a)
ret <2 x bfloat> %r
@@ -711,35 +678,34 @@ define <2 x bfloat> @test_round(<2 x bfloat> %a) #0 {
; CHECK: {
; CHECK-NEXT: .reg .pred %p<5>;
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<21>;
+; CHECK-NEXT: .reg .b32 %r<20>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [test_round_param_0];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.f32.bf16 %r2, %rs1;
-; CHECK-NEXT: and.b32 %r3, %r2, -2147483648;
-; CHECK-NEXT: or.b32 %r4, %r3, 1056964608;
-; CHECK-NEXT: add.rn.f32 %r5, %r2, %r4;
-; CHECK-NEXT: cvt.rzi.f32.f32 %r6, %r5;
-; CHECK-NEXT: abs.f32 %r7, %r2;
-; CHECK-NEXT: setp.gt.f32 %p1, %r7, 0f4B000000;
-; CHECK-NEXT: selp.f32 %r8, %r2, %r6, %p1;
-; CHECK-NEXT: cvt.rzi.f32.f32 %r9, %r2;
-; CHECK-NEXT: setp.lt.f32 %p2, %r7, 0f3F000000;
-; CHECK-NEXT: selp.f32 %r10, %r9, %r8, %p2;
-; CHECK-NEXT: cvt.f32.bf16 %r11, %rs2;
-; CHECK-NEXT: and.b32 %r12, %r11, -2147483648;
-; CHECK-NEXT: or.b32 %r13, %r12, 1056964608;
-; CHECK-NEXT: add.rn.f32 %r14, %r11, %r13;
-; CHECK-NEXT: cvt.rzi.f32.f32 %r15, %r14;
-; CHECK-NEXT: abs.f32 %r16, %r11;
-; CHECK-NEXT: setp.gt.f32 %p3, %r16, 0f4B000000;
-; CHECK-NEXT: selp.f32 %r17, %r11, %r15, %p3;
-; CHECK-NEXT: cvt.rzi.f32.f32 %r18, %r11;
-; CHECK-NEXT: setp.lt.f32 %p4, %r16, 0f3F000000;
-; CHECK-NEXT: selp.f32 %r19, %r18, %r17, %p4;
-; CHECK-NEXT: cvt.rn.bf16x2.f32 %r20, %r19, %r10;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r20;
+; CHECK-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_round_param_0];
+; CHECK-NEXT: cvt.f32.bf16 %r1, %rs1;
+; CHECK-NEXT: and.b32 %r2, %r1, -2147483648;
+; CHECK-NEXT: or.b32 %r3, %r2, 1056964608;
+; CHECK-NEXT: add.rn.f32 %r4, %r1, %r3;
+; CHECK-NEXT: cvt.rzi.f32.f32 %r5, %r4;
+; CHECK-NEXT: abs.f32 %r6, %r1;
+; CHECK-NEXT: setp.gt.f32 %p1, %r6, 0f4B000000;
+; CHECK-NEXT: selp.f32 %r7, %r1, %r5, %p1;
+; CHECK-NEXT: cvt.rzi.f32.f32 %r8, %r1;
+; CHECK-NEXT: setp.lt.f32 %p2, %r6, 0f3F000000;
+; CHECK-NEXT: selp.f32 %r9, %r8, %r7, %p2;
+; CHECK-NEXT: cvt.f32.bf16 %r10, %rs2;
+; CHECK-NEXT: and.b32 %r11, %r10, -2147483648;
+; CHECK-NEXT: or.b32 %r12, %r11, 1056964608;
+; CHECK-NEXT: add.rn.f32 %r13, %r10, %r12;
+; CHECK-NEXT: cvt.rzi.f32.f32 %r14, %r13;
+; CHECK-NEXT: abs.f32 %r15, %r10;
+; CHECK-NEXT: setp.gt.f32 %p3, %r15, 0f4B000000;
+; CHECK-NEXT: selp.f32 %r16, %r10, %r14, %p3;
+; CHECK-NEXT: cvt.rzi.f32.f32 %r17, %r10;
+; CHECK-NEXT: setp.lt.f32 %p4, %r15, 0f3F000000;
+; CHECK-NEXT: selp.f32 %r18, %r17, %r16, %p4;
+; CHECK-NEXT: cvt.rn.bf16x2.f32 %r19, %r18, %r9;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r19;
; CHECK-NEXT: ret;
%r = call <2 x bfloat> @llvm.round.f16(<2 x bfloat> %a)
ret <2 x bfloat> %r
diff --git a/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll
index 8c89f82dbf9c1..fc7f53c5fdca3 100644
--- a/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll
@@ -420,14 +420,14 @@ define void @test_ldst_v3f16(ptr %a, ptr %b) {
define void @test_ldst_v4f16(ptr %a, ptr %b) {
; CHECK-LABEL: test_ldst_v4f16(
; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd2, [test_ldst_v4f16_param_1];
; CHECK-NEXT: ld.param.b64 %rd1, [test_ldst_v4f16_param_0];
-; CHECK-NEXT: ld.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [%rd1];
-; CHECK-NEXT: st.v4.b16 [%rd2], {%rs1, %rs2, %rs3, %rs4};
+; CHECK-NEXT: ld.v2.b32 {%r1, %r2}, [%rd1];
+; CHECK-NEXT: st.v2.b32 [%rd2], {%r1, %r2};
; CHECK-NEXT: ret;
%t1 = load <4 x half>, ptr %a
store <4 x half> %t1, ptr %b, align 16
@@ -562,7 +562,7 @@ define <2 x half> @test_select_cc(<2 x half> %a, <2 x half> %b, <2 x half> %c, <
; CHECK-F16: {
; CHECK-F16-NEXT: .reg .pred %p<3>;
; CHECK-F16-NEXT: .reg .b16 %rs<7>;
-; CHECK-F16-NEXT: .reg .b32 %r<6>;
+; CHECK-F16-NEXT: .reg .b32 %r<5>;
; CHECK-F16-EMPTY:
; CHECK-F16-NEXT: // %bb.0:
; CHECK-F16-NEXT: ld.param.b32 %r4, [test_select_cc_param_3];
@@ -574,15 +574,14 @@ define <2 x half> @test_select_cc(<2 x half> %a, <2 x half> %b, <2 x half> %c, <
; CHECK-F16-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; CHECK-F16-NEXT: selp.b16 %rs5, %rs4, %rs2, %p2;
; CHECK-F16-NEXT: selp.b16 %rs6, %rs3, %rs1, %p1;
-; CHECK-F16-NEXT: mov.b32 %r5, {%rs6, %rs5};
-; CHECK-F16-NEXT: st.param.b32 [func_retval0], %r5;
+; CHECK-F16-NEXT: st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; CHECK-F16-NEXT: ret;
;
; CHECK-NOF16-LABEL: test_select_cc(
; CHECK-NOF16: {
; CHECK-NOF16-NEXT: .reg .pred %p<3>;
; CHECK-NOF16-NEXT: .reg .b16 %rs<11>;
-; CHECK-NOF16-NEXT: .reg .b32 %r<10>;
+; CHECK-NOF16-NEXT: .reg .b32 %r<9>;
; CHECK-NOF16-EMPTY:
; CHECK-NOF16-NEXT: // %bb.0:
; CHECK-NOF16-NEXT: ld.param.b32 %r4, [test_select_cc_param_3];
@@ -601,8 +600,7 @@ define <2 x half> @test_select_cc(<2 x half> %a, <2 x half> %b, <2 x half> %c, <
; CHECK-NOF16-NEXT: mov.b32 {%rs7, %rs8}, %r1;
; CHECK-NOF16-NEXT: selp.b16 %rs9, %rs8, %rs6, %p2;
; CHECK-NOF16-NEXT: selp.b16 %rs10, %rs7, %rs5, %p1;
-; CHECK-NOF16-NEXT: mov.b32 %r9, {%rs10, %rs9};
-; CHECK-NOF16-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-NOF16-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs9};
; CHECK-NOF16-NEXT: ret;
%cc = fcmp une <2 x half> %c, %d
%r = select <2 x i1> %cc, <2 x half> %a, <2 x half> %b
@@ -660,7 +658,7 @@ define <2 x half> @test_select_cc_f16_f32(<2 x half> %a, <2 x half> %b,
; CHECK: {
; CHECK-NEXT: .reg .pred %p<3>;
; CHECK-NEXT: .reg .b16 %rs<7>;
-; CHECK-NEXT: .reg .b32 %r<8>;
+; CHECK-NEXT: .reg .b32 %r<7>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.v2.b32 {%r5, %r6}, [test_select_cc_f16_f32_param_3];
@@ -673,8 +671,7 @@ define <2 x half> @test_select_cc_f16_f32(<2 x half> %a, <2 x half> %b,
; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; CHECK-NEXT: selp.b16 %rs5, %rs4, %rs2, %p2;
; CHECK-NEXT: selp.b16 %rs6, %rs3, %rs1, %p1;
-; CHECK-NEXT: mov.b32 %r7, {%rs6, %rs5};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r7;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; CHECK-NEXT: ret;
<2 x float> %c, <2 x float> %d) #0 {
%cc = fcmp une <2 x float> %c, %d
@@ -1359,14 +1356,13 @@ define <2 x half> @test_uitofp_2xi32(<2 x i32> %a) #0 {
; CHECK-LABEL: test_uitofp_2xi32(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.v2.b32 {%r1, %r2}, [test_uitofp_2xi32_param_0];
; CHECK-NEXT: cvt.rn.f16.u32 %rs1, %r2;
; CHECK-NEXT: cvt.rn.f16.u32 %rs2, %r1;
-; CHECK-NEXT: mov.b32 %r3, {%rs2, %rs1};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs2, %rs1};
; CHECK-NEXT: ret;
%r = uitofp <2 x i32> %a to <2 x half>
ret <2 x half> %r
@@ -1376,15 +1372,13 @@ define <2 x half> @test_uitofp_2xi64(<2 x i64> %a) #0 {
; CHECK-LABEL: test_uitofp_2xi64(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [test_uitofp_2xi64_param_0];
; CHECK-NEXT: cvt.rn.f16.u64 %rs1, %rd2;
; CHECK-NEXT: cvt.rn.f16.u64 %rs2, %rd1;
-; CHECK-NEXT: mov.b32 %r1, {%rs2, %rs1};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs2, %rs1};
; CHECK-NEXT: ret;
%r = uitofp <2 x i64> %a to <2 x half>
ret <2 x half> %r
@@ -1394,14 +1388,13 @@ define <2 x half> @test_sitofp_2xi32(<2 x i32> %a) #0 {
; CHECK-LABEL: test_sitofp_2xi32(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.v2.b32 {%r1, %r2}, [test_sitofp_2xi32_param_0];
; CHECK-NEXT: cvt.rn.f16.s32 %rs1, %r2;
; CHECK-NEXT: cvt.rn.f16.s32 %rs2, %r1;
-; CHECK-NEXT: mov.b32 %r3, {%rs2, %rs1};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs2, %rs1};
; CHECK-NEXT: ret;
%r = sitofp <2 x i32> %a to <2 x half>
ret <2 x half> %r
@@ -1411,15 +1404,13 @@ define <2 x half> @test_sitofp_2xi64(<2 x i64> %a) #0 {
; CHECK-LABEL: test_sitofp_2xi64(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-NEXT: .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [test_sitofp_2xi64_param_0];
; CHECK-NEXT: cvt.rn.f16.s64 %rs1, %rd2;
; CHECK-NEXT: cvt.rn.f16.s64 %rs2, %rd1;
-; CHECK-NEXT: mov.b32 %r1, {%rs2, %rs1};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs2, %rs1};
; CHECK-NEXT: ret;
%r = sitofp <2 x i64> %a to <2 x half>
ret <2 x half> %r
@@ -1918,7 +1909,7 @@ define <2 x half> @test_copysign(<2 x half> %a, <2 x half> %b) #0 {
; CHECK-NOF16-LABEL: test_copysign(
; CHECK-NOF16: {
; CHECK-NOF16-NEXT: .reg .b16 %rs<11>;
-; CHECK-NOF16-NEXT: .reg .b32 %r<4>;
+; CHECK-NOF16-NEXT: .reg .b32 %r<3>;
; CHECK-NOF16-EMPTY:
; CHECK-NOF16-NEXT: // %bb.0:
; CHECK-NOF16-NEXT: ld.param.b32 %r2, [test_copysign_param_1];
@@ -1931,8 +1922,7 @@ define <2 x half> @test_copysign(<2 x half> %a, <2 x half> %b) #0 {
; CHECK-NOF16-NEXT: and.b16 %rs8, %rs1, -32768;
; CHECK-NOF16-NEXT: and.b16 %rs9, %rs4, 32767;
; CHECK-NOF16-NEXT: or.b16 %rs10, %rs9, %rs8;
-; CHECK-NOF16-NEXT: mov.b32 %r3, {%rs10, %rs7};
-; CHECK-NOF16-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NOF16-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs7};
; CHECK-NOF16-NEXT: ret;
%r = call <2 x half> @llvm.copysign.f16(<2 x half> %a, <2 x half> %b)
ret <2 x half> %r
@@ -1959,7 +1949,7 @@ define <2 x half> @test_copysign_f32(<2 x half> %a, <2 x float> %b) #0 {
; CHECK-NOF16-LABEL: test_copysign_f32(
; CHECK-NOF16: {
; CHECK-NOF16-NEXT: .reg .b16 %rs<9>;
-; CHECK-NOF16-NEXT: .reg .b32 %r<7>;
+; CHECK-NOF16-NEXT: .reg .b32 %r<6>;
; CHECK-NOF16-EMPTY:
; CHECK-NOF16-NEXT: // %bb.0:
; CHECK-NOF16-NEXT: ld.param.v2.b32 {%r2, %r3}, [test_copysign_f32_param_1];
@@ -1973,8 +1963,7 @@ define <2 x half> @test_copysign_f32(<2 x half> %a, <2 x float> %b) #0 {
; CHECK-NOF16-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs6}, %r5; }
; CHECK-NOF16-NEXT: and.b16 %rs7, %rs2, 32767;
; CHECK-NOF16-NEXT: or.b16 %rs8, %rs7, %rs6;
-; CHECK-NOF16-NEXT: mov.b32 %r6, {%rs8, %rs5};
-; CHECK-NOF16-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-NOF16-NEXT: st.param.v2.b16 [func_retval0], {%rs8, %rs5};
; CHECK-NOF16-NEXT: ret;
%tb = fptrunc <2 x float> %b to <2 x half>
%r = call <2 x half> @llvm.copysign.f16(<2 x half> %a, <2 x half> %tb)
@@ -2003,7 +1992,7 @@ define <2 x half> @test_copysign_f64(<2 x half> %a, <2 x double> %b) #0 {
; CHECK-NOF16-LABEL: test_copysign_f64(
; CHECK-NOF16: {
; CHECK-NOF16-NEXT: .reg .b16 %rs<9>;
-; CHECK-NOF16-NEXT: .reg .b32 %r<3>;
+; CHECK-NOF16-NEXT: .reg .b32 %r<2>;
; CHECK-NOF16-NEXT: .reg .b64 %rd<7>;
; CHECK-NOF16-EMPTY:
; CHECK-NOF16-NEXT: // %bb.0:
@@ -2020,8 +2009,7 @@ define <2 x half> @test_copysign_f64(<2 x half> %a, <2 x double> %b) #0 {
; CHECK-NOF16-NEXT: shr.u64 %rd6, %rd5, 48;
; CHECK-NOF16-NEXT: cvt.u16.u64 %rs7, %rd6;
; CHECK-NOF16-NEXT: or.b16 %rs8, %rs6, %rs7;
-; CHECK-NOF16-NEXT: mov.b32 %r2, {%rs8, %rs5};
-; CHECK-NOF16-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NOF16-NEXT: st.param.v2.b16 [func_retval0], {%rs8, %rs5};
; CHECK-NOF16-NEXT: ret;
%tb = fptrunc <2 x double> %b to <2 x half>
%r = call <2 x half> @llvm.copysign.f16(<2 x half> %a, <2 x half> %tb)
@@ -2075,15 +2063,14 @@ define <2 x half> @test_floor(<2 x half> %a) #0 {
; CHECK-LABEL: test_floor(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_floor_param_0];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; CHECK-NEXT: cvt.rmi.f16.f16 %rs3, %rs2;
; CHECK-NEXT: cvt.rmi.f16.f16 %rs4, %rs1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-NEXT: ret;
%r = call <2 x half> @llvm.floor.f16(<2 x half> %a)
ret <2 x half> %r
@@ -2093,15 +2080,14 @@ define <2 x half> @test_ceil(<2 x half> %a) #0 {
; CHECK-LABEL: test_ceil(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_ceil_param_0];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; CHECK-NEXT: cvt.rpi.f16.f16 %rs3, %rs2;
; CHECK-NEXT: cvt.rpi.f16.f16 %rs4, %rs1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-NEXT: ret;
%r = call <2 x half> @llvm.ceil.f16(<2 x half> %a)
ret <2 x half> %r
@@ -2111,15 +2097,14 @@ define <2 x half> @test_trunc(<2 x half> %a) #0 {
; CHECK-LABEL: test_trunc(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_trunc_param_0];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; CHECK-NEXT: cvt.rzi.f16.f16 %rs3, %rs2;
; CHECK-NEXT: cvt.rzi.f16.f16 %rs4, %rs1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-NEXT: ret;
%r = call <2 x half> @llvm.trunc.f16(<2 x half> %a)
ret <2 x half> %r
@@ -2129,15 +2114,14 @@ define <2 x half> @test_rint(<2 x half> %a) #0 {
; CHECK-LABEL: test_rint(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_rint_param_0];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; CHECK-NEXT: cvt.rni.f16.f16 %rs3, %rs2;
; CHECK-NEXT: cvt.rni.f16.f16 %rs4, %rs1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-NEXT: ret;
%r = call <2 x half> @llvm.rint.f16(<2 x half> %a)
ret <2 x half> %r
@@ -2147,15 +2131,14 @@ define <2 x half> @test_nearbyint(<2 x half> %a) #0 {
; CHECK-LABEL: test_nearbyint(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_nearbyint_param_0];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; CHECK-NEXT: cvt.rni.f16.f16 %rs3, %rs2;
; CHECK-NEXT: cvt.rni.f16.f16 %rs4, %rs1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-NEXT: ret;
%r = call <2 x half> @llvm.nearbyint.f16(<2 x half> %a)
ret <2 x half> %r
@@ -2165,15 +2148,14 @@ define <2 x half> @test_roundeven(<2 x half> %a) #0 {
; CHECK-LABEL: test_roundeven(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_roundeven_param_0];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; CHECK-NEXT: cvt.rni.f16.f16 %rs3, %rs2;
; CHECK-NEXT: cvt.rni.f16.f16 %rs4, %rs1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-NEXT: ret;
%r = call <2 x half> @llvm.roundeven.f16(<2 x half> %a)
ret <2 x half> %r
@@ -2267,13 +2249,12 @@ define <2 x half> @test_shufflevector(<2 x half> %a) #0 {
; CHECK-LABEL: test_shufflevector(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_shufflevector_param_0];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: mov.b32 %r2, {%rs2, %rs1};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs2, %rs1};
; CHECK-NEXT: ret;
%s = shufflevector <2 x half> %a, <2 x half> undef, <2 x i32> <i32 1, i32 0>
ret <2 x half> %s
@@ -2283,14 +2264,13 @@ define <2 x half> @test_insertelement(<2 x half> %a, half %x) #0 {
; CHECK-LABEL: test_insertelement(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b16 %rs1, [test_insertelement_param_1];
; CHECK-NEXT: ld.param.b32 %r1, [test_insertelement_param_0];
; CHECK-NEXT: { .reg .b16 tmp; mov.b32 {%rs2, tmp}, %r1; }
-; CHECK-NEXT: mov.b32 %r2, {%rs2, %rs1};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs2, %rs1};
; CHECK-NEXT: ret;
%i = insertelement <2 x half> %a, half %x, i64 1
ret <2 x half> %i
@@ -2300,15 +2280,14 @@ define <2 x half> @test_sitofp_2xi16_to_2xhalf(<2 x i16> %a) #0 {
; CHECK-LABEL: test_sitofp_2xi16_to_2xhalf(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_sitofp_2xi16_to_2xhalf_param_0];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; CHECK-NEXT: cvt.rn.f16.s16 %rs3, %rs2;
; CHECK-NEXT: cvt.rn.f16.s16 %rs4, %rs1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-NEXT: ret;
%r = sitofp <2 x i16> %a to <2 x half>
ret <2 x half> %r
@@ -2318,15 +2297,14 @@ define <2 x half> @test_uitofp_2xi16_to_2xhalf(<2 x i16> %a) #0 {
; CHECK-LABEL: test_uitofp_2xi16_to_2xhalf(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [test_uitofp_2xi16_to_2xhalf_param_0];
; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; CHECK-NEXT: cvt.rn.f16.u16 %rs3, %rs2;
; CHECK-NEXT: cvt.rn.f16.u16 %rs4, %rs1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-NEXT: ret;
%r = uitofp <2 x i16> %a to <2 x half>
ret <2 x half> %r
diff --git a/llvm/test/CodeGen/NVPTX/fexp2.ll b/llvm/test/CodeGen/NVPTX/fexp2.ll
index ef2a788bb8267..7c5e2f83d62c8 100644
--- a/llvm/test/CodeGen/NVPTX/fexp2.ll
+++ b/llvm/test/CodeGen/NVPTX/fexp2.ll
@@ -206,19 +206,18 @@ define <2 x half> @exp2_f16_test_v(<2 x half> %in) {
; CHECK-LABEL: exp2_f16_test_v(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b32 %r<6>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0: // %entry
-; CHECK-NEXT: ld.param.b32 %r1, [exp2_f16_test_v_param_0];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.f32.f16 %r2, %rs2;
-; CHECK-NEXT: ex2.approx.f32 %r3, %r2;
-; CHECK-NEXT: cvt.rn.f16.f32 %rs3, %r3;
-; CHECK-NEXT: cvt.f32.f16 %r4, %rs1;
-; CHECK-NEXT: ex2.approx.f32 %r5, %r4;
-; CHECK-NEXT: cvt.rn.f16.f32 %rs4, %r5;
-; CHECK-NEXT: mov.b32 %r6, {%rs4, %rs3};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [exp2_f16_test_v_param_0];
+; CHECK-NEXT: cvt.f32.f16 %r1, %rs2;
+; CHECK-NEXT: ex2.approx.f32 %r2, %r1;
+; CHECK-NEXT: cvt.rn.f16.f32 %rs3, %r2;
+; CHECK-NEXT: cvt.f32.f16 %r3, %rs1;
+; CHECK-NEXT: ex2.approx.f32 %r4, %r3;
+; CHECK-NEXT: cvt.rn.f16.f32 %rs4, %r4;
+; CHECK-NEXT: mov.b32 %r5, {%rs4, %rs3};
+; CHECK-NEXT: st.param.b32 [func_retval0], %r5;
; CHECK-NEXT: ret;
;
; CHECK-FP16-LABEL: exp2_f16_test_v(
@@ -310,62 +309,60 @@ define <2 x bfloat> @exp2_bf16_test_v(<2 x bfloat> %in) {
; CHECK: {
; CHECK-NEXT: .reg .pred %p<3>;
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<19>;
+; CHECK-NEXT: .reg .b32 %r<18>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0: // %entry
-; CHECK-NEXT: ld.param.b32 %r1, [exp2_bf16_test_v_param_0];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.u32.u16 %r2, %rs2;
-; CHECK-NEXT: shl.b32 %r3, %r2, 16;
-; CHECK-NEXT: ex2.approx.f32 %r4, %r3;
-; CHECK-NEXT: bfe.u32 %r5, %r4, 16, 1;
-; CHECK-NEXT: add.s32 %r6, %r5, %r4;
-; CHECK-NEXT: add.s32 %r7, %r6, 32767;
-; CHECK-NEXT: setp.nan.f32 %p1, %r4, %r4;
-; CHECK-NEXT: or.b32 %r8, %r4, 4194304;
-; CHECK-NEXT: selp.b32 %r9, %r8, %r7, %p1;
-; CHECK-NEXT: cvt.u32.u16 %r10, %rs1;
-; CHECK-NEXT: shl.b32 %r11, %r10, 16;
-; CHECK-NEXT: ex2.approx.f32 %r12, %r11;
-; CHECK-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; CHECK-NEXT: add.s32 %r14, %r13, %r12;
-; CHECK-NEXT: add.s32 %r15, %r14, 32767;
-; CHECK-NEXT: setp.nan.f32 %p2, %r12, %r12;
-; CHECK-NEXT: or.b32 %r16, %r12, 4194304;
-; CHECK-NEXT: selp.b32 %r17, %r16, %r15, %p2;
-; CHECK-NEXT: prmt.b32 %r18, %r17, %r9, 0x7632U;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r18;
+; CHECK-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [exp2_bf16_test_v_param_0];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs2;
+; CHECK-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-NEXT: ex2.approx.f32 %r3, %r2;
+; CHECK-NEXT: bfe.u32 %r4, %r3, 16, 1;
+; CHECK-NEXT: add.s32 %r5, %r4, %r3;
+; CHECK-NEXT: add.s32 %r6, %r5, 32767;
+; CHECK-NEXT: setp.nan.f32 %p1, %r3, %r3;
+; CHECK-NEXT: or.b32 %r7, %r3, 4194304;
+; CHECK-NEXT: selp.b32 %r8, %r7, %r6, %p1;
+; CHECK-NEXT: cvt.u32.u16 %r9, %rs1;
+; CHECK-NEXT: shl.b32 %r10, %r9, 16;
+; CHECK-NEXT: ex2.approx.f32 %r11, %r10;
+; CHECK-NEXT: bfe.u32 %r12, %r11, 16, 1;
+; CHECK-NEXT: add.s32 %r13, %r12, %r11;
+; CHECK-NEXT: add.s32 %r14, %r13, 32767;
+; CHECK-NEXT: setp.nan.f32 %p2, %r11, %r11;
+; CHECK-NEXT: or.b32 %r15, %r11, 4194304;
+; CHECK-NEXT: selp.b32 %r16, %r15, %r14, %p2;
+; CHECK-NEXT: prmt.b32 %r17, %r16, %r8, 0x7632U;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r17;
; CHECK-NEXT: ret;
;
; CHECK-FP16-LABEL: exp2_bf16_test_v(
; CHECK-FP16: {
; CHECK-FP16-NEXT: .reg .pred %p<3>;
; CHECK-FP16-NEXT: .reg .b16 %rs<3>;
-; CHECK-FP16-NEXT: .reg .b32 %r<19>;
+; CHECK-FP16-NEXT: .reg .b32 %r<18>;
; CHECK-FP16-EMPTY:
; CHECK-FP16-NEXT: // %bb.0: // %entry
-; CHECK-FP16-NEXT: ld.param.b32 %r1, [exp2_bf16_test_v_param_0];
-; CHECK-FP16-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-FP16-NEXT: cvt.u32.u16 %r2, %rs2;
-; CHECK-FP16-NEXT: shl.b32 %r3, %r2, 16;
-; CHECK-FP16-NEXT: ex2.approx.f32 %r4, %r3;
-; CHECK-FP16-NEXT: bfe.u32 %r5, %r4, 16, 1;
-; CHECK-FP16-NEXT: add.s32 %r6, %r5, %r4;
-; CHECK-FP16-NEXT: add.s32 %r7, %r6, 32767;
-; CHECK-FP16-NEXT: setp.nan.f32 %p1, %r4, %r4;
-; CHECK-FP16-NEXT: or.b32 %r8, %r4, 4194304;
-; CHECK-FP16-NEXT: selp.b32 %r9, %r8, %r7, %p1;
-; CHECK-FP16-NEXT: cvt.u32.u16 %r10, %rs1;
-; CHECK-FP16-NEXT: shl.b32 %r11, %r10, 16;
-; CHECK-FP16-NEXT: ex2.approx.f32 %r12, %r11;
-; CHECK-FP16-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; CHECK-FP16-NEXT: add.s32 %r14, %r13, %r12;
-; CHECK-FP16-NEXT: add.s32 %r15, %r14, 32767;
-; CHECK-FP16-NEXT: setp.nan.f32 %p2, %r12, %r12;
-; CHECK-FP16-NEXT: or.b32 %r16, %r12, 4194304;
-; CHECK-FP16-NEXT: selp.b32 %r17, %r16, %r15, %p2;
-; CHECK-FP16-NEXT: prmt.b32 %r18, %r17, %r9, 0x7632U;
-; CHECK-FP16-NEXT: st.param.b32 [func_retval0], %r18;
+; CHECK-FP16-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [exp2_bf16_test_v_param_0];
+; CHECK-FP16-NEXT: cvt.u32.u16 %r1, %rs2;
+; CHECK-FP16-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-FP16-NEXT: ex2.approx.f32 %r3, %r2;
+; CHECK-FP16-NEXT: bfe.u32 %r4, %r3, 16, 1;
+; CHECK-FP16-NEXT: add.s32 %r5, %r4, %r3;
+; CHECK-FP16-NEXT: add.s32 %r6, %r5, 32767;
+; CHECK-FP16-NEXT: setp.nan.f32 %p1, %r3, %r3;
+; CHECK-FP16-NEXT: or.b32 %r7, %r3, 4194304;
+; CHECK-FP16-NEXT: selp.b32 %r8, %r7, %r6, %p1;
+; CHECK-FP16-NEXT: cvt.u32.u16 %r9, %rs1;
+; CHECK-FP16-NEXT: shl.b32 %r10, %r9, 16;
+; CHECK-FP16-NEXT: ex2.approx.f32 %r11, %r10;
+; CHECK-FP16-NEXT: bfe.u32 %r12, %r11, 16, 1;
+; CHECK-FP16-NEXT: add.s32 %r13, %r12, %r11;
+; CHECK-FP16-NEXT: add.s32 %r14, %r13, 32767;
+; CHECK-FP16-NEXT: setp.nan.f32 %p2, %r11, %r11;
+; CHECK-FP16-NEXT: or.b32 %r15, %r11, 4194304;
+; CHECK-FP16-NEXT: selp.b32 %r16, %r15, %r14, %p2;
+; CHECK-FP16-NEXT: prmt.b32 %r17, %r16, %r8, 0x7632U;
+; CHECK-FP16-NEXT: st.param.b32 [func_retval0], %r17;
; CHECK-FP16-NEXT: ret;
;
; CHECK-BF16-LABEL: exp2_bf16_test_v(
diff --git a/llvm/test/CodeGen/NVPTX/flog2.ll b/llvm/test/CodeGen/NVPTX/flog2.ll
index 7a5b1bb0ddef6..890c2f8a2678d 100644
--- a/llvm/test/CodeGen/NVPTX/flog2.ll
+++ b/llvm/test/CodeGen/NVPTX/flog2.ll
@@ -97,19 +97,18 @@ define <2 x half> @log2_f16_test_v(<2 x half> %in) {
; CHECK-LABEL: log2_f16_test_v(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .b32 %r<6>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0: // %entry
-; CHECK-NEXT: ld.param.b32 %r1, [log2_f16_test_v_param_0];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.f32.f16 %r2, %rs2;
-; CHECK-NEXT: lg2.approx.f32 %r3, %r2;
-; CHECK-NEXT: cvt.rn.f16.f32 %rs3, %r3;
-; CHECK-NEXT: cvt.f32.f16 %r4, %rs1;
-; CHECK-NEXT: lg2.approx.f32 %r5, %r4;
-; CHECK-NEXT: cvt.rn.f16.f32 %rs4, %r5;
-; CHECK-NEXT: mov.b32 %r6, {%rs4, %rs3};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [log2_f16_test_v_param_0];
+; CHECK-NEXT: cvt.f32.f16 %r1, %rs2;
+; CHECK-NEXT: lg2.approx.f32 %r2, %r1;
+; CHECK-NEXT: cvt.rn.f16.f32 %rs3, %r2;
+; CHECK-NEXT: cvt.f32.f16 %r3, %rs1;
+; CHECK-NEXT: lg2.approx.f32 %r4, %r3;
+; CHECK-NEXT: cvt.rn.f16.f32 %rs4, %r4;
+; CHECK-NEXT: mov.b32 %r5, {%rs4, %rs3};
+; CHECK-NEXT: st.param.b32 [func_retval0], %r5;
; CHECK-NEXT: ret;
entry:
%log2 = call <2 x half> @llvm.log2.v2f16(<2 x half> %in)
@@ -176,31 +175,30 @@ define <2 x bfloat> @log2_bf16_test_v(<2 x bfloat> %in) {
; CHECK: {
; CHECK-NEXT: .reg .pred %p<3>;
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<19>;
+; CHECK-NEXT: .reg .b32 %r<18>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0: // %entry
-; CHECK-NEXT: ld.param.b32 %r1, [log2_bf16_test_v_param_0];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.u32.u16 %r2, %rs2;
-; CHECK-NEXT: shl.b32 %r3, %r2, 16;
-; CHECK-NEXT: lg2.approx.f32 %r4, %r3;
-; CHECK-NEXT: bfe.u32 %r5, %r4, 16, 1;
-; CHECK-NEXT: add.s32 %r6, %r5, %r4;
-; CHECK-NEXT: add.s32 %r7, %r6, 32767;
-; CHECK-NEXT: setp.nan.f32 %p1, %r4, %r4;
-; CHECK-NEXT: or.b32 %r8, %r4, 4194304;
-; CHECK-NEXT: selp.b32 %r9, %r8, %r7, %p1;
-; CHECK-NEXT: cvt.u32.u16 %r10, %rs1;
-; CHECK-NEXT: shl.b32 %r11, %r10, 16;
-; CHECK-NEXT: lg2.approx.f32 %r12, %r11;
-; CHECK-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; CHECK-NEXT: add.s32 %r14, %r13, %r12;
-; CHECK-NEXT: add.s32 %r15, %r14, 32767;
-; CHECK-NEXT: setp.nan.f32 %p2, %r12, %r12;
-; CHECK-NEXT: or.b32 %r16, %r12, 4194304;
-; CHECK-NEXT: selp.b32 %r17, %r16, %r15, %p2;
-; CHECK-NEXT: prmt.b32 %r18, %r17, %r9, 0x7632U;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r18;
+; CHECK-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [log2_bf16_test_v_param_0];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs2;
+; CHECK-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-NEXT: lg2.approx.f32 %r3, %r2;
+; CHECK-NEXT: bfe.u32 %r4, %r3, 16, 1;
+; CHECK-NEXT: add.s32 %r5, %r4, %r3;
+; CHECK-NEXT: add.s32 %r6, %r5, 32767;
+; CHECK-NEXT: setp.nan.f32 %p1, %r3, %r3;
+; CHECK-NEXT: or.b32 %r7, %r3, 4194304;
+; CHECK-NEXT: selp.b32 %r8, %r7, %r6, %p1;
+; CHECK-NEXT: cvt.u32.u16 %r9, %rs1;
+; CHECK-NEXT: shl.b32 %r10, %r9, 16;
+; CHECK-NEXT: lg2.approx.f32 %r11, %r10;
+; CHECK-NEXT: bfe.u32 %r12, %r11, 16, 1;
+; CHECK-NEXT: add.s32 %r13, %r12, %r11;
+; CHECK-NEXT: add.s32 %r14, %r13, 32767;
+; CHECK-NEXT: setp.nan.f32 %p2, %r11, %r11;
+; CHECK-NEXT: or.b32 %r15, %r11, 4194304;
+; CHECK-NEXT: selp.b32 %r16, %r15, %r14, %p2;
+; CHECK-NEXT: prmt.b32 %r17, %r16, %r8, 0x7632U;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r17;
; CHECK-NEXT: ret;
entry:
%log2 = call <2 x bfloat> @llvm.log2.v2bf16(<2 x bfloat> %in)
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll b/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
index 9c11f169a89df..0bf28a2ba13ba 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
@@ -502,10 +502,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -514,10 +514,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -525,20 +525,19 @@ define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
-; CHECK-SM70-NEXT: .reg .b32 %r<7>;
+; CHECK-SM70-NEXT: .reg .b32 %r<6>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_0];
; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
-; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-SM70-NEXT: mov.b32 %r5, 0;
; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; CHECK-SM70-NEXT: selp.b16 %rs3, %rs2, 0x0000, %p2;
; CHECK-SM70-NEXT: selp.b16 %rs4, %rs1, 0x0000, %p1;
-; CHECK-SM70-NEXT: mov.b32 %r6, {%rs4, %rs3};
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-SM70-NEXT: ret;
%1 = fmul <2 x half> %a, %b
%2 = fadd <2 x half> %1, %c
@@ -554,10 +553,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a
; CHECK-NEXT: .reg .b32 %r<10>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: mov.b32 %r5, 0;
; CHECK-NEXT: max.f16x2 %r6, %r4, %r5;
; CHECK-NEXT: mov.b32 %r7, 1191200512;
@@ -571,10 +570,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a
; CHECK-FTZ-NEXT: .reg .b32 %r<10>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
; CHECK-FTZ-NEXT: max.ftz.f16x2 %r6, %r4, %r5;
; CHECK-FTZ-NEXT: mov.b32 %r7, 1191200512;
@@ -590,10 +589,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a
; CHECK-SM70-NEXT: .reg .b32 %r<10>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-SM70-NEXT: mov.b32 %r5, 0;
; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
@@ -620,10 +619,10 @@ define <2 x half> @fma_f16x2_expanded_unsafe_with_nans(<2 x half> %a, <2 x half>
; CHECK-NEXT: .reg .b32 %r<7>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_unsafe_with_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_0];
-; CHECK-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: mov.b32 %r5, 0;
; CHECK-NEXT: max.f16x2 %r6, %r4, %r5;
; CHECK-NEXT: st.param.b32 [func_retval0], %r6;
@@ -634,10 +633,10 @@ define <2 x half> @fma_f16x2_expanded_unsafe_with_nans(<2 x half> %a, <2 x half>
; CHECK-FTZ-NEXT: .reg .b32 %r<7>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_unsafe_with_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
; CHECK-FTZ-NEXT: max.ftz.f16x2 %r6, %r4, %r5;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r6;
@@ -647,20 +646,19 @@ define <2 x half> @fma_f16x2_expanded_unsafe_with_nans(<2 x half> %a, <2 x half>
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
-; CHECK-SM70-NEXT: .reg .b32 %r<7>;
+; CHECK-SM70-NEXT: .reg .b32 %r<6>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_0];
; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_unsafe_with_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_0];
-; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-SM70-NEXT: mov.b32 %r5, 0;
; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; CHECK-SM70-NEXT: selp.b16 %rs3, %rs2, 0x0000, %p2;
; CHECK-SM70-NEXT: selp.b16 %rs4, %rs1, 0x0000, %p1;
-; CHECK-SM70-NEXT: mov.b32 %r6, {%rs4, %rs3};
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-SM70-NEXT: ret;
%1 = fmul <2 x half> %a, %b
%2 = fadd <2 x half> %1, %c
@@ -675,10 +673,10 @@ define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -687,10 +685,10 @@ define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -700,10 +698,10 @@ define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %
; CHECK-SM70-NEXT: .reg .b32 %r<10>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_0];
; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; CHECK-SM70-NEXT: cvt.f32.f16 %r5, %rs2;
; CHECK-SM70-NEXT: max.f32 %r6, %r5, 0f00000000;
@@ -726,10 +724,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_unsafe_with_nans(<2 x bfloat> %a, <2 x
; CHECK-NEXT: .reg .b32 %r<7>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_unsafe_with_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
-; CHECK-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-NEXT: fma.rn.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: mov.b32 %r5, 0;
; CHECK-NEXT: max.bf16x2 %r6, %r4, %r5;
; CHECK-NEXT: st.param.b32 [func_retval0], %r6;
@@ -740,10 +738,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_unsafe_with_nans(<2 x bfloat> %a, <2 x
; CHECK-FTZ-NEXT: .reg .b32 %r<7>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_unsafe_with_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
; CHECK-FTZ-NEXT: max.bf16x2 %r6, %r4, %r5;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r6;
@@ -753,51 +751,47 @@ define <2 x bfloat> @fma_bf16x2_expanded_unsafe_with_nans(<2 x bfloat> %a, <2 x
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<31>;
+; CHECK-SM70-NEXT: .reg .b32 %r<27>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
-; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_unsafe_with_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
-; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_unsafe_with_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r1, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r3, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT: mov.b32 %r30, {%rs10, %rs9};
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs9};
; CHECK-SM70-NEXT: ret;
%1 = fmul <2 x bfloat> %a, %b
%2 = fadd <2 x bfloat> %1, %c
@@ -812,10 +806,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -824,10 +818,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -835,51 +829,47 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<31>;
+; CHECK-SM70-NEXT: .reg .b32 %r<27>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
-; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
-; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r1, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r3, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT: mov.b32 %r30, {%rs10, %rs9};
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs9};
; CHECK-SM70-NEXT: ret;
%1 = fmul <2 x bfloat> %a, %b
%2 = fadd <2 x bfloat> %1, %c
@@ -895,10 +885,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-NEXT: .reg .b32 %r<11>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: mov.b32 %r5, 0;
; CHECK-NEXT: max.bf16x2 %r6, %r4, %r5;
; CHECK-NEXT: mov.b32 %r7, 1065369472;
@@ -914,10 +904,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-FTZ-NEXT: .reg .b32 %r<24>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
; CHECK-FTZ-NEXT: max.bf16x2 %r6, %r4, %r5;
; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r4;
@@ -948,85 +938,82 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<9>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<61>;
+; CHECK-SM70-NEXT: .reg .b32 %r<58>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
-; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r1, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r3, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT: add.f32 %r30, %r29, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r31, %r30, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r32, %r31, %r30;
-; CHECK-SM70-NEXT: add.s32 %r33, %r32, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r30, %r30;
-; CHECK-SM70-NEXT: or.b32 %r34, %r30, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r35, %r34, %r33, %p5;
-; CHECK-SM70-NEXT: add.f32 %r36, %r28, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r36, %r36;
-; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p6;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r42, %rs10;
-; CHECK-SM70-NEXT: shl.b32 %r43, %r42, 16;
-; CHECK-SM70-NEXT: and.b32 %r44, %r41, -65536;
-; CHECK-SM70-NEXT: add.f32 %r45, %r43, %r44;
-; CHECK-SM70-NEXT: bfe.u32 %r46, %r45, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r47, %r46, %r45;
-; CHECK-SM70-NEXT: add.s32 %r48, %r47, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p7, %r45, %r45;
-; CHECK-SM70-NEXT: or.b32 %r49, %r45, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r50, %r49, %r48, %p7;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r51, %rs9;
-; CHECK-SM70-NEXT: shl.b32 %r52, %r51, 16;
-; CHECK-SM70-NEXT: and.b32 %r53, %r35, -65536;
-; CHECK-SM70-NEXT: add.f32 %r54, %r52, %r53;
-; CHECK-SM70-NEXT: bfe.u32 %r55, %r54, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r56, %r55, %r54;
-; CHECK-SM70-NEXT: add.s32 %r57, %r56, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p8, %r54, %r54;
-; CHECK-SM70-NEXT: or.b32 %r58, %r54, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r59, %r58, %r57, %p8;
-; CHECK-SM70-NEXT: prmt.b32 %r60, %r59, %r50, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r60;
+; CHECK-SM70-NEXT: add.f32 %r27, %r26, 0f40E00000;
+; CHECK-SM70-NEXT: bfe.u32 %r28, %r27, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r29, %r28, %r27;
+; CHECK-SM70-NEXT: add.s32 %r30, %r29, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r27, %r27;
+; CHECK-SM70-NEXT: or.b32 %r31, %r27, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r32, %r31, %r30, %p5;
+; CHECK-SM70-NEXT: add.f32 %r33, %r25, 0f40E00000;
+; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r33, %r33;
+; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p6;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r39, %rs10;
+; CHECK-SM70-NEXT: shl.b32 %r40, %r39, 16;
+; CHECK-SM70-NEXT: and.b32 %r41, %r38, -65536;
+; CHECK-SM70-NEXT: add.f32 %r42, %r40, %r41;
+; CHECK-SM70-NEXT: bfe.u32 %r43, %r42, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r44, %r43, %r42;
+; CHECK-SM70-NEXT: add.s32 %r45, %r44, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p7, %r42, %r42;
+; CHECK-SM70-NEXT: or.b32 %r46, %r42, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r47, %r46, %r45, %p7;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r48, %rs9;
+; CHECK-SM70-NEXT: shl.b32 %r49, %r48, 16;
+; CHECK-SM70-NEXT: and.b32 %r50, %r32, -65536;
+; CHECK-SM70-NEXT: add.f32 %r51, %r49, %r50;
+; CHECK-SM70-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r53, %r52, %r51;
+; CHECK-SM70-NEXT: add.s32 %r54, %r53, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p8, %r51, %r51;
+; CHECK-SM70-NEXT: or.b32 %r55, %r51, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r56, %r55, %r54, %p8;
+; CHECK-SM70-NEXT: prmt.b32 %r57, %r56, %r47, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r57;
; CHECK-SM70-NEXT: ret;
%1 = fmul <2 x bfloat> %a, %b
%2 = fadd <2 x bfloat> %1, %c
@@ -1043,10 +1030,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -1055,10 +1042,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -1066,59 +1053,56 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT: .reg .b32 %r<43>;
+; CHECK-SM70-NEXT: .reg .b32 %r<40>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
-; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT: and.b32 %r28, %r27, -65536;
-; CHECK-SM70-NEXT: max.f32 %r29, %r28, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r30, %r29, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r29;
-; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r29, %r29;
-; CHECK-SM70-NEXT: or.b32 %r33, %r29, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
-; CHECK-SM70-NEXT: and.b32 %r35, %r15, -65536;
-; CHECK-SM70-NEXT: max.f32 %r36, %r35, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r36, %r36;
-; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p4;
-; CHECK-SM70-NEXT: prmt.b32 %r42, %r41, %r34, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r42;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r1, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r3, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT: and.b32 %r25, %r24, -65536;
+; CHECK-SM70-NEXT: max.f32 %r26, %r25, 0f00000000;
+; CHECK-SM70-NEXT: bfe.u32 %r27, %r26, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r26;
+; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r26, %r26;
+; CHECK-SM70-NEXT: or.b32 %r30, %r26, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p3;
+; CHECK-SM70-NEXT: and.b32 %r32, %r12, -65536;
+; CHECK-SM70-NEXT: max.f32 %r33, %r32, 0f00000000;
+; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r33, %r33;
+; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p4;
+; CHECK-SM70-NEXT: prmt.b32 %r39, %r38, %r31, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r39;
; CHECK-SM70-NEXT: ret;
%1 = fmul <2 x bfloat> %a, %b
%2 = fadd <2 x bfloat> %1, %c
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
index c725b797526a3..6f6c5d7340789 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
@@ -364,10 +364,10 @@ define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -376,10 +376,10 @@ define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -387,20 +387,19 @@ define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
-; CHECK-SM70-NEXT: .reg .b32 %r<7>;
+; CHECK-SM70-NEXT: .reg .b32 %r<6>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_0];
; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
-; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-SM70-NEXT: mov.b32 %r5, 0;
; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; CHECK-SM70-NEXT: selp.b16 %rs3, %rs2, 0x0000, %p2;
; CHECK-SM70-NEXT: selp.b16 %rs4, %rs1, 0x0000, %p1;
-; CHECK-SM70-NEXT: mov.b32 %r6, {%rs4, %rs3};
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-SM70-NEXT: ret;
%1 = call <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
%2 = fcmp ogt <2 x half> %1, <half 0.0, half 0.0>
@@ -415,10 +414,10 @@ define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x ha
; CHECK-NEXT: .reg .b32 %r<8>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: mov.b32 %r5, 1191200512;
; CHECK-NEXT: add.f16x2 %r6, %r4, %r5;
; CHECK-NEXT: add.f16x2 %r7, %r6, %r4;
@@ -430,10 +429,10 @@ define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x ha
; CHECK-FTZ-NEXT: .reg .b32 %r<8>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: mov.b32 %r5, 1191200512;
; CHECK-FTZ-NEXT: add.ftz.f16x2 %r6, %r4, %r5;
; CHECK-FTZ-NEXT: add.ftz.f16x2 %r7, %r6, %r4;
@@ -445,10 +444,10 @@ define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x ha
; CHECK-SM70-NEXT: .reg .b32 %r<8>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-SM70-NEXT: mov.b32 %r5, 1191200512;
; CHECK-SM70-NEXT: add.f16x2 %r6, %r4, %r5;
; CHECK-SM70-NEXT: add.f16x2 %r7, %r6, %r4;
@@ -468,10 +467,10 @@ define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x h
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -480,10 +479,10 @@ define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x h
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -493,10 +492,10 @@ define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x h
; CHECK-SM70-NEXT: .reg .b32 %r<10>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_0];
; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; CHECK-SM70-NEXT: cvt.f32.f16 %r5, %rs2;
; CHECK-SM70-NEXT: max.f32 %r6, %r5, 0f00000000;
@@ -518,10 +517,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -530,10 +529,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -541,51 +540,47 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<31>;
+; CHECK-SM70-NEXT: .reg .b32 %r<27>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
-; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
-; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_no_nans_param_2];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r1, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_no_nans_param_1];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r3, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT: mov.b32 %r30, {%rs10, %rs9};
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs9};
; CHECK-SM70-NEXT: ret;
%1 = call <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
%2 = fcmp ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
@@ -600,10 +595,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-NEXT: .reg .b32 %r<9>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: mov.b32 %r5, 1065369472;
; CHECK-NEXT: mov.b32 %r6, 1088438496;
; CHECK-NEXT: fma.rn.bf16x2 %r7, %r4, %r5, %r6;
@@ -617,10 +612,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-FTZ-NEXT: .reg .b32 %r<18>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; CHECK-FTZ-NEXT: cvt.u32.u16 %r5, %rs2;
; CHECK-FTZ-NEXT: shl.b32 %r6, %r5, 16;
@@ -644,75 +639,72 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<7>;
; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT: .reg .b32 %r<57>;
+; CHECK-SM70-NEXT: .reg .b32 %r<54>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
-; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT: and.b32 %r28, %r27, -65536;
-; CHECK-SM70-NEXT: add.f32 %r29, %r28, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r30, %r29, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r29;
-; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r29, %r29;
-; CHECK-SM70-NEXT: or.b32 %r33, %r29, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
-; CHECK-SM70-NEXT: and.b32 %r35, %r15, -65536;
-; CHECK-SM70-NEXT: add.f32 %r36, %r35, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r36, %r36;
-; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p4;
-; CHECK-SM70-NEXT: and.b32 %r42, %r41, -65536;
-; CHECK-SM70-NEXT: add.f32 %r43, %r42, %r35;
-; CHECK-SM70-NEXT: bfe.u32 %r44, %r43, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r45, %r44, %r43;
-; CHECK-SM70-NEXT: add.s32 %r46, %r45, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r43, %r43;
-; CHECK-SM70-NEXT: or.b32 %r47, %r43, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r48, %r47, %r46, %p5;
-; CHECK-SM70-NEXT: and.b32 %r49, %r34, -65536;
-; CHECK-SM70-NEXT: add.f32 %r50, %r49, %r28;
-; CHECK-SM70-NEXT: bfe.u32 %r51, %r50, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r52, %r51, %r50;
-; CHECK-SM70-NEXT: add.s32 %r53, %r52, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r50, %r50;
-; CHECK-SM70-NEXT: or.b32 %r54, %r50, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r55, %r54, %r53, %p6;
-; CHECK-SM70-NEXT: prmt.b32 %r56, %r55, %r48, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r56;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r1, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r3, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT: and.b32 %r25, %r24, -65536;
+; CHECK-SM70-NEXT: add.f32 %r26, %r25, 0f40E00000;
+; CHECK-SM70-NEXT: bfe.u32 %r27, %r26, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r26;
+; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r26, %r26;
+; CHECK-SM70-NEXT: or.b32 %r30, %r26, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p3;
+; CHECK-SM70-NEXT: and.b32 %r32, %r12, -65536;
+; CHECK-SM70-NEXT: add.f32 %r33, %r32, 0f40E00000;
+; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r33, %r33;
+; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p4;
+; CHECK-SM70-NEXT: and.b32 %r39, %r38, -65536;
+; CHECK-SM70-NEXT: add.f32 %r40, %r39, %r32;
+; CHECK-SM70-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r42, %r41, %r40;
+; CHECK-SM70-NEXT: add.s32 %r43, %r42, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r40, %r40;
+; CHECK-SM70-NEXT: or.b32 %r44, %r40, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r45, %r44, %r43, %p5;
+; CHECK-SM70-NEXT: and.b32 %r46, %r31, -65536;
+; CHECK-SM70-NEXT: add.f32 %r47, %r46, %r25;
+; CHECK-SM70-NEXT: bfe.u32 %r48, %r47, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r49, %r48, %r47;
+; CHECK-SM70-NEXT: add.s32 %r50, %r49, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r47, %r47;
+; CHECK-SM70-NEXT: or.b32 %r51, %r47, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r52, %r51, %r50, %p6;
+; CHECK-SM70-NEXT: prmt.b32 %r53, %r52, %r45, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r53;
; CHECK-SM70-NEXT: ret;
%1 = call <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
%2 = fcmp ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
@@ -728,10 +720,10 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -740,10 +732,10 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -751,59 +743,56 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT: .reg .b32 %r<43>;
+; CHECK-SM70-NEXT: .reg .b32 %r<40>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
-; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT: and.b32 %r28, %r27, -65536;
-; CHECK-SM70-NEXT: max.f32 %r29, %r28, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r30, %r29, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r29;
-; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r29, %r29;
-; CHECK-SM70-NEXT: or.b32 %r33, %r29, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
-; CHECK-SM70-NEXT: and.b32 %r35, %r15, -65536;
-; CHECK-SM70-NEXT: max.f32 %r36, %r35, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r36, %r36;
-; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p4;
-; CHECK-SM70-NEXT: prmt.b32 %r42, %r41, %r34, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r42;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r1, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r3, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT: and.b32 %r25, %r24, -65536;
+; CHECK-SM70-NEXT: max.f32 %r26, %r25, 0f00000000;
+; CHECK-SM70-NEXT: bfe.u32 %r27, %r26, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r26;
+; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r26, %r26;
+; CHECK-SM70-NEXT: or.b32 %r30, %r26, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p3;
+; CHECK-SM70-NEXT: and.b32 %r32, %r12, -65536;
+; CHECK-SM70-NEXT: max.f32 %r33, %r32, 0f00000000;
+; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r33, %r33;
+; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p4;
+; CHECK-SM70-NEXT: prmt.b32 %r39, %r38, %r31, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r39;
; CHECK-SM70-NEXT: ret;
%1 = call <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
%2 = call <2 x bfloat> @llvm.maxnum.bf16x2(<2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll b/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
index 6b462f8468596..60bfe3fa2cbf6 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
@@ -393,10 +393,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -405,10 +405,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -416,20 +416,19 @@ define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
-; CHECK-SM70-NEXT: .reg .b32 %r<7>;
+; CHECK-SM70-NEXT: .reg .b32 %r<6>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_0];
; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
-; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-SM70-NEXT: mov.b32 %r5, 0;
; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; CHECK-SM70-NEXT: selp.b16 %rs3, %rs2, 0x0000, %p2;
; CHECK-SM70-NEXT: selp.b16 %rs4, %rs1, 0x0000, %p1;
-; CHECK-SM70-NEXT: mov.b32 %r6, {%rs4, %rs3};
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-SM70-NEXT: ret;
%1 = fmul fast <2 x half> %a, %b
%2 = fadd fast <2 x half> %1, %c
@@ -447,10 +446,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a
; CHECK-NEXT: .reg .b32 %r<10>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: mov.b32 %r5, 0;
; CHECK-NEXT: max.f16x2 %r6, %r4, %r5;
; CHECK-NEXT: mov.b32 %r7, 1191200512;
@@ -464,10 +463,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a
; CHECK-FTZ-NEXT: .reg .b32 %r<10>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
; CHECK-FTZ-NEXT: max.ftz.f16x2 %r6, %r4, %r5;
; CHECK-FTZ-NEXT: mov.b32 %r7, 1191200512;
@@ -483,10 +482,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a
; CHECK-SM70-NEXT: .reg .b32 %r<10>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-SM70-NEXT: mov.b32 %r5, 0;
; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
@@ -513,10 +512,10 @@ define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -525,10 +524,10 @@ define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -538,10 +537,10 @@ define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %
; CHECK-SM70-NEXT: .reg .b32 %r<10>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_0];
; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; CHECK-SM70-NEXT: cvt.f32.f16 %r5, %rs2;
; CHECK-SM70-NEXT: max.f32 %r6, %r5, 0f00000000;
@@ -564,10 +563,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -576,10 +575,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -587,51 +586,47 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<31>;
+; CHECK-SM70-NEXT: .reg .b32 %r<27>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
-; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
-; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r1, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r3, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT: mov.b32 %r30, {%rs10, %rs9};
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs9};
; CHECK-SM70-NEXT: ret;
%1 = fmul fast <2 x bfloat> %a, %b
%2 = fadd fast <2 x bfloat> %1, %c
@@ -647,10 +642,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-NEXT: .reg .b32 %r<11>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: mov.b32 %r5, 0;
; CHECK-NEXT: max.bf16x2 %r6, %r4, %r5;
; CHECK-NEXT: mov.b32 %r7, 1065369472;
@@ -666,10 +661,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-FTZ-NEXT: .reg .b32 %r<24>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
; CHECK-FTZ-NEXT: max.bf16x2 %r6, %r4, %r5;
; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r4;
@@ -700,85 +695,82 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<9>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<61>;
+; CHECK-SM70-NEXT: .reg .b32 %r<58>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
-; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r1, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r3, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT: add.rn.f32 %r30, %r29, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r31, %r30, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r32, %r31, %r30;
-; CHECK-SM70-NEXT: add.s32 %r33, %r32, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r30, %r30;
-; CHECK-SM70-NEXT: or.b32 %r34, %r30, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r35, %r34, %r33, %p5;
-; CHECK-SM70-NEXT: add.rn.f32 %r36, %r28, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r36, %r36;
-; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p6;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r42, %rs10;
-; CHECK-SM70-NEXT: shl.b32 %r43, %r42, 16;
-; CHECK-SM70-NEXT: and.b32 %r44, %r41, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r45, %r43, %r44;
-; CHECK-SM70-NEXT: bfe.u32 %r46, %r45, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r47, %r46, %r45;
-; CHECK-SM70-NEXT: add.s32 %r48, %r47, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p7, %r45, %r45;
-; CHECK-SM70-NEXT: or.b32 %r49, %r45, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r50, %r49, %r48, %p7;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r51, %rs9;
-; CHECK-SM70-NEXT: shl.b32 %r52, %r51, 16;
-; CHECK-SM70-NEXT: and.b32 %r53, %r35, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r54, %r52, %r53;
-; CHECK-SM70-NEXT: bfe.u32 %r55, %r54, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r56, %r55, %r54;
-; CHECK-SM70-NEXT: add.s32 %r57, %r56, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p8, %r54, %r54;
-; CHECK-SM70-NEXT: or.b32 %r58, %r54, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r59, %r58, %r57, %p8;
-; CHECK-SM70-NEXT: prmt.b32 %r60, %r59, %r50, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r60;
+; CHECK-SM70-NEXT: add.rn.f32 %r27, %r26, 0f40E00000;
+; CHECK-SM70-NEXT: bfe.u32 %r28, %r27, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r29, %r28, %r27;
+; CHECK-SM70-NEXT: add.s32 %r30, %r29, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r27, %r27;
+; CHECK-SM70-NEXT: or.b32 %r31, %r27, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r32, %r31, %r30, %p5;
+; CHECK-SM70-NEXT: add.rn.f32 %r33, %r25, 0f40E00000;
+; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r33, %r33;
+; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p6;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r39, %rs10;
+; CHECK-SM70-NEXT: shl.b32 %r40, %r39, 16;
+; CHECK-SM70-NEXT: and.b32 %r41, %r38, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r42, %r40, %r41;
+; CHECK-SM70-NEXT: bfe.u32 %r43, %r42, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r44, %r43, %r42;
+; CHECK-SM70-NEXT: add.s32 %r45, %r44, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p7, %r42, %r42;
+; CHECK-SM70-NEXT: or.b32 %r46, %r42, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r47, %r46, %r45, %p7;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r48, %rs9;
+; CHECK-SM70-NEXT: shl.b32 %r49, %r48, 16;
+; CHECK-SM70-NEXT: and.b32 %r50, %r32, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r51, %r49, %r50;
+; CHECK-SM70-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r53, %r52, %r51;
+; CHECK-SM70-NEXT: add.s32 %r54, %r53, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p8, %r51, %r51;
+; CHECK-SM70-NEXT: or.b32 %r55, %r51, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r56, %r55, %r54, %p8;
+; CHECK-SM70-NEXT: prmt.b32 %r57, %r56, %r47, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r57;
; CHECK-SM70-NEXT: ret;
%1 = fmul fast <2 x bfloat> %a, %b
%2 = fadd fast <2 x bfloat> %1, %c
@@ -795,10 +787,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -807,10 +799,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -818,59 +810,56 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT: .reg .b32 %r<43>;
+; CHECK-SM70-NEXT: .reg .b32 %r<40>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
-; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT: and.b32 %r28, %r27, -65536;
-; CHECK-SM70-NEXT: max.f32 %r29, %r28, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r30, %r29, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r29;
-; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r29, %r29;
-; CHECK-SM70-NEXT: or.b32 %r33, %r29, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
-; CHECK-SM70-NEXT: and.b32 %r35, %r15, -65536;
-; CHECK-SM70-NEXT: max.f32 %r36, %r35, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r36, %r36;
-; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p4;
-; CHECK-SM70-NEXT: prmt.b32 %r42, %r41, %r34, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r42;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r1, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r3, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT: and.b32 %r25, %r24, -65536;
+; CHECK-SM70-NEXT: max.f32 %r26, %r25, 0f00000000;
+; CHECK-SM70-NEXT: bfe.u32 %r27, %r26, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r26;
+; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r26, %r26;
+; CHECK-SM70-NEXT: or.b32 %r30, %r26, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p3;
+; CHECK-SM70-NEXT: and.b32 %r32, %r12, -65536;
+; CHECK-SM70-NEXT: max.f32 %r33, %r32, 0f00000000;
+; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r33, %r33;
+; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p4;
+; CHECK-SM70-NEXT: prmt.b32 %r39, %r38, %r31, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r39;
; CHECK-SM70-NEXT: ret;
%1 = fmul fast <2 x bfloat> %a, %b
%2 = fadd fast <2 x bfloat> %1, %c
@@ -1233,10 +1222,10 @@ define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c
; CHECK-NEXT: .reg .b32 %r<7>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
-; CHECK-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_2];
+; CHECK-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: mov.b32 %r5, 0;
; CHECK-NEXT: max.f16x2 %r6, %r4, %r5;
; CHECK-NEXT: st.param.b32 [func_retval0], %r6;
@@ -1247,10 +1236,10 @@ define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c
; CHECK-FTZ-NEXT: .reg .b32 %r<7>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
; CHECK-FTZ-NEXT: max.ftz.f16x2 %r6, %r4, %r5;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r6;
@@ -1260,20 +1249,19 @@ define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
-; CHECK-SM70-NEXT: .reg .b32 %r<7>;
+; CHECK-SM70-NEXT: .reg .b32 %r<6>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_0];
; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
-; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-SM70-NEXT: mov.b32 %r5, 0;
; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; CHECK-SM70-NEXT: selp.b16 %rs3, %rs2, 0x0000, %p2;
; CHECK-SM70-NEXT: selp.b16 %rs4, %rs1, 0x0000, %p1;
-; CHECK-SM70-NEXT: mov.b32 %r6, {%rs4, %rs3};
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-SM70-NEXT: ret;
%1 = call <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
%2 = fcmp nsz ogt <2 x half> %1, <half 0.0, half 0.0>
@@ -1288,10 +1276,10 @@ define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x ha
; CHECK-NEXT: .reg .b32 %r<8>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: mov.b32 %r5, 1191200512;
; CHECK-NEXT: add.rn.f16x2 %r6, %r4, %r5;
; CHECK-NEXT: add.rn.f16x2 %r7, %r6, %r4;
@@ -1303,10 +1291,10 @@ define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x ha
; CHECK-FTZ-NEXT: .reg .b32 %r<8>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: mov.b32 %r5, 1191200512;
; CHECK-FTZ-NEXT: add.rn.ftz.f16x2 %r6, %r4, %r5;
; CHECK-FTZ-NEXT: add.rn.ftz.f16x2 %r7, %r6, %r4;
@@ -1318,10 +1306,10 @@ define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x ha
; CHECK-SM70-NEXT: .reg .b32 %r<8>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-SM70-NEXT: mov.b32 %r5, 1191200512;
; CHECK-SM70-NEXT: add.rn.f16x2 %r6, %r4, %r5;
; CHECK-SM70-NEXT: add.rn.f16x2 %r7, %r6, %r4;
@@ -1341,10 +1329,10 @@ define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x h
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -1353,10 +1341,10 @@ define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x h
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -1366,10 +1354,10 @@ define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x h
; CHECK-SM70-NEXT: .reg .b32 %r<10>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_0];
; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r1, %r2, %r3;
; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; CHECK-SM70-NEXT: cvt.f32.f16 %r5, %rs2;
; CHECK-SM70-NEXT: max.f32 %r6, %r5, 0f00000000;
@@ -1391,10 +1379,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -1403,10 +1391,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -1414,51 +1402,47 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<31>;
+; CHECK-SM70-NEXT: .reg .b32 %r<27>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
-; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
-; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_no_nans_param_2];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r1, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_no_nans_param_1];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r3, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT: mov.b32 %r30, {%rs10, %rs9};
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs9};
; CHECK-SM70-NEXT: ret;
%1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
%2 = fcmp nsz ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
@@ -1473,10 +1457,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-NEXT: .reg .b32 %r<9>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: mov.b32 %r5, 1065369472;
; CHECK-NEXT: mov.b32 %r6, 1088438496;
; CHECK-NEXT: fma.rn.bf16x2 %r7, %r4, %r5, %r6;
@@ -1490,10 +1474,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-FTZ-NEXT: .reg .b32 %r<18>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; CHECK-FTZ-NEXT: cvt.u32.u16 %r5, %rs2;
; CHECK-FTZ-NEXT: shl.b32 %r6, %r5, 16;
@@ -1517,75 +1501,72 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<7>;
; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT: .reg .b32 %r<57>;
+; CHECK-SM70-NEXT: .reg .b32 %r<54>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
-; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT: and.b32 %r28, %r27, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r29, %r28, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r30, %r29, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r29;
-; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r29, %r29;
-; CHECK-SM70-NEXT: or.b32 %r33, %r29, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
-; CHECK-SM70-NEXT: and.b32 %r35, %r15, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r36, %r35, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r36, %r36;
-; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p4;
-; CHECK-SM70-NEXT: and.b32 %r42, %r41, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r43, %r42, %r35;
-; CHECK-SM70-NEXT: bfe.u32 %r44, %r43, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r45, %r44, %r43;
-; CHECK-SM70-NEXT: add.s32 %r46, %r45, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r43, %r43;
-; CHECK-SM70-NEXT: or.b32 %r47, %r43, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r48, %r47, %r46, %p5;
-; CHECK-SM70-NEXT: and.b32 %r49, %r34, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r50, %r49, %r28;
-; CHECK-SM70-NEXT: bfe.u32 %r51, %r50, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r52, %r51, %r50;
-; CHECK-SM70-NEXT: add.s32 %r53, %r52, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r50, %r50;
-; CHECK-SM70-NEXT: or.b32 %r54, %r50, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r55, %r54, %r53, %p6;
-; CHECK-SM70-NEXT: prmt.b32 %r56, %r55, %r48, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r56;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r1, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r3, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT: and.b32 %r25, %r24, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r26, %r25, 0f40E00000;
+; CHECK-SM70-NEXT: bfe.u32 %r27, %r26, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r26;
+; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r26, %r26;
+; CHECK-SM70-NEXT: or.b32 %r30, %r26, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p3;
+; CHECK-SM70-NEXT: and.b32 %r32, %r12, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r33, %r32, 0f40E00000;
+; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r33, %r33;
+; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p4;
+; CHECK-SM70-NEXT: and.b32 %r39, %r38, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r40, %r39, %r32;
+; CHECK-SM70-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r42, %r41, %r40;
+; CHECK-SM70-NEXT: add.s32 %r43, %r42, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r40, %r40;
+; CHECK-SM70-NEXT: or.b32 %r44, %r40, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r45, %r44, %r43, %p5;
+; CHECK-SM70-NEXT: and.b32 %r46, %r31, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r47, %r46, %r25;
+; CHECK-SM70-NEXT: bfe.u32 %r48, %r47, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r49, %r48, %r47;
+; CHECK-SM70-NEXT: add.s32 %r50, %r49, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r47, %r47;
+; CHECK-SM70-NEXT: or.b32 %r51, %r47, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r52, %r51, %r50, %p6;
+; CHECK-SM70-NEXT: prmt.b32 %r53, %r52, %r45, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r53;
; CHECK-SM70-NEXT: ret;
%1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
%2 = fcmp nsz ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
@@ -1601,10 +1582,10 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
-; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_0];
-; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
;
@@ -1613,10 +1594,10 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
; CHECK-FTZ-EMPTY:
; CHECK-FTZ-NEXT: // %bb.0:
-; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-FTZ-NEXT: ret;
;
@@ -1624,59 +1605,56 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT: .reg .b32 %r<43>;
+; CHECK-SM70-NEXT: .reg .b32 %r<40>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
-; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
-; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT: and.b32 %r28, %r27, -65536;
-; CHECK-SM70-NEXT: max.f32 %r29, %r28, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r30, %r29, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r29;
-; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r29, %r29;
-; CHECK-SM70-NEXT: or.b32 %r33, %r29, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
-; CHECK-SM70-NEXT: and.b32 %r35, %r15, -65536;
-; CHECK-SM70-NEXT: max.f32 %r36, %r35, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r36, %r36;
-; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p4;
-; CHECK-SM70-NEXT: prmt.b32 %r42, %r41, %r34, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r42;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r1, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: cvt.u32.u16 %r3, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT: and.b32 %r25, %r24, -65536;
+; CHECK-SM70-NEXT: max.f32 %r26, %r25, 0f00000000;
+; CHECK-SM70-NEXT: bfe.u32 %r27, %r26, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r26;
+; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r26, %r26;
+; CHECK-SM70-NEXT: or.b32 %r30, %r26, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p3;
+; CHECK-SM70-NEXT: and.b32 %r32, %r12, -65536;
+; CHECK-SM70-NEXT: max.f32 %r33, %r32, 0f00000000;
+; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r33, %r33;
+; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p4;
+; CHECK-SM70-NEXT: prmt.b32 %r39, %r38, %r31, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r39;
; CHECK-SM70-NEXT: ret;
%1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
%2 = call nsz <2 x bfloat> @llvm.maxnum.bf16x2(<2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
diff --git a/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll
index 5bfa5b2bc63a5..bf1fb06c44688 100644
--- a/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll
@@ -33,35 +33,57 @@ define <2 x i16> @test_ret_const() #0 {
}
define i16 @test_extract_0(<2 x i16> %a) #0 {
-; COMMON-LABEL: test_extract_0(
-; COMMON: {
-; COMMON-NEXT: .reg .b16 %rs<2>;
-; COMMON-NEXT: .reg .b32 %r<3>;
-; COMMON-EMPTY:
-; COMMON-NEXT: // %bb.0:
-; COMMON-NEXT: ld.param.b32 %r1, [test_extract_0_param_0];
+; I16x2-LABEL: test_extract_0(
+; I16x2: {
+; I16x2-NEXT: .reg .b16 %rs<2>;
+; I16x2-NEXT: .reg .b32 %r<3>;
+; I16x2-EMPTY:
+; I16x2-NEXT: // %bb.0:
+; I16x2-NEXT: ld.param.b32 %r1, [test_extract_0_param_0];
; I16x2-NEXT: mov.b32 {%rs1, _}, %r1;
-; NO-I16x2-NEXT: { .reg .b16 tmp; mov.b32 {%rs1, tmp}, %r1; }
-; COMMON-NEXT: cvt.u32.u16 %r2, %rs1;
-; COMMON-NEXT: st.param.b32 [func_retval0], %r2;
-; COMMON-NEXT: ret;
+; I16x2-NEXT: cvt.u32.u16 %r2, %rs1;
+; I16x2-NEXT: st.param.b32 [func_retval0], %r2;
+; I16x2-NEXT: ret;
+;
+; NO-I16x2-LABEL: test_extract_0(
+; NO-I16x2: {
+; NO-I16x2-NEXT: .reg .b16 %rs<2>;
+; NO-I16x2-NEXT: .reg .b32 %r<3>;
+; NO-I16x2-EMPTY:
+; NO-I16x2-NEXT: // %bb.0:
+; NO-I16x2-NEXT: ld.param.b32 %r1, [test_extract_0_param_0];
+; NO-I16x2-NEXT: { .reg .b16 tmp; mov.b32 {%rs1, tmp}, %r1; }
+; NO-I16x2-NEXT: cvt.u32.u16 %r2, %rs1;
+; NO-I16x2-NEXT: st.param.b32 [func_retval0], %r2;
+; NO-I16x2-NEXT: ret;
%e = extractelement <2 x i16> %a, i32 0
ret i16 %e
}
define i16 @test_extract_1(<2 x i16> %a) #0 {
-; COMMON-LABEL: test_extract_1(
-; COMMON: {
-; COMMON-NEXT: .reg .b16 %rs<2>;
-; COMMON-NEXT: .reg .b32 %r<3>;
-; COMMON-EMPTY:
-; COMMON-NEXT: // %bb.0:
-; COMMON-NEXT: ld.param.b32 %r1, [test_extract_1_param_0];
+; I16x2-LABEL: test_extract_1(
+; I16x2: {
+; I16x2-NEXT: .reg .b16 %rs<2>;
+; I16x2-NEXT: .reg .b32 %r<3>;
+; I16x2-EMPTY:
+; I16x2-NEXT: // %bb.0:
+; I16x2-NEXT: ld.param.b32 %r1, [test_extract_1_param_0];
; I16x2-NEXT: mov.b32 {_, %rs1}, %r1;
-; NO-I16x2-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r1; }
-; COMMON-NEXT: cvt.u32.u16 %r2, %rs1;
-; COMMON-NEXT: st.param.b32 [func_retval0], %r2;
-; COMMON-NEXT: ret;
+; I16x2-NEXT: cvt.u32.u16 %r2, %rs1;
+; I16x2-NEXT: st.param.b32 [func_retval0], %r2;
+; I16x2-NEXT: ret;
+;
+; NO-I16x2-LABEL: test_extract_1(
+; NO-I16x2: {
+; NO-I16x2-NEXT: .reg .b16 %rs<2>;
+; NO-I16x2-NEXT: .reg .b32 %r<3>;
+; NO-I16x2-EMPTY:
+; NO-I16x2-NEXT: // %bb.0:
+; NO-I16x2-NEXT: ld.param.b32 %r1, [test_extract_1_param_0];
+; NO-I16x2-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r1; }
+; NO-I16x2-NEXT: cvt.u32.u16 %r2, %rs1;
+; NO-I16x2-NEXT: st.param.b32 [func_retval0], %r2;
+; NO-I16x2-NEXT: ret;
%e = extractelement <2 x i16> %a, i32 1
ret i16 %e
}
@@ -102,7 +124,7 @@ define <2 x i16> @test_add(<2 x i16> %a, <2 x i16> %b) #0 {
; NO-I16x2-LABEL: test_add(
; NO-I16x2: {
; NO-I16x2-NEXT: .reg .b16 %rs<7>;
-; NO-I16x2-NEXT: .reg .b32 %r<4>;
+; NO-I16x2-NEXT: .reg .b32 %r<3>;
; NO-I16x2-EMPTY:
; NO-I16x2-NEXT: // %bb.0:
; NO-I16x2-NEXT: ld.param.b32 %r2, [test_add_param_1];
@@ -111,8 +133,7 @@ define <2 x i16> @test_add(<2 x i16> %a, <2 x i16> %b) #0 {
; NO-I16x2-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; NO-I16x2-NEXT: add.s16 %rs5, %rs4, %rs2;
; NO-I16x2-NEXT: add.s16 %rs6, %rs3, %rs1;
-; NO-I16x2-NEXT: mov.b32 %r3, {%rs6, %rs5};
-; NO-I16x2-NEXT: st.param.b32 [func_retval0], %r3;
+; NO-I16x2-NEXT: st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; NO-I16x2-NEXT: ret;
%r = add <2 x i16> %a, %b
ret <2 x i16> %r
@@ -134,15 +155,14 @@ define <2 x i16> @test_add_imm_0(<2 x i16> %a) #0 {
; NO-I16x2-LABEL: test_add_imm_0(
; NO-I16x2: {
; NO-I16x2-NEXT: .reg .b16 %rs<5>;
-; NO-I16x2-NEXT: .reg .b32 %r<3>;
+; NO-I16x2-NEXT: .reg .b32 %r<2>;
; NO-I16x2-EMPTY:
; NO-I16x2-NEXT: // %bb.0:
; NO-I16x2-NEXT: ld.param.b32 %r1, [test_add_imm_0_param_0];
; NO-I16x2-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; NO-I16x2-NEXT: add.s16 %rs3, %rs2, 2;
; NO-I16x2-NEXT: add.s16 %rs4, %rs1, 1;
-; NO-I16x2-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; NO-I16x2-NEXT: st.param.b32 [func_retval0], %r2;
+; NO-I16x2-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; NO-I16x2-NEXT: ret;
%r = add <2 x i16> <i16 1, i16 2>, %a
ret <2 x i16> %r
@@ -163,15 +183,14 @@ define <2 x i16> @test_add_imm_1(<2 x i16> %a) #0 {
; NO-I16x2-LABEL: test_add_imm_1(
; NO-I16x2: {
; NO-I16x2-NEXT: .reg .b16 %rs<5>;
-; NO-I16x2-NEXT: .reg .b32 %r<3>;
+; NO-I16x2-NEXT: .reg .b32 %r<2>;
; NO-I16x2-EMPTY:
; NO-I16x2-NEXT: // %bb.0:
; NO-I16x2-NEXT: ld.param.b32 %r1, [test_add_imm_1_param_0];
; NO-I16x2-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; NO-I16x2-NEXT: add.s16 %rs3, %rs2, 2;
; NO-I16x2-NEXT: add.s16 %rs4, %rs1, 1;
-; NO-I16x2-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; NO-I16x2-NEXT: st.param.b32 [func_retval0], %r2;
+; NO-I16x2-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; NO-I16x2-NEXT: ret;
%r = add <2 x i16> %a, <i16 1, i16 2>
ret <2 x i16> %r
@@ -181,7 +200,7 @@ define <2 x i16> @test_sub(<2 x i16> %a, <2 x i16> %b) #0 {
; COMMON-LABEL: test_sub(
; COMMON: {
; COMMON-NEXT: .reg .b16 %rs<7>;
-; COMMON-NEXT: .reg .b32 %r<4>;
+; COMMON-NEXT: .reg .b32 %r<3>;
; COMMON-EMPTY:
; COMMON-NEXT: // %bb.0:
; COMMON-NEXT: ld.param.b32 %r2, [test_sub_param_1];
@@ -190,8 +209,7 @@ define <2 x i16> @test_sub(<2 x i16> %a, <2 x i16> %b) #0 {
; COMMON-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; COMMON-NEXT: sub.s16 %rs5, %rs4, %rs2;
; COMMON-NEXT: sub.s16 %rs6, %rs3, %rs1;
-; COMMON-NEXT: mov.b32 %r3, {%rs6, %rs5};
-; COMMON-NEXT: st.param.b32 [func_retval0], %r3;
+; COMMON-NEXT: st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; COMMON-NEXT: ret;
%r = sub <2 x i16> %a, %b
ret <2 x i16> %r
@@ -212,7 +230,7 @@ define <2 x i16> @test_smax(<2 x i16> %a, <2 x i16> %b) #0 {
; NO-I16x2-LABEL: test_smax(
; NO-I16x2: {
; NO-I16x2-NEXT: .reg .b16 %rs<7>;
-; NO-I16x2-NEXT: .reg .b32 %r<4>;
+; NO-I16x2-NEXT: .reg .b32 %r<3>;
; NO-I16x2-EMPTY:
; NO-I16x2-NEXT: // %bb.0:
; NO-I16x2-NEXT: ld.param.b32 %r2, [test_smax_param_1];
@@ -221,8 +239,7 @@ define <2 x i16> @test_smax(<2 x i16> %a, <2 x i16> %b) #0 {
; NO-I16x2-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; NO-I16x2-NEXT: max.s16 %rs5, %rs4, %rs2;
; NO-I16x2-NEXT: max.s16 %rs6, %rs3, %rs1;
-; NO-I16x2-NEXT: mov.b32 %r3, {%rs6, %rs5};
-; NO-I16x2-NEXT: st.param.b32 [func_retval0], %r3;
+; NO-I16x2-NEXT: st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; NO-I16x2-NEXT: ret;
%cmp = icmp sgt <2 x i16> %a, %b
%r = select <2 x i1> %cmp, <2 x i16> %a, <2 x i16> %b
@@ -244,7 +261,7 @@ define <2 x i16> @test_umax(<2 x i16> %a, <2 x i16> %b) #0 {
; NO-I16x2-LABEL: test_umax(
; NO-I16x2: {
; NO-I16x2-NEXT: .reg .b16 %rs<7>;
-; NO-I16x2-NEXT: .reg .b32 %r<4>;
+; NO-I16x2-NEXT: .reg .b32 %r<3>;
; NO-I16x2-EMPTY:
; NO-I16x2-NEXT: // %bb.0:
; NO-I16x2-NEXT: ld.param.b32 %r2, [test_umax_param_1];
@@ -253,8 +270,7 @@ define <2 x i16> @test_umax(<2 x i16> %a, <2 x i16> %b) #0 {
; NO-I16x2-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; NO-I16x2-NEXT: max.u16 %rs5, %rs4, %rs2;
; NO-I16x2-NEXT: max.u16 %rs6, %rs3, %rs1;
-; NO-I16x2-NEXT: mov.b32 %r3, {%rs6, %rs5};
-; NO-I16x2-NEXT: st.param.b32 [func_retval0], %r3;
+; NO-I16x2-NEXT: st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; NO-I16x2-NEXT: ret;
%cmp = icmp ugt <2 x i16> %a, %b
%r = select <2 x i1> %cmp, <2 x i16> %a, <2 x i16> %b
@@ -276,7 +292,7 @@ define <2 x i16> @test_smin(<2 x i16> %a, <2 x i16> %b) #0 {
; NO-I16x2-LABEL: test_smin(
; NO-I16x2: {
; NO-I16x2-NEXT: .reg .b16 %rs<7>;
-; NO-I16x2-NEXT: .reg .b32 %r<4>;
+; NO-I16x2-NEXT: .reg .b32 %r<3>;
; NO-I16x2-EMPTY:
; NO-I16x2-NEXT: // %bb.0:
; NO-I16x2-NEXT: ld.param.b32 %r2, [test_smin_param_1];
@@ -285,8 +301,7 @@ define <2 x i16> @test_smin(<2 x i16> %a, <2 x i16> %b) #0 {
; NO-I16x2-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; NO-I16x2-NEXT: min.s16 %rs5, %rs4, %rs2;
; NO-I16x2-NEXT: min.s16 %rs6, %rs3, %rs1;
-; NO-I16x2-NEXT: mov.b32 %r3, {%rs6, %rs5};
-; NO-I16x2-NEXT: st.param.b32 [func_retval0], %r3;
+; NO-I16x2-NEXT: st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; NO-I16x2-NEXT: ret;
%cmp = icmp sle <2 x i16> %a, %b
%r = select <2 x i1> %cmp, <2 x i16> %a, <2 x i16> %b
@@ -308,7 +323,7 @@ define <2 x i16> @test_umin(<2 x i16> %a, <2 x i16> %b) #0 {
; NO-I16x2-LABEL: test_umin(
; NO-I16x2: {
; NO-I16x2-NEXT: .reg .b16 %rs<7>;
-; NO-I16x2-NEXT: .reg .b32 %r<4>;
+; NO-I16x2-NEXT: .reg .b32 %r<3>;
; NO-I16x2-EMPTY:
; NO-I16x2-NEXT: // %bb.0:
; NO-I16x2-NEXT: ld.param.b32 %r2, [test_umin_param_1];
@@ -317,8 +332,7 @@ define <2 x i16> @test_umin(<2 x i16> %a, <2 x i16> %b) #0 {
; NO-I16x2-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; NO-I16x2-NEXT: min.u16 %rs5, %rs4, %rs2;
; NO-I16x2-NEXT: min.u16 %rs6, %rs3, %rs1;
-; NO-I16x2-NEXT: mov.b32 %r3, {%rs6, %rs5};
-; NO-I16x2-NEXT: st.param.b32 [func_retval0], %r3;
+; NO-I16x2-NEXT: st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; NO-I16x2-NEXT: ret;
%cmp = icmp ule <2 x i16> %a, %b
%r = select <2 x i1> %cmp, <2 x i16> %a, <2 x i16> %b
@@ -329,7 +343,7 @@ define <2 x i16> @test_mul(<2 x i16> %a, <2 x i16> %b) #0 {
; COMMON-LABEL: test_mul(
; COMMON: {
; COMMON-NEXT: .reg .b16 %rs<7>;
-; COMMON-NEXT: .reg .b32 %r<4>;
+; COMMON-NEXT: .reg .b32 %r<3>;
; COMMON-EMPTY:
; COMMON-NEXT: // %bb.0:
; COMMON-NEXT: ld.param.b32 %r2, [test_mul_param_1];
@@ -338,8 +352,7 @@ define <2 x i16> @test_mul(<2 x i16> %a, <2 x i16> %b) #0 {
; COMMON-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; COMMON-NEXT: mul.lo.s16 %rs5, %rs4, %rs2;
; COMMON-NEXT: mul.lo.s16 %rs6, %rs3, %rs1;
-; COMMON-NEXT: mov.b32 %r3, {%rs6, %rs5};
-; COMMON-NEXT: st.param.b32 [func_retval0], %r3;
+; COMMON-NEXT: st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; COMMON-NEXT: ret;
%r = mul <2 x i16> %a, %b
ret <2 x i16> %r
@@ -587,14 +600,14 @@ define void @test_ldst_v3i16(ptr %a, ptr %b) {
define void @test_ldst_v4i16(ptr %a, ptr %b) {
; COMMON-LABEL: test_ldst_v4i16(
; COMMON: {
-; COMMON-NEXT: .reg .b16 %rs<5>;
+; COMMON-NEXT: .reg .b32 %r<3>;
; COMMON-NEXT: .reg .b64 %rd<3>;
; COMMON-EMPTY:
; COMMON-NEXT: // %bb.0:
; COMMON-NEXT: ld.param.b64 %rd2, [test_ldst_v4i16_param_1];
; COMMON-NEXT: ld.param.b64 %rd1, [test_ldst_v4i16_param_0];
-; COMMON-NEXT: ld.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [%rd1];
-; COMMON-NEXT: st.v4.b16 [%rd2], {%rs1, %rs2, %rs3, %rs4};
+; COMMON-NEXT: ld.v2.b32 {%r1, %r2}, [%rd1];
+; COMMON-NEXT: st.v2.b32 [%rd2], {%r1, %r2};
; COMMON-NEXT: ret;
%t1 = load <4 x i16>, ptr %a
store <4 x i16> %t1, ptr %b, align 16
@@ -729,7 +742,7 @@ define <2 x i16> @test_select_cc(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x
; COMMON: {
; COMMON-NEXT: .reg .pred %p<3>;
; COMMON-NEXT: .reg .b16 %rs<11>;
-; COMMON-NEXT: .reg .b32 %r<6>;
+; COMMON-NEXT: .reg .b32 %r<5>;
; COMMON-EMPTY:
; COMMON-NEXT: // %bb.0:
; COMMON-NEXT: ld.param.b32 %r4, [test_select_cc_param_3];
@@ -744,8 +757,7 @@ define <2 x i16> @test_select_cc(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x
; COMMON-NEXT: mov.b32 {%rs7, %rs8}, %r1;
; COMMON-NEXT: selp.b16 %rs9, %rs8, %rs6, %p2;
; COMMON-NEXT: selp.b16 %rs10, %rs7, %rs5, %p1;
-; COMMON-NEXT: mov.b32 %r5, {%rs10, %rs9};
-; COMMON-NEXT: st.param.b32 [func_retval0], %r5;
+; COMMON-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs9};
; COMMON-NEXT: ret;
%cc = icmp ne <2 x i16> %c, %d
%r = select <2 x i1> %cc, <2 x i16> %a, <2 x i16> %b
@@ -783,7 +795,7 @@ define <2 x i16> @test_select_cc_i16_i32(<2 x i16> %a, <2 x i16> %b,
; COMMON: {
; COMMON-NEXT: .reg .pred %p<3>;
; COMMON-NEXT: .reg .b16 %rs<7>;
-; COMMON-NEXT: .reg .b32 %r<8>;
+; COMMON-NEXT: .reg .b32 %r<7>;
; COMMON-EMPTY:
; COMMON-NEXT: // %bb.0:
; COMMON-NEXT: ld.param.v2.b32 {%r5, %r6}, [test_select_cc_i16_i32_param_3];
@@ -796,8 +808,7 @@ define <2 x i16> @test_select_cc_i16_i32(<2 x i16> %a, <2 x i16> %b,
; COMMON-NEXT: mov.b32 {%rs3, %rs4}, %r1;
; COMMON-NEXT: selp.b16 %rs5, %rs4, %rs2, %p2;
; COMMON-NEXT: selp.b16 %rs6, %rs3, %rs1, %p1;
-; COMMON-NEXT: mov.b32 %r7, {%rs6, %rs5};
-; COMMON-NEXT: st.param.b32 [func_retval0], %r7;
+; COMMON-NEXT: st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; COMMON-NEXT: ret;
<2 x i32> %c, <2 x i32> %d) #0 {
%cc = icmp ne <2 x i32> %c, %d
@@ -839,7 +850,7 @@ define <2 x i16> @test_trunc_2xi32_muliple_use0(<2 x i32> %a, ptr %p) #0 {
; NO-I16x2-LABEL: test_trunc_2xi32_muliple_use0(
; NO-I16x2: {
; NO-I16x2-NEXT: .reg .b16 %rs<5>;
-; NO-I16x2-NEXT: .reg .b32 %r<5>;
+; NO-I16x2-NEXT: .reg .b32 %r<4>;
; NO-I16x2-NEXT: .reg .b64 %rd<2>;
; NO-I16x2-EMPTY:
; NO-I16x2-NEXT: // %bb.0:
@@ -850,8 +861,7 @@ define <2 x i16> @test_trunc_2xi32_muliple_use0(<2 x i32> %a, ptr %p) #0 {
; NO-I16x2-NEXT: mov.b32 %r3, {%rs2, %rs1};
; NO-I16x2-NEXT: add.s16 %rs3, %rs1, 1;
; NO-I16x2-NEXT: add.s16 %rs4, %rs2, 1;
-; NO-I16x2-NEXT: mov.b32 %r4, {%rs4, %rs3};
-; NO-I16x2-NEXT: st.b32 [%rd1], %r4;
+; NO-I16x2-NEXT: st.v2.b16 [%rd1], {%rs4, %rs3};
; NO-I16x2-NEXT: st.param.b32 [func_retval0], %r3;
; NO-I16x2-NEXT: ret;
%r = trunc <2 x i32> %a to <2 x i16>
@@ -889,15 +899,13 @@ define <2 x i16> @test_trunc_2xi64(<2 x i64> %a) #0 {
; COMMON-LABEL: test_trunc_2xi64(
; COMMON: {
; COMMON-NEXT: .reg .b16 %rs<3>;
-; COMMON-NEXT: .reg .b32 %r<2>;
; COMMON-NEXT: .reg .b64 %rd<3>;
; COMMON-EMPTY:
; COMMON-NEXT: // %bb.0:
; COMMON-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [test_trunc_2xi64_param_0];
; COMMON-NEXT: cvt.u16.u64 %rs1, %rd2;
; COMMON-NEXT: cvt.u16.u64 %rs2, %rd1;
-; COMMON-NEXT: mov.b32 %r1, {%rs2, %rs1};
-; COMMON-NEXT: st.param.b32 [func_retval0], %r1;
+; COMMON-NEXT: st.param.v2.b16 [func_retval0], {%rs2, %rs1};
; COMMON-NEXT: ret;
%r = trunc <2 x i64> %a to <2 x i16>
ret <2 x i16> %r
@@ -987,32 +995,41 @@ define <2 x i16> @test_shufflevector(<2 x i16> %a) #0 {
; COMMON-LABEL: test_shufflevector(
; COMMON: {
; COMMON-NEXT: .reg .b16 %rs<3>;
-; COMMON-NEXT: .reg .b32 %r<3>;
+; COMMON-NEXT: .reg .b32 %r<2>;
; COMMON-EMPTY:
; COMMON-NEXT: // %bb.0:
; COMMON-NEXT: ld.param.b32 %r1, [test_shufflevector_param_0];
; COMMON-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; COMMON-NEXT: mov.b32 %r2, {%rs2, %rs1};
-; COMMON-NEXT: st.param.b32 [func_retval0], %r2;
+; COMMON-NEXT: st.param.v2.b16 [func_retval0], {%rs2, %rs1};
; COMMON-NEXT: ret;
%s = shufflevector <2 x i16> %a, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
ret <2 x i16> %s
}
define <2 x i16> @test_insertelement(<2 x i16> %a, i16 %x) #0 {
-; COMMON-LABEL: test_insertelement(
-; COMMON: {
-; COMMON-NEXT: .reg .b16 %rs<3>;
-; COMMON-NEXT: .reg .b32 %r<3>;
-; COMMON-EMPTY:
-; COMMON-NEXT: // %bb.0:
-; COMMON-NEXT: ld.param.b16 %rs1, [test_insertelement_param_1];
-; COMMON-NEXT: ld.param.b32 %r1, [test_insertelement_param_0];
+; I16x2-LABEL: test_insertelement(
+; I16x2: {
+; I16x2-NEXT: .reg .b16 %rs<3>;
+; I16x2-NEXT: .reg .b32 %r<2>;
+; I16x2-EMPTY:
+; I16x2-NEXT: // %bb.0:
+; I16x2-NEXT: ld.param.b16 %rs1, [test_insertelement_param_1];
+; I16x2-NEXT: ld.param.b32 %r1, [test_insertelement_param_0];
; I16x2-NEXT: mov.b32 {%rs2, _}, %r1;
-; NO-I16x2-NEXT: { .reg .b16 tmp; mov.b32 {%rs2, tmp}, %r1; }
-; COMMON-NEXT: mov.b32 %r2, {%rs2, %rs1};
-; COMMON-NEXT: st.param.b32 [func_retval0], %r2;
-; COMMON-NEXT: ret;
+; I16x2-NEXT: st.param.v2.b16 [func_retval0], {%rs2, %rs1};
+; I16x2-NEXT: ret;
+;
+; NO-I16x2-LABEL: test_insertelement(
+; NO-I16x2: {
+; NO-I16x2-NEXT: .reg .b16 %rs<3>;
+; NO-I16x2-NEXT: .reg .b32 %r<2>;
+; NO-I16x2-EMPTY:
+; NO-I16x2-NEXT: // %bb.0:
+; NO-I16x2-NEXT: ld.param.b16 %rs1, [test_insertelement_param_1];
+; NO-I16x2-NEXT: ld.param.b32 %r1, [test_insertelement_param_0];
+; NO-I16x2-NEXT: { .reg .b16 tmp; mov.b32 {%rs2, tmp}, %r1; }
+; NO-I16x2-NEXT: st.param.v2.b16 [func_retval0], {%rs2, %rs1};
+; NO-I16x2-NEXT: ret;
%i = insertelement <2 x i16> %a, i16 %x, i64 1
ret <2 x i16> %i
}
@@ -1021,15 +1038,14 @@ define <2 x i16> @test_fptosi_2xhalf_to_2xi16(<2 x half> %a) #0 {
; COMMON-LABEL: test_fptosi_2xhalf_to_2xi16(
; COMMON: {
; COMMON-NEXT: .reg .b16 %rs<5>;
-; COMMON-NEXT: .reg .b32 %r<3>;
+; COMMON-NEXT: .reg .b32 %r<2>;
; COMMON-EMPTY:
; COMMON-NEXT: // %bb.0:
; COMMON-NEXT: ld.param.b32 %r1, [test_fptosi_2xhalf_to_2xi16_param_0];
; COMMON-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; COMMON-NEXT: cvt.rzi.s16.f16 %rs3, %rs2;
; COMMON-NEXT: cvt.rzi.s16.f16 %rs4, %rs1;
-; COMMON-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; COMMON-NEXT: st.param.b32 [func_retval0], %r2;
+; COMMON-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; COMMON-NEXT: ret;
%r = fptosi <2 x half> %a to <2 x i16>
ret <2 x i16> %r
@@ -1039,15 +1055,14 @@ define <2 x i16> @test_fptoui_2xhalf_to_2xi16(<2 x half> %a) #0 {
; COMMON-LABEL: test_fptoui_2xhalf_to_2xi16(
; COMMON: {
; COMMON-NEXT: .reg .b16 %rs<5>;
-; COMMON-NEXT: .reg .b32 %r<3>;
+; COMMON-NEXT: .reg .b32 %r<2>;
; COMMON-EMPTY:
; COMMON-NEXT: // %bb.0:
; COMMON-NEXT: ld.param.b32 %r1, [test_fptoui_2xhalf_to_2xi16_param_0];
; COMMON-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; COMMON-NEXT: cvt.rzi.u16.f16 %rs3, %rs2;
; COMMON-NEXT: cvt.rzi.u16.f16 %rs4, %rs1;
-; COMMON-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; COMMON-NEXT: st.param.b32 [func_retval0], %r2;
+; COMMON-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; COMMON-NEXT: ret;
%r = fptoui <2 x half> %a to <2 x i16>
ret <2 x i16> %r
diff --git a/llvm/test/CodeGen/NVPTX/i8x2-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x2-instructions.ll
index fe81134895926..718840897c696 100644
--- a/llvm/test/CodeGen/NVPTX/i8x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i8x2-instructions.ll
@@ -33,13 +33,11 @@ define <2 x i8> @test_bitcast_i16_2xi8(i16 %a) {
; CHECK-LABEL: test_bitcast_i16_2xi8(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b16 %rs1, [test_bitcast_i16_2xi8_param_0];
; CHECK-NEXT: shr.u16 %rs2, %rs1, 8;
-; CHECK-NEXT: mov.b32 %r1, {%rs1, %rs2};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs1, %rs2};
; CHECK-NEXT: ret;
%res = bitcast i16 %a to <2 x i8>
ret <2 x i8> %res
diff --git a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
index 4e1d13696edfe..7cc7468bc7de7 100644
--- a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
@@ -268,19 +268,19 @@ define <4 x i8> @test_smax(<4 x i8> %a, <4 x i8> %b) #0 {
; CHECK-NEXT: bfe.s32 %r9, %r2, 24, 8;
; CHECK-NEXT: bfe.s32 %r10, %r1, 24, 8;
; CHECK-NEXT: setp.gt.s32 %p4, %r10, %r9;
-; CHECK-NEXT: bfe.u32 %r11, %r1, 0, 8;
-; CHECK-NEXT: bfe.u32 %r12, %r1, 8, 8;
-; CHECK-NEXT: bfe.u32 %r13, %r1, 16, 8;
-; CHECK-NEXT: bfe.u32 %r14, %r1, 24, 8;
-; CHECK-NEXT: bfe.u32 %r15, %r2, 24, 8;
-; CHECK-NEXT: selp.b32 %r16, %r14, %r15, %p4;
-; CHECK-NEXT: bfe.u32 %r17, %r2, 16, 8;
-; CHECK-NEXT: selp.b32 %r18, %r13, %r17, %p3;
+; CHECK-NEXT: bfe.u32 %r11, %r2, 0, 8;
+; CHECK-NEXT: bfe.u32 %r12, %r2, 8, 8;
+; CHECK-NEXT: bfe.u32 %r13, %r2, 16, 8;
+; CHECK-NEXT: bfe.u32 %r14, %r2, 24, 8;
+; CHECK-NEXT: bfe.u32 %r15, %r1, 24, 8;
+; CHECK-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; CHECK-NEXT: bfe.u32 %r17, %r1, 16, 8;
+; CHECK-NEXT: selp.b32 %r18, %r17, %r13, %p3;
; CHECK-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
-; CHECK-NEXT: bfe.u32 %r20, %r2, 8, 8;
-; CHECK-NEXT: selp.b32 %r21, %r12, %r20, %p2;
-; CHECK-NEXT: bfe.u32 %r22, %r2, 0, 8;
-; CHECK-NEXT: selp.b32 %r23, %r11, %r22, %p1;
+; CHECK-NEXT: bfe.u32 %r20, %r1, 8, 8;
+; CHECK-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; CHECK-NEXT: bfe.u32 %r22, %r1, 0, 8;
+; CHECK-NEXT: selp.b32 %r23, %r22, %r11, %p1;
; CHECK-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
; CHECK-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
; CHECK-NEXT: st.param.b32 [func_retval0], %r25;
@@ -346,19 +346,19 @@ define <4 x i8> @test_smin(<4 x i8> %a, <4 x i8> %b) #0 {
; CHECK-NEXT: bfe.s32 %r9, %r2, 24, 8;
; CHECK-NEXT: bfe.s32 %r10, %r1, 24, 8;
; CHECK-NEXT: setp.le.s32 %p4, %r10, %r9;
-; CHECK-NEXT: bfe.u32 %r11, %r1, 0, 8;
-; CHECK-NEXT: bfe.u32 %r12, %r1, 8, 8;
-; CHECK-NEXT: bfe.u32 %r13, %r1, 16, 8;
-; CHECK-NEXT: bfe.u32 %r14, %r1, 24, 8;
-; CHECK-NEXT: bfe.u32 %r15, %r2, 24, 8;
-; CHECK-NEXT: selp.b32 %r16, %r14, %r15, %p4;
-; CHECK-NEXT: bfe.u32 %r17, %r2, 16, 8;
-; CHECK-NEXT: selp.b32 %r18, %r13, %r17, %p3;
+; CHECK-NEXT: bfe.u32 %r11, %r2, 0, 8;
+; CHECK-NEXT: bfe.u32 %r12, %r2, 8, 8;
+; CHECK-NEXT: bfe.u32 %r13, %r2, 16, 8;
+; CHECK-NEXT: bfe.u32 %r14, %r2, 24, 8;
+; CHECK-NEXT: bfe.u32 %r15, %r1, 24, 8;
+; CHECK-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; CHECK-NEXT: bfe.u32 %r17, %r1, 16, 8;
+; CHECK-NEXT: selp.b32 %r18, %r17, %r13, %p3;
; CHECK-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
-; CHECK-NEXT: bfe.u32 %r20, %r2, 8, 8;
-; CHECK-NEXT: selp.b32 %r21, %r12, %r20, %p2;
-; CHECK-NEXT: bfe.u32 %r22, %r2, 0, 8;
-; CHECK-NEXT: selp.b32 %r23, %r11, %r22, %p1;
+; CHECK-NEXT: bfe.u32 %r20, %r1, 8, 8;
+; CHECK-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; CHECK-NEXT: bfe.u32 %r22, %r1, 0, 8;
+; CHECK-NEXT: selp.b32 %r23, %r22, %r11, %p1;
; CHECK-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
; CHECK-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
; CHECK-NEXT: st.param.b32 [func_retval0], %r25;
diff --git a/llvm/test/CodeGen/NVPTX/ldg-invariant.ll b/llvm/test/CodeGen/NVPTX/ldg-invariant.ll
index c5c5de4c1b85e..06143debb6838 100644
--- a/llvm/test/CodeGen/NVPTX/ldg-invariant.ll
+++ b/llvm/test/CodeGen/NVPTX/ldg-invariant.ll
@@ -26,17 +26,16 @@ define half @ld_global_v2f16(ptr addrspace(1) %ptr) {
; CHECK-LABEL: ld_global_v2f16(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<4>;
-; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b32 %r<4>;
; CHECK-NEXT: .reg .b64 %rd<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [ld_global_v2f16_param_0];
-; CHECK-NEXT: ld.global.nc.b32 %r1, [%rd1];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: cvt.f32.f16 %r2, %rs2;
-; CHECK-NEXT: cvt.f32.f16 %r3, %rs1;
-; CHECK-NEXT: add.rn.f32 %r4, %r3, %r2;
-; CHECK-NEXT: cvt.rn.f16.f32 %rs3, %r4;
+; CHECK-NEXT: ld.global.nc.v2.b16 {%rs1, %rs2}, [%rd1];
+; CHECK-NEXT: cvt.f32.f16 %r1, %rs2;
+; CHECK-NEXT: cvt.f32.f16 %r2, %rs1;
+; CHECK-NEXT: add.rn.f32 %r3, %r2, %r1;
+; CHECK-NEXT: cvt.rn.f16.f32 %rs3, %r3;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%a = load <2 x half>, ptr addrspace(1) %ptr, !invariant.load !0
diff --git a/llvm/test/CodeGen/NVPTX/load-store-vectors.ll b/llvm/test/CodeGen/NVPTX/load-store-vectors.ll
index 9e7e940ab5a75..3c90323da01d7 100644
--- a/llvm/test/CodeGen/NVPTX/load-store-vectors.ll
+++ b/llvm/test/CodeGen/NVPTX/load-store-vectors.ll
@@ -237,17 +237,14 @@ define void @generic_2xi16(ptr %a) {
; CHECK-LABEL: generic_2xi16(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-NEXT: .reg .b64 %rd<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [generic_2xi16_param_0];
-; CHECK-NEXT: ld.b32 %r1, [%rd1];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT: ld.v2.b16 {%rs1, %rs2}, [%rd1];
; CHECK-NEXT: add.s16 %rs3, %rs2, 1;
; CHECK-NEXT: add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.b32 [%rd1], %r2;
+; CHECK-NEXT: st.v2.b16 [%rd1], {%rs4, %rs3};
; CHECK-NEXT: ret;
%a.load = load <2 x i16>, ptr %a
%a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -657,17 +654,14 @@ define void @generic_volatile_2xi16(ptr %a) {
; CHECK-LABEL: generic_volatile_2xi16(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-NEXT: .reg .b64 %rd<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [generic_volatile_2xi16_param_0];
-; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT: ld.volatile.v2.b16 {%rs1, %rs2}, [%rd1];
; CHECK-NEXT: add.s16 %rs3, %rs2, 1;
; CHECK-NEXT: add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.volatile.b32 [%rd1], %r2;
+; CHECK-NEXT: st.volatile.v2.b16 [%rd1], {%rs4, %rs3};
; CHECK-NEXT: ret;
%a.load = load volatile <2 x i16>, ptr %a
%a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -1060,17 +1054,14 @@ define void @global_2xi16(ptr addrspace(1) %a) {
; CHECK-LABEL: global_2xi16(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-NEXT: .reg .b64 %rd<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [global_2xi16_param_0];
-; CHECK-NEXT: ld.global.b32 %r1, [%rd1];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT: ld.global.v2.b16 {%rs1, %rs2}, [%rd1];
; CHECK-NEXT: add.s16 %rs3, %rs2, 1;
; CHECK-NEXT: add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.global.b32 [%rd1], %r2;
+; CHECK-NEXT: st.global.v2.b16 [%rd1], {%rs4, %rs3};
; CHECK-NEXT: ret;
%a.load = load <2 x i16>, ptr addrspace(1) %a
%a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -1461,17 +1452,14 @@ define void @global_volatile_2xi16(ptr addrspace(1) %a) {
; CHECK-LABEL: global_volatile_2xi16(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-NEXT: .reg .b64 %rd<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [global_volatile_2xi16_param_0];
-; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT: ld.volatile.global.v2.b16 {%rs1, %rs2}, [%rd1];
; CHECK-NEXT: add.s16 %rs3, %rs2, 1;
; CHECK-NEXT: add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r2;
+; CHECK-NEXT: st.volatile.global.v2.b16 [%rd1], {%rs4, %rs3};
; CHECK-NEXT: ret;
%a.load = load volatile <2 x i16>, ptr addrspace(1) %a
%a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -1864,17 +1852,14 @@ define void @shared_2xi16(ptr addrspace(3) %a) {
; CHECK-LABEL: shared_2xi16(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-NEXT: .reg .b64 %rd<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [shared_2xi16_param_0];
-; CHECK-NEXT: ld.shared.b32 %r1, [%rd1];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT: ld.shared.v2.b16 {%rs1, %rs2}, [%rd1];
; CHECK-NEXT: add.s16 %rs3, %rs2, 1;
; CHECK-NEXT: add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.shared.b32 [%rd1], %r2;
+; CHECK-NEXT: st.shared.v2.b16 [%rd1], {%rs4, %rs3};
; CHECK-NEXT: ret;
%a.load = load <2 x i16>, ptr addrspace(3) %a
%a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -2265,17 +2250,14 @@ define void @shared_volatile_2xi16(ptr addrspace(3) %a) {
; CHECK-LABEL: shared_volatile_2xi16(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-NEXT: .reg .b64 %rd<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [shared_volatile_2xi16_param_0];
-; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT: ld.volatile.shared.v2.b16 {%rs1, %rs2}, [%rd1];
; CHECK-NEXT: add.s16 %rs3, %rs2, 1;
; CHECK-NEXT: add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r2;
+; CHECK-NEXT: st.volatile.shared.v2.b16 [%rd1], {%rs4, %rs3};
; CHECK-NEXT: ret;
%a.load = load volatile <2 x i16>, ptr addrspace(3) %a
%a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -2668,17 +2650,14 @@ define void @local_2xi16(ptr addrspace(5) %a) {
; CHECK-LABEL: local_2xi16(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-NEXT: .reg .b64 %rd<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [local_2xi16_param_0];
-; CHECK-NEXT: ld.local.b32 %r1, [%rd1];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT: ld.local.v2.b16 {%rs1, %rs2}, [%rd1];
; CHECK-NEXT: add.s16 %rs3, %rs2, 1;
; CHECK-NEXT: add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.local.b32 [%rd1], %r2;
+; CHECK-NEXT: st.local.v2.b16 [%rd1], {%rs4, %rs3};
; CHECK-NEXT: ret;
%a.load = load <2 x i16>, ptr addrspace(5) %a
%a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -3069,17 +3048,14 @@ define void @local_volatile_2xi16(ptr addrspace(5) %a) {
; CHECK-LABEL: local_volatile_2xi16(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-NEXT: .reg .b64 %rd<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [local_volatile_2xi16_param_0];
-; CHECK-NEXT: ld.local.b32 %r1, [%rd1];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT: ld.local.v2.b16 {%rs1, %rs2}, [%rd1];
; CHECK-NEXT: add.s16 %rs3, %rs2, 1;
; CHECK-NEXT: add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.local.b32 [%rd1], %r2;
+; CHECK-NEXT: st.local.v2.b16 [%rd1], {%rs4, %rs3};
; CHECK-NEXT: ret;
%a.load = load volatile <2 x i16>, ptr addrspace(5) %a
%a.add = add <2 x i16> %a.load, <i16 1, i16 1>
diff --git a/llvm/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll b/llvm/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll
index 4d7a4b50e8940..d494ee30c2821 100644
--- a/llvm/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll
+++ b/llvm/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 -verify-machineinstrs | FileCheck -check-prefix=SM20 %s
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_35 -verify-machineinstrs | FileCheck -check-prefix=SM35 %s
; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
@@ -6,195 +7,642 @@
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"
target triple = "nvptx64-unknown-unknown"
-; SM20-LABEL: .visible .entry foo1(
-; SM20: ld.global.b32
-; SM35-LABEL: .visible .entry foo1(
-; SM35: ld.global.nc.b32
define ptx_kernel void @foo1(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo1(
+; SM20: {
+; SM20-NEXT: .reg .b32 %r<2>;
+; SM20-NEXT: .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo1_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo1_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.b32 %r1, [%rd2];
+; SM20-NEXT: st.global.b32 [%rd4], %r1;
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo1(
+; SM35: {
+; SM35-NEXT: .reg .b32 %r<2>;
+; SM35-NEXT: .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo1_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo1_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.b32 %r1, [%rd2];
+; SM35-NEXT: st.global.b32 [%rd4], %r1;
+; SM35-NEXT: ret;
%1 = load float, ptr %from
store float %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo2(
-; SM20: ld.global.b64
-; SM35-LABEL: .visible .entry foo2(
-; SM35: ld.global.nc.b64
define ptx_kernel void @foo2(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo2(
+; SM20: {
+; SM20-NEXT: .reg .b64 %rd<6>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo2_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo2_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.b64 %rd5, [%rd2];
+; SM20-NEXT: st.global.b64 [%rd4], %rd5;
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo2(
+; SM35: {
+; SM35-NEXT: .reg .b64 %rd<6>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo2_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo2_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.b64 %rd5, [%rd2];
+; SM35-NEXT: st.global.b64 [%rd4], %rd5;
+; SM35-NEXT: ret;
%1 = load double, ptr %from
store double %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo3(
-; SM20: ld.global.b16
-; SM35-LABEL: .visible .entry foo3(
-; SM35: ld.global.nc.b16
define ptx_kernel void @foo3(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo3(
+; SM20: {
+; SM20-NEXT: .reg .b16 %rs<2>;
+; SM20-NEXT: .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo3_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo3_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.b16 %rs1, [%rd2];
+; SM20-NEXT: st.global.b16 [%rd4], %rs1;
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo3(
+; SM35: {
+; SM35-NEXT: .reg .b16 %rs<2>;
+; SM35-NEXT: .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo3_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo3_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.b16 %rs1, [%rd2];
+; SM35-NEXT: st.global.b16 [%rd4], %rs1;
+; SM35-NEXT: ret;
%1 = load i16, ptr %from
store i16 %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo4(
-; SM20: ld.global.b32
-; SM35-LABEL: .visible .entry foo4(
-; SM35: ld.global.nc.b32
define ptx_kernel void @foo4(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo4(
+; SM20: {
+; SM20-NEXT: .reg .b32 %r<2>;
+; SM20-NEXT: .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo4_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo4_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.b32 %r1, [%rd2];
+; SM20-NEXT: st.global.b32 [%rd4], %r1;
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo4(
+; SM35: {
+; SM35-NEXT: .reg .b32 %r<2>;
+; SM35-NEXT: .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo4_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo4_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.b32 %r1, [%rd2];
+; SM35-NEXT: st.global.b32 [%rd4], %r1;
+; SM35-NEXT: ret;
%1 = load i32, ptr %from
store i32 %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo5(
-; SM20: ld.global.b64
-; SM35-LABEL: .visible .entry foo5(
-; SM35: ld.global.nc.b64
define ptx_kernel void @foo5(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo5(
+; SM20: {
+; SM20-NEXT: .reg .b64 %rd<6>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo5_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo5_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.b64 %rd5, [%rd2];
+; SM20-NEXT: st.global.b64 [%rd4], %rd5;
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo5(
+; SM35: {
+; SM35-NEXT: .reg .b64 %rd<6>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo5_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo5_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.b64 %rd5, [%rd2];
+; SM35-NEXT: st.global.b64 [%rd4], %rd5;
+; SM35-NEXT: ret;
%1 = load i64, ptr %from
store i64 %1, ptr %to
ret void
}
; i128 is non standard integer in nvptx64
-; SM20-LABEL: .visible .entry foo6(
-; SM20: ld.global.v2.b64
-; SM35-LABEL: .visible .entry foo6(
-; SM35: ld.global.nc.v2.b64
define ptx_kernel void @foo6(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo6(
+; SM20: {
+; SM20-NEXT: .reg .b64 %rd<7>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo6_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo6_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM20-NEXT: st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo6(
+; SM35: {
+; SM35-NEXT: .reg .b64 %rd<7>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo6_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo6_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM35-NEXT: st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM35-NEXT: ret;
%1 = load i128, ptr %from
store i128 %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo7(
-; SM20: ld.global.v2.b8
-; SM35-LABEL: .visible .entry foo7(
-; SM35: ld.global.nc.v2.b8
define ptx_kernel void @foo7(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo7(
+; SM20: {
+; SM20-NEXT: .reg .b16 %rs<3>;
+; SM20-NEXT: .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo7_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo7_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.v2.b8 {%rs1, %rs2}, [%rd2];
+; SM20-NEXT: st.global.v2.b8 [%rd4], {%rs1, %rs2};
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo7(
+; SM35: {
+; SM35-NEXT: .reg .b16 %rs<3>;
+; SM35-NEXT: .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo7_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo7_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.v2.b8 {%rs1, %rs2}, [%rd2];
+; SM35-NEXT: st.global.v2.b8 [%rd4], {%rs1, %rs2};
+; SM35-NEXT: ret;
%1 = load <2 x i8>, ptr %from
store <2 x i8> %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo8(
-; SM20: ld.global.b32
-; SM35-LABEL: .visible .entry foo8(
-; SM35: ld.global.nc.b32
define ptx_kernel void @foo8(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo8(
+; SM20: {
+; SM20-NEXT: .reg .b32 %r<2>;
+; SM20-NEXT: .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo8_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo8_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.b32 %r1, [%rd2];
+; SM20-NEXT: st.global.b32 [%rd4], %r1;
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo8(
+; SM35: {
+; SM35-NEXT: .reg .b32 %r<2>;
+; SM35-NEXT: .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo8_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo8_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.b32 %r1, [%rd2];
+; SM35-NEXT: st.global.b32 [%rd4], %r1;
+; SM35-NEXT: ret;
%1 = load <2 x i16>, ptr %from
store <2 x i16> %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo9(
-; SM20: ld.global.v2.b32
-; SM35-LABEL: .visible .entry foo9(
-; SM35: ld.global.nc.v2.b32
define ptx_kernel void @foo9(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo9(
+; SM20: {
+; SM20-NEXT: .reg .b32 %r<3>;
+; SM20-NEXT: .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo9_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo9_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.v2.b32 {%r1, %r2}, [%rd2];
+; SM20-NEXT: st.global.v2.b32 [%rd4], {%r1, %r2};
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo9(
+; SM35: {
+; SM35-NEXT: .reg .b32 %r<3>;
+; SM35-NEXT: .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo9_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo9_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.v2.b32 {%r1, %r2}, [%rd2];
+; SM35-NEXT: st.global.v2.b32 [%rd4], {%r1, %r2};
+; SM35-NEXT: ret;
%1 = load <2 x i32>, ptr %from
store <2 x i32> %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo10(
-; SM20: ld.global.v2.b64
-; SM35-LABEL: .visible .entry foo10(
-; SM35: ld.global.nc.v2.b64
define ptx_kernel void @foo10(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo10(
+; SM20: {
+; SM20-NEXT: .reg .b64 %rd<7>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo10_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo10_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM20-NEXT: st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo10(
+; SM35: {
+; SM35-NEXT: .reg .b64 %rd<7>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo10_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo10_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM35-NEXT: st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM35-NEXT: ret;
%1 = load <2 x i64>, ptr %from
store <2 x i64> %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo11(
-; SM20: ld.global.v2.b32
-; SM35-LABEL: .visible .entry foo11(
-; SM35: ld.global.nc.v2.b32
define ptx_kernel void @foo11(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo11(
+; SM20: {
+; SM20-NEXT: .reg .b32 %r<3>;
+; SM20-NEXT: .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo11_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo11_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.v2.b32 {%r1, %r2}, [%rd2];
+; SM20-NEXT: st.global.v2.b32 [%rd4], {%r1, %r2};
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo11(
+; SM35: {
+; SM35-NEXT: .reg .b32 %r<3>;
+; SM35-NEXT: .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo11_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo11_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.v2.b32 {%r1, %r2}, [%rd2];
+; SM35-NEXT: st.global.v2.b32 [%rd4], {%r1, %r2};
+; SM35-NEXT: ret;
%1 = load <2 x float>, ptr %from
store <2 x float> %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo12(
-; SM20: ld.global.v2.b64
-; SM35-LABEL: .visible .entry foo12(
-; SM35: ld.global.nc.v2.b64
define ptx_kernel void @foo12(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo12(
+; SM20: {
+; SM20-NEXT: .reg .b64 %rd<7>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo12_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo12_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM20-NEXT: st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo12(
+; SM35: {
+; SM35-NEXT: .reg .b64 %rd<7>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo12_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo12_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM35-NEXT: st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM35-NEXT: ret;
%1 = load <2 x double>, ptr %from
store <2 x double> %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo13(
-; SM20: ld.global.b32
-; SM35-LABEL: .visible .entry foo13(
-; SM35: ld.global.nc.b32
define ptx_kernel void @foo13(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo13(
+; SM20: {
+; SM20-NEXT: .reg .b32 %r<2>;
+; SM20-NEXT: .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo13_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo13_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.b32 %r1, [%rd2];
+; SM20-NEXT: st.global.b32 [%rd4], %r1;
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo13(
+; SM35: {
+; SM35-NEXT: .reg .b32 %r<2>;
+; SM35-NEXT: .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo13_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo13_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.b32 %r1, [%rd2];
+; SM35-NEXT: st.global.b32 [%rd4], %r1;
+; SM35-NEXT: ret;
%1 = load <4 x i8>, ptr %from
store <4 x i8> %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo14(
-; SM20: ld.global.v4.b16
-; SM35-LABEL: .visible .entry foo14(
-; SM35: ld.global.nc.v4.b16
define ptx_kernel void @foo14(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo14(
+; SM20: {
+; SM20-NEXT: .reg .b32 %r<3>;
+; SM20-NEXT: .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo14_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo14_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.v2.b32 {%r1, %r2}, [%rd2];
+; SM20-NEXT: st.global.v2.b32 [%rd4], {%r1, %r2};
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo14(
+; SM35: {
+; SM35-NEXT: .reg .b32 %r<3>;
+; SM35-NEXT: .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo14_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo14_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.v2.b32 {%r1, %r2}, [%rd2];
+; SM35-NEXT: st.global.v2.b32 [%rd4], {%r1, %r2};
+; SM35-NEXT: ret;
%1 = load <4 x i16>, ptr %from
store <4 x i16> %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo15(
-; SM20: ld.global.v4.b32
-; SM35-LABEL: .visible .entry foo15(
-; SM35: ld.global.nc.v4.b32
define ptx_kernel void @foo15(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo15(
+; SM20: {
+; SM20-NEXT: .reg .b32 %r<5>;
+; SM20-NEXT: .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo15_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo15_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; SM20-NEXT: st.global.v4.b32 [%rd4], {%r1, %r2, %r3, %r4};
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo15(
+; SM35: {
+; SM35-NEXT: .reg .b32 %r<5>;
+; SM35-NEXT: .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo15_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo15_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; SM35-NEXT: st.global.v4.b32 [%rd4], {%r1, %r2, %r3, %r4};
+; SM35-NEXT: ret;
%1 = load <4 x i32>, ptr %from
store <4 x i32> %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo16(
-; SM20: ld.global.v4.b32
-; SM35-LABEL: .visible .entry foo16(
-; SM35: ld.global.nc.v4.b32
define ptx_kernel void @foo16(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo16(
+; SM20: {
+; SM20-NEXT: .reg .b32 %r<5>;
+; SM20-NEXT: .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo16_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo16_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; SM20-NEXT: st.global.v4.b32 [%rd4], {%r1, %r2, %r3, %r4};
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo16(
+; SM35: {
+; SM35-NEXT: .reg .b32 %r<5>;
+; SM35-NEXT: .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo16_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo16_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; SM35-NEXT: st.global.v4.b32 [%rd4], {%r1, %r2, %r3, %r4};
+; SM35-NEXT: ret;
%1 = load <4 x float>, ptr %from
store <4 x float> %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo17(
-; SM20: ld.global.v2.b64
-; SM20: ld.global.v2.b64
-; SM35-LABEL: .visible .entry foo17(
-; SM35: ld.global.nc.v2.b64
-; SM35: ld.global.nc.v2.b64
define ptx_kernel void @foo17(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo17(
+; SM20: {
+; SM20-NEXT: .reg .b64 %rd<9>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo17_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo17_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM20-NEXT: ld.global.v2.b64 {%rd7, %rd8}, [%rd2+16];
+; SM20-NEXT: st.global.v2.b64 [%rd4+16], {%rd7, %rd8};
+; SM20-NEXT: st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo17(
+; SM35: {
+; SM35-NEXT: .reg .b64 %rd<9>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo17_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo17_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM35-NEXT: ld.global.nc.v2.b64 {%rd7, %rd8}, [%rd2+16];
+; SM35-NEXT: st.global.v2.b64 [%rd4+16], {%rd7, %rd8};
+; SM35-NEXT: st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM35-NEXT: ret;
%1 = load <4 x double>, ptr %from
store <4 x double> %1, ptr %to
ret void
}
-; SM20-LABEL: .visible .entry foo18(
-; SM20: ld.global.b64
-; SM35-LABEL: .visible .entry foo18(
-; SM35: ld.global.nc.b64
define ptx_kernel void @foo18(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo18(
+; SM20: {
+; SM20-NEXT: .reg .b64 %rd<6>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [foo18_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT: ld.param.b64 %rd3, [foo18_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT: ld.global.b64 %rd5, [%rd2];
+; SM20-NEXT: st.global.b64 [%rd4], %rd5;
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo18(
+; SM35: {
+; SM35-NEXT: .reg .b64 %rd<6>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [foo18_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT: ld.param.b64 %rd3, [foo18_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT: ld.global.nc.b64 %rd5, [%rd2];
+; SM35-NEXT: st.global.b64 [%rd4], %rd5;
+; SM35-NEXT: ret;
%1 = load ptr, ptr %from
store ptr %1, ptr %to
ret void
}
; Test that we can infer a cached load for a pointer induction variable.
-; SM20-LABEL: .visible .entry foo19(
-; SM20: ld.global.b32
-; SM35-LABEL: .visible .entry foo19(
-; SM35: ld.global.nc.b32
define ptx_kernel void @foo19(ptr noalias readonly %from, ptr %to, i32 %n) {
+; SM20-LABEL: foo19(
+; SM20: {
+; SM20-NEXT: .reg .pred %p<2>;
+; SM20-NEXT: .reg .b32 %r<10>;
+; SM20-NEXT: .reg .b64 %rd<8>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0: // %entry
+; SM20-NEXT: ld.param.b32 %r8, [foo19_param_2];
+; SM20-NEXT: ld.param.b64 %rd5, [foo19_param_0];
+; SM20-NEXT: cvta.to.global.u64 %rd7, %rd5;
+; SM20-NEXT: ld.param.b64 %rd6, [foo19_param_1];
+; SM20-NEXT: cvta.to.global.u64 %rd2, %rd6;
+; SM20-NEXT: mov.b32 %r9, 0f00000000;
+; SM20-NEXT: $L__BB18_1: // %loop
+; SM20-NEXT: // =>This Inner Loop Header: Depth=1
+; SM20-NEXT: ld.global.b32 %r7, [%rd7];
+; SM20-NEXT: add.rn.f32 %r9, %r7, %r9;
+; SM20-NEXT: add.s64 %rd7, %rd7, 4;
+; SM20-NEXT: add.s32 %r8, %r8, -1;
+; SM20-NEXT: setp.ne.s32 %p1, %r8, 0;
+; SM20-NEXT: @%p1 bra $L__BB18_1;
+; SM20-NEXT: // %bb.2: // %exit
+; SM20-NEXT: st.global.b32 [%rd2], %r9;
+; SM20-NEXT: ret;
+;
+; SM35-LABEL: foo19(
+; SM35: {
+; SM35-NEXT: .reg .pred %p<2>;
+; SM35-NEXT: .reg .b32 %r<10>;
+; SM35-NEXT: .reg .b64 %rd<8>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0: // %entry
+; SM35-NEXT: ld.param.b32 %r8, [foo19_param_2];
+; SM35-NEXT: ld.param.b64 %rd5, [foo19_param_0];
+; SM35-NEXT: cvta.to.global.u64 %rd7, %rd5;
+; SM35-NEXT: ld.param.b64 %rd6, [foo19_param_1];
+; SM35-NEXT: cvta.to.global.u64 %rd2, %rd6;
+; SM35-NEXT: mov.b32 %r9, 0f00000000;
+; SM35-NEXT: $L__BB18_1: // %loop
+; SM35-NEXT: // =>This Inner Loop Header: Depth=1
+; SM35-NEXT: ld.global.nc.b32 %r7, [%rd7];
+; SM35-NEXT: add.rn.f32 %r9, %r7, %r9;
+; SM35-NEXT: add.s64 %rd7, %rd7, 4;
+; SM35-NEXT: add.s32 %r8, %r8, -1;
+; SM35-NEXT: setp.ne.s32 %p1, %r8, 0;
+; SM35-NEXT: @%p1 bra $L__BB18_1;
+; SM35-NEXT: // %bb.2: // %exit
+; SM35-NEXT: st.global.b32 [%rd2], %r9;
+; SM35-NEXT: ret;
entry:
br label %loop
@@ -218,11 +666,30 @@ exit:
; know that the parameter is global. We also do not know that the
; pointed-to memory is never written to (for the duration of the
; kernel). For both reasons, we cannot use a cached load here.
+define void @notkernel(ptr noalias readonly %from, ptr %to) {
; SM20-LABEL: notkernel(
-; SM20: ld.b32
+; SM20: {
+; SM20-NEXT: .reg .b32 %r<2>;
+; SM20-NEXT: .reg .b64 %rd<3>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [notkernel_param_0];
+; SM20-NEXT: ld.b32 %r1, [%rd1];
+; SM20-NEXT: ld.param.b64 %rd2, [notkernel_param_1];
+; SM20-NEXT: st.b32 [%rd2], %r1;
+; SM20-NEXT: ret;
+;
; SM35-LABEL: notkernel(
-; SM35: ld.b32
-define void @notkernel(ptr noalias readonly %from, ptr %to) {
+; SM35: {
+; SM35-NEXT: .reg .b32 %r<2>;
+; SM35-NEXT: .reg .b64 %rd<3>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [notkernel_param_0];
+; SM35-NEXT: ld.b32 %r1, [%rd1];
+; SM35-NEXT: ld.param.b64 %rd2, [notkernel_param_1];
+; SM35-NEXT: st.b32 [%rd2], %r1;
+; SM35-NEXT: ret;
%1 = load float, ptr %from
store float %1, ptr %to
ret void
@@ -232,11 +699,30 @@ define void @notkernel(ptr noalias readonly %from, ptr %to) {
; do not know that the parameter is never written to (for the duration of the
; kernel). This case does not currently come up normally since we do not infer
; that pointers are global interprocedurally as of 2015-08-05.
+define void @notkernel2(ptr addrspace(1) noalias readonly %from, ptr %to) {
; SM20-LABEL: notkernel2(
-; SM20: ld.global.b32
+; SM20: {
+; SM20-NEXT: .reg .b32 %r<2>;
+; SM20-NEXT: .reg .b64 %rd<3>;
+; SM20-EMPTY:
+; SM20-NEXT: // %bb.0:
+; SM20-NEXT: ld.param.b64 %rd1, [notkernel2_param_0];
+; SM20-NEXT: ld.global.b32 %r1, [%rd1];
+; SM20-NEXT: ld.param.b64 %rd2, [notkernel2_param_1];
+; SM20-NEXT: st.b32 [%rd2], %r1;
+; SM20-NEXT: ret;
+;
; SM35-LABEL: notkernel2(
-; SM35: ld.global.b32
-define void @notkernel2(ptr addrspace(1) noalias readonly %from, ptr %to) {
+; SM35: {
+; SM35-NEXT: .reg .b32 %r<2>;
+; SM35-NEXT: .reg .b64 %rd<3>;
+; SM35-EMPTY:
+; SM35-NEXT: // %bb.0:
+; SM35-NEXT: ld.param.b64 %rd1, [notkernel2_param_0];
+; SM35-NEXT: ld.global.b32 %r1, [%rd1];
+; SM35-NEXT: ld.param.b64 %rd2, [notkernel2_param_1];
+; SM35-NEXT: st.b32 [%rd2], %r1;
+; SM35-NEXT: ret;
%1 = load float, ptr addrspace(1) %from
store float %1, ptr %to
ret void
diff --git a/llvm/test/CodeGen/NVPTX/math-intrins.ll b/llvm/test/CodeGen/NVPTX/math-intrins.ll
index 71af7a7d475d3..dde71b009d564 100644
--- a/llvm/test/CodeGen/NVPTX/math-intrins.ll
+++ b/llvm/test/CodeGen/NVPTX/math-intrins.ll
@@ -552,23 +552,21 @@ define <2 x half> @minnum_v2half(<2 x half> %a, <2 x half> %b) {
; CHECK-NOF16-LABEL: minnum_v2half(
; CHECK-NOF16: {
; CHECK-NOF16-NEXT: .reg .b16 %rs<7>;
-; CHECK-NOF16-NEXT: .reg .b32 %r<10>;
+; CHECK-NOF16-NEXT: .reg .b32 %r<8>;
; CHECK-NOF16-EMPTY:
; CHECK-NOF16-NEXT: // %bb.0:
-; CHECK-NOF16-NEXT: ld.param.b32 %r1, [minnum_v2half_param_0];
-; CHECK-NOF16-NEXT: ld.param.b32 %r2, [minnum_v2half_param_1];
-; CHECK-NOF16-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r3, %rs2;
-; CHECK-NOF16-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r4, %rs4;
-; CHECK-NOF16-NEXT: min.f32 %r5, %r4, %r3;
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 %rs5, %r5;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r6, %rs1;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r7, %rs3;
-; CHECK-NOF16-NEXT: min.f32 %r8, %r7, %r6;
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 %rs6, %r8;
-; CHECK-NOF16-NEXT: mov.b32 %r9, {%rs6, %rs5};
-; CHECK-NOF16-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-NOF16-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [minnum_v2half_param_0];
+; CHECK-NOF16-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [minnum_v2half_param_1];
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r1, %rs4;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r2, %rs2;
+; CHECK-NOF16-NEXT: min.f32 %r3, %r2, %r1;
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 %rs5, %r3;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r4, %rs3;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r5, %rs1;
+; CHECK-NOF16-NEXT: min.f32 %r6, %r5, %r4;
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 %rs6, %r6;
+; CHECK-NOF16-NEXT: mov.b32 %r7, {%rs6, %rs5};
+; CHECK-NOF16-NEXT: st.param.b32 [func_retval0], %r7;
; CHECK-NOF16-NEXT: ret;
;
; CHECK-F16-LABEL: minnum_v2half(
@@ -576,32 +574,30 @@ define <2 x half> @minnum_v2half(<2 x half> %a, <2 x half> %b) {
; CHECK-F16-NEXT: .reg .b32 %r<4>;
; CHECK-F16-EMPTY:
; CHECK-F16-NEXT: // %bb.0:
-; CHECK-F16-NEXT: ld.param.b32 %r1, [minnum_v2half_param_1];
-; CHECK-F16-NEXT: ld.param.b32 %r2, [minnum_v2half_param_0];
-; CHECK-F16-NEXT: min.f16x2 %r3, %r2, %r1;
+; CHECK-F16-NEXT: ld.param.b32 %r1, [minnum_v2half_param_0];
+; CHECK-F16-NEXT: ld.param.b32 %r2, [minnum_v2half_param_1];
+; CHECK-F16-NEXT: min.f16x2 %r3, %r1, %r2;
; CHECK-F16-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-F16-NEXT: ret;
;
; CHECK-SM80-NOF16-LABEL: minnum_v2half(
; CHECK-SM80-NOF16: {
; CHECK-SM80-NOF16-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM80-NOF16-NEXT: .reg .b32 %r<10>;
+; CHECK-SM80-NOF16-NEXT: .reg .b32 %r<8>;
; CHECK-SM80-NOF16-EMPTY:
; CHECK-SM80-NOF16-NEXT: // %bb.0:
-; CHECK-SM80-NOF16-NEXT: ld.param.b32 %r1, [minnum_v2half_param_0];
-; CHECK-SM80-NOF16-NEXT: ld.param.b32 %r2, [minnum_v2half_param_1];
-; CHECK-SM80-NOF16-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r3, %rs2;
-; CHECK-SM80-NOF16-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r4, %rs4;
-; CHECK-SM80-NOF16-NEXT: min.f32 %r5, %r4, %r3;
-; CHECK-SM80-NOF16-NEXT: cvt.rn.f16.f32 %rs5, %r5;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r6, %rs1;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r7, %rs3;
-; CHECK-SM80-NOF16-NEXT: min.f32 %r8, %r7, %r6;
-; CHECK-SM80-NOF16-NEXT: cvt.rn.f16.f32 %rs6, %r8;
-; CHECK-SM80-NOF16-NEXT: mov.b32 %r9, {%rs6, %rs5};
-; CHECK-SM80-NOF16-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-SM80-NOF16-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [minnum_v2half_param_0];
+; CHECK-SM80-NOF16-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [minnum_v2half_param_1];
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r1, %rs4;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r2, %rs2;
+; CHECK-SM80-NOF16-NEXT: min.f32 %r3, %r2, %r1;
+; CHECK-SM80-NOF16-NEXT: cvt.rn.f16.f32 %rs5, %r3;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r4, %rs3;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r5, %rs1;
+; CHECK-SM80-NOF16-NEXT: min.f32 %r6, %r5, %r4;
+; CHECK-SM80-NOF16-NEXT: cvt.rn.f16.f32 %rs6, %r6;
+; CHECK-SM80-NOF16-NEXT: mov.b32 %r7, {%rs6, %rs5};
+; CHECK-SM80-NOF16-NEXT: st.param.b32 [func_retval0], %r7;
; CHECK-SM80-NOF16-NEXT: ret;
%x = call <2 x half> @llvm.minnum.v2f16(<2 x half> %a, <2 x half> %b)
ret <2 x half> %x
@@ -877,41 +873,38 @@ define <2 x half> @minimum_v2half(<2 x half> %a, <2 x half> %b) {
; CHECK-NOF16: {
; CHECK-NOF16-NEXT: .reg .pred %p<11>;
; CHECK-NOF16-NEXT: .reg .b16 %rs<15>;
-; CHECK-NOF16-NEXT: .reg .b32 %r<10>;
+; CHECK-NOF16-NEXT: .reg .b32 %r<7>;
; CHECK-NOF16-EMPTY:
; CHECK-NOF16-NEXT: // %bb.0:
-; CHECK-NOF16-NEXT: ld.param.b32 %r1, [minimum_v2half_param_0];
-; CHECK-NOF16-NEXT: ld.param.b32 %r2, [minimum_v2half_param_1];
-; CHECK-NOF16-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r3, %rs2;
-; CHECK-NOF16-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r4, %rs4;
-; CHECK-NOF16-NEXT: setp.lt.f32 %p1, %r4, %r3;
-; CHECK-NOF16-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; CHECK-NOF16-NEXT: setp.nan.f32 %p2, %r4, %r3;
+; CHECK-NOF16-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [minimum_v2half_param_0];
+; CHECK-NOF16-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [minimum_v2half_param_1];
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r1, %rs4;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r2, %rs2;
+; CHECK-NOF16-NEXT: setp.lt.f32 %p1, %r2, %r1;
+; CHECK-NOF16-NEXT: selp.b16 %rs5, %rs2, %rs4, %p1;
+; CHECK-NOF16-NEXT: setp.nan.f32 %p2, %r2, %r1;
; CHECK-NOF16-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; CHECK-NOF16-NEXT: setp.eq.s16 %p3, %rs4, -32768;
-; CHECK-NOF16-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; CHECK-NOF16-NEXT: setp.eq.s16 %p4, %rs2, -32768;
-; CHECK-NOF16-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r5, %rs6;
-; CHECK-NOF16-NEXT: setp.eq.f32 %p5, %r5, 0f00000000;
+; CHECK-NOF16-NEXT: setp.eq.s16 %p3, %rs2, -32768;
+; CHECK-NOF16-NEXT: selp.b16 %rs7, %rs2, %rs6, %p3;
+; CHECK-NOF16-NEXT: setp.eq.s16 %p4, %rs4, -32768;
+; CHECK-NOF16-NEXT: selp.b16 %rs8, %rs4, %rs7, %p4;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r3, %rs6;
+; CHECK-NOF16-NEXT: setp.eq.f32 %p5, %r3, 0f00000000;
; CHECK-NOF16-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r6, %rs1;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r7, %rs3;
-; CHECK-NOF16-NEXT: setp.lt.f32 %p6, %r7, %r6;
-; CHECK-NOF16-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; CHECK-NOF16-NEXT: setp.nan.f32 %p7, %r7, %r6;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r4, %rs3;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r5, %rs1;
+; CHECK-NOF16-NEXT: setp.lt.f32 %p6, %r5, %r4;
+; CHECK-NOF16-NEXT: selp.b16 %rs10, %rs1, %rs3, %p6;
+; CHECK-NOF16-NEXT: setp.nan.f32 %p7, %r5, %r4;
; CHECK-NOF16-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
-; CHECK-NOF16-NEXT: setp.eq.s16 %p8, %rs3, -32768;
-; CHECK-NOF16-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; CHECK-NOF16-NEXT: setp.eq.s16 %p9, %rs1, -32768;
-; CHECK-NOF16-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r8, %rs11;
-; CHECK-NOF16-NEXT: setp.eq.f32 %p10, %r8, 0f00000000;
+; CHECK-NOF16-NEXT: setp.eq.s16 %p8, %rs1, -32768;
+; CHECK-NOF16-NEXT: selp.b16 %rs12, %rs1, %rs11, %p8;
+; CHECK-NOF16-NEXT: setp.eq.s16 %p9, %rs3, -32768;
+; CHECK-NOF16-NEXT: selp.b16 %rs13, %rs3, %rs12, %p9;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r6, %rs11;
+; CHECK-NOF16-NEXT: setp.eq.f32 %p10, %r6, 0f00000000;
; CHECK-NOF16-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; CHECK-NOF16-NEXT: mov.b32 %r9, {%rs14, %rs9};
-; CHECK-NOF16-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-NOF16-NEXT: st.param.v2.b16 [func_retval0], {%rs14, %rs9};
; CHECK-NOF16-NEXT: ret;
;
; CHECK-F16-LABEL: minimum_v2half(
@@ -919,9 +912,9 @@ define <2 x half> @minimum_v2half(<2 x half> %a, <2 x half> %b) {
; CHECK-F16-NEXT: .reg .b32 %r<4>;
; CHECK-F16-EMPTY:
; CHECK-F16-NEXT: // %bb.0:
-; CHECK-F16-NEXT: ld.param.b32 %r1, [minimum_v2half_param_1];
-; CHECK-F16-NEXT: ld.param.b32 %r2, [minimum_v2half_param_0];
-; CHECK-F16-NEXT: min.NaN.f16x2 %r3, %r2, %r1;
+; CHECK-F16-NEXT: ld.param.b32 %r1, [minimum_v2half_param_0];
+; CHECK-F16-NEXT: ld.param.b32 %r2, [minimum_v2half_param_1];
+; CHECK-F16-NEXT: min.NaN.f16x2 %r3, %r1, %r2;
; CHECK-F16-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-F16-NEXT: ret;
;
@@ -929,41 +922,38 @@ define <2 x half> @minimum_v2half(<2 x half> %a, <2 x half> %b) {
; CHECK-SM80-NOF16: {
; CHECK-SM80-NOF16-NEXT: .reg .pred %p<11>;
; CHECK-SM80-NOF16-NEXT: .reg .b16 %rs<15>;
-; CHECK-SM80-NOF16-NEXT: .reg .b32 %r<10>;
+; CHECK-SM80-NOF16-NEXT: .reg .b32 %r<7>;
; CHECK-SM80-NOF16-EMPTY:
; CHECK-SM80-NOF16-NEXT: // %bb.0:
-; CHECK-SM80-NOF16-NEXT: ld.param.b32 %r1, [minimum_v2half_param_0];
-; CHECK-SM80-NOF16-NEXT: ld.param.b32 %r2, [minimum_v2half_param_1];
-; CHECK-SM80-NOF16-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r3, %rs2;
-; CHECK-SM80-NOF16-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r4, %rs4;
-; CHECK-SM80-NOF16-NEXT: setp.lt.f32 %p1, %r4, %r3;
-; CHECK-SM80-NOF16-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; CHECK-SM80-NOF16-NEXT: setp.nan.f32 %p2, %r4, %r3;
+; CHECK-SM80-NOF16-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [minimum_v2half_param_0];
+; CHECK-SM80-NOF16-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [minimum_v2half_param_1];
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r1, %rs4;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r2, %rs2;
+; CHECK-SM80-NOF16-NEXT: setp.lt.f32 %p1, %r2, %r1;
+; CHECK-SM80-NOF16-NEXT: selp.b16 %rs5, %rs2, %rs4, %p1;
+; CHECK-SM80-NOF16-NEXT: setp.nan.f32 %p2, %r2, %r1;
; CHECK-SM80-NOF16-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p3, %rs4, -32768;
-; CHECK-SM80-NOF16-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p4, %rs2, -32768;
-; CHECK-SM80-NOF16-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r5, %rs6;
-; CHECK-SM80-NOF16-NEXT: setp.eq.f32 %p5, %r5, 0f00000000;
+; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p3, %rs2, -32768;
+; CHECK-SM80-NOF16-NEXT: selp.b16 %rs7, %rs2, %rs6, %p3;
+; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p4, %rs4, -32768;
+; CHECK-SM80-NOF16-NEXT: selp.b16 %rs8, %rs4, %rs7, %p4;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r3, %rs6;
+; CHECK-SM80-NOF16-NEXT: setp.eq.f32 %p5, %r3, 0f00000000;
; CHECK-SM80-NOF16-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r6, %rs1;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r7, %rs3;
-; CHECK-SM80-NOF16-NEXT: setp.lt.f32 %p6, %r7, %r6;
-; CHECK-SM80-NOF16-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; CHECK-SM80-NOF16-NEXT: setp.nan.f32 %p7, %r7, %r6;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r4, %rs3;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r5, %rs1;
+; CHECK-SM80-NOF16-NEXT: setp.lt.f32 %p6, %r5, %r4;
+; CHECK-SM80-NOF16-NEXT: selp.b16 %rs10, %rs1, %rs3, %p6;
+; CHECK-SM80-NOF16-NEXT: setp.nan.f32 %p7, %r5, %r4;
; CHECK-SM80-NOF16-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
-; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p8, %rs3, -32768;
-; CHECK-SM80-NOF16-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p9, %rs1, -32768;
-; CHECK-SM80-NOF16-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r8, %rs11;
-; CHECK-SM80-NOF16-NEXT: setp.eq.f32 %p10, %r8, 0f00000000;
+; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p8, %rs1, -32768;
+; CHECK-SM80-NOF16-NEXT: selp.b16 %rs12, %rs1, %rs11, %p8;
+; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p9, %rs3, -32768;
+; CHECK-SM80-NOF16-NEXT: selp.b16 %rs13, %rs3, %rs12, %p9;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r6, %rs11;
+; CHECK-SM80-NOF16-NEXT: setp.eq.f32 %p10, %r6, 0f00000000;
; CHECK-SM80-NOF16-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; CHECK-SM80-NOF16-NEXT: mov.b32 %r9, {%rs14, %rs9};
-; CHECK-SM80-NOF16-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-SM80-NOF16-NEXT: st.param.v2.b16 [func_retval0], {%rs14, %rs9};
; CHECK-SM80-NOF16-NEXT: ret;
%x = call <2 x half> @llvm.minimum.v2f16(<2 x half> %a, <2 x half> %b)
ret <2 x half> %x
@@ -1093,23 +1083,21 @@ define <2 x half> @maxnum_v2half(<2 x half> %a, <2 x half> %b) {
; CHECK-NOF16-LABEL: maxnum_v2half(
; CHECK-NOF16: {
; CHECK-NOF16-NEXT: .reg .b16 %rs<7>;
-; CHECK-NOF16-NEXT: .reg .b32 %r<10>;
+; CHECK-NOF16-NEXT: .reg .b32 %r<8>;
; CHECK-NOF16-EMPTY:
; CHECK-NOF16-NEXT: // %bb.0:
-; CHECK-NOF16-NEXT: ld.param.b32 %r1, [maxnum_v2half_param_0];
-; CHECK-NOF16-NEXT: ld.param.b32 %r2, [maxnum_v2half_param_1];
-; CHECK-NOF16-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r3, %rs2;
-; CHECK-NOF16-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r4, %rs4;
-; CHECK-NOF16-NEXT: max.f32 %r5, %r4, %r3;
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 %rs5, %r5;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r6, %rs1;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r7, %rs3;
-; CHECK-NOF16-NEXT: max.f32 %r8, %r7, %r6;
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 %rs6, %r8;
-; CHECK-NOF16-NEXT: mov.b32 %r9, {%rs6, %rs5};
-; CHECK-NOF16-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-NOF16-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [maxnum_v2half_param_0];
+; CHECK-NOF16-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [maxnum_v2half_param_1];
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r1, %rs4;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r2, %rs2;
+; CHECK-NOF16-NEXT: max.f32 %r3, %r2, %r1;
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 %rs5, %r3;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r4, %rs3;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r5, %rs1;
+; CHECK-NOF16-NEXT: max.f32 %r6, %r5, %r4;
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 %rs6, %r6;
+; CHECK-NOF16-NEXT: mov.b32 %r7, {%rs6, %rs5};
+; CHECK-NOF16-NEXT: st.param.b32 [func_retval0], %r7;
; CHECK-NOF16-NEXT: ret;
;
; CHECK-F16-LABEL: maxnum_v2half(
@@ -1117,32 +1105,30 @@ define <2 x half> @maxnum_v2half(<2 x half> %a, <2 x half> %b) {
; CHECK-F16-NEXT: .reg .b32 %r<4>;
; CHECK-F16-EMPTY:
; CHECK-F16-NEXT: // %bb.0:
-; CHECK-F16-NEXT: ld.param.b32 %r1, [maxnum_v2half_param_1];
-; CHECK-F16-NEXT: ld.param.b32 %r2, [maxnum_v2half_param_0];
-; CHECK-F16-NEXT: max.f16x2 %r3, %r2, %r1;
+; CHECK-F16-NEXT: ld.param.b32 %r1, [maxnum_v2half_param_0];
+; CHECK-F16-NEXT: ld.param.b32 %r2, [maxnum_v2half_param_1];
+; CHECK-F16-NEXT: max.f16x2 %r3, %r1, %r2;
; CHECK-F16-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-F16-NEXT: ret;
;
; CHECK-SM80-NOF16-LABEL: maxnum_v2half(
; CHECK-SM80-NOF16: {
; CHECK-SM80-NOF16-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM80-NOF16-NEXT: .reg .b32 %r<10>;
+; CHECK-SM80-NOF16-NEXT: .reg .b32 %r<8>;
; CHECK-SM80-NOF16-EMPTY:
; CHECK-SM80-NOF16-NEXT: // %bb.0:
-; CHECK-SM80-NOF16-NEXT: ld.param.b32 %r1, [maxnum_v2half_param_0];
-; CHECK-SM80-NOF16-NEXT: ld.param.b32 %r2, [maxnum_v2half_param_1];
-; CHECK-SM80-NOF16-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r3, %rs2;
-; CHECK-SM80-NOF16-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r4, %rs4;
-; CHECK-SM80-NOF16-NEXT: max.f32 %r5, %r4, %r3;
-; CHECK-SM80-NOF16-NEXT: cvt.rn.f16.f32 %rs5, %r5;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r6, %rs1;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r7, %rs3;
-; CHECK-SM80-NOF16-NEXT: max.f32 %r8, %r7, %r6;
-; CHECK-SM80-NOF16-NEXT: cvt.rn.f16.f32 %rs6, %r8;
-; CHECK-SM80-NOF16-NEXT: mov.b32 %r9, {%rs6, %rs5};
-; CHECK-SM80-NOF16-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-SM80-NOF16-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [maxnum_v2half_param_0];
+; CHECK-SM80-NOF16-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [maxnum_v2half_param_1];
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r1, %rs4;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r2, %rs2;
+; CHECK-SM80-NOF16-NEXT: max.f32 %r3, %r2, %r1;
+; CHECK-SM80-NOF16-NEXT: cvt.rn.f16.f32 %rs5, %r3;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r4, %rs3;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r5, %rs1;
+; CHECK-SM80-NOF16-NEXT: max.f32 %r6, %r5, %r4;
+; CHECK-SM80-NOF16-NEXT: cvt.rn.f16.f32 %rs6, %r6;
+; CHECK-SM80-NOF16-NEXT: mov.b32 %r7, {%rs6, %rs5};
+; CHECK-SM80-NOF16-NEXT: st.param.b32 [func_retval0], %r7;
; CHECK-SM80-NOF16-NEXT: ret;
%x = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %a, <2 x half> %b)
ret <2 x half> %x
@@ -1414,41 +1400,38 @@ define <2 x half> @maximum_v2half(<2 x half> %a, <2 x half> %b) {
; CHECK-NOF16: {
; CHECK-NOF16-NEXT: .reg .pred %p<11>;
; CHECK-NOF16-NEXT: .reg .b16 %rs<15>;
-; CHECK-NOF16-NEXT: .reg .b32 %r<10>;
+; CHECK-NOF16-NEXT: .reg .b32 %r<7>;
; CHECK-NOF16-EMPTY:
; CHECK-NOF16-NEXT: // %bb.0:
-; CHECK-NOF16-NEXT: ld.param.b32 %r1, [maximum_v2half_param_0];
-; CHECK-NOF16-NEXT: ld.param.b32 %r2, [maximum_v2half_param_1];
-; CHECK-NOF16-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r3, %rs2;
-; CHECK-NOF16-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r4, %rs4;
-; CHECK-NOF16-NEXT: setp.gt.f32 %p1, %r4, %r3;
-; CHECK-NOF16-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; CHECK-NOF16-NEXT: setp.nan.f32 %p2, %r4, %r3;
+; CHECK-NOF16-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [maximum_v2half_param_0];
+; CHECK-NOF16-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [maximum_v2half_param_1];
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r1, %rs4;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r2, %rs2;
+; CHECK-NOF16-NEXT: setp.gt.f32 %p1, %r2, %r1;
+; CHECK-NOF16-NEXT: selp.b16 %rs5, %rs2, %rs4, %p1;
+; CHECK-NOF16-NEXT: setp.nan.f32 %p2, %r2, %r1;
; CHECK-NOF16-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; CHECK-NOF16-NEXT: setp.eq.s16 %p3, %rs4, 0;
-; CHECK-NOF16-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; CHECK-NOF16-NEXT: setp.eq.s16 %p4, %rs2, 0;
-; CHECK-NOF16-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r5, %rs6;
-; CHECK-NOF16-NEXT: setp.eq.f32 %p5, %r5, 0f00000000;
+; CHECK-NOF16-NEXT: setp.eq.s16 %p3, %rs2, 0;
+; CHECK-NOF16-NEXT: selp.b16 %rs7, %rs2, %rs6, %p3;
+; CHECK-NOF16-NEXT: setp.eq.s16 %p4, %rs4, 0;
+; CHECK-NOF16-NEXT: selp.b16 %rs8, %rs4, %rs7, %p4;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r3, %rs6;
+; CHECK-NOF16-NEXT: setp.eq.f32 %p5, %r3, 0f00000000;
; CHECK-NOF16-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r6, %rs1;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r7, %rs3;
-; CHECK-NOF16-NEXT: setp.gt.f32 %p6, %r7, %r6;
-; CHECK-NOF16-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; CHECK-NOF16-NEXT: setp.nan.f32 %p7, %r7, %r6;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r4, %rs3;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r5, %rs1;
+; CHECK-NOF16-NEXT: setp.gt.f32 %p6, %r5, %r4;
+; CHECK-NOF16-NEXT: selp.b16 %rs10, %rs1, %rs3, %p6;
+; CHECK-NOF16-NEXT: setp.nan.f32 %p7, %r5, %r4;
; CHECK-NOF16-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
-; CHECK-NOF16-NEXT: setp.eq.s16 %p8, %rs3, 0;
-; CHECK-NOF16-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; CHECK-NOF16-NEXT: setp.eq.s16 %p9, %rs1, 0;
-; CHECK-NOF16-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; CHECK-NOF16-NEXT: cvt.f32.f16 %r8, %rs11;
-; CHECK-NOF16-NEXT: setp.eq.f32 %p10, %r8, 0f00000000;
+; CHECK-NOF16-NEXT: setp.eq.s16 %p8, %rs1, 0;
+; CHECK-NOF16-NEXT: selp.b16 %rs12, %rs1, %rs11, %p8;
+; CHECK-NOF16-NEXT: setp.eq.s16 %p9, %rs3, 0;
+; CHECK-NOF16-NEXT: selp.b16 %rs13, %rs3, %rs12, %p9;
+; CHECK-NOF16-NEXT: cvt.f32.f16 %r6, %rs11;
+; CHECK-NOF16-NEXT: setp.eq.f32 %p10, %r6, 0f00000000;
; CHECK-NOF16-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; CHECK-NOF16-NEXT: mov.b32 %r9, {%rs14, %rs9};
-; CHECK-NOF16-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-NOF16-NEXT: st.param.v2.b16 [func_retval0], {%rs14, %rs9};
; CHECK-NOF16-NEXT: ret;
;
; CHECK-F16-LABEL: maximum_v2half(
@@ -1456,9 +1439,9 @@ define <2 x half> @maximum_v2half(<2 x half> %a, <2 x half> %b) {
; CHECK-F16-NEXT: .reg .b32 %r<4>;
; CHECK-F16-EMPTY:
; CHECK-F16-NEXT: // %bb.0:
-; CHECK-F16-NEXT: ld.param.b32 %r1, [maximum_v2half_param_1];
-; CHECK-F16-NEXT: ld.param.b32 %r2, [maximum_v2half_param_0];
-; CHECK-F16-NEXT: max.NaN.f16x2 %r3, %r2, %r1;
+; CHECK-F16-NEXT: ld.param.b32 %r1, [maximum_v2half_param_0];
+; CHECK-F16-NEXT: ld.param.b32 %r2, [maximum_v2half_param_1];
+; CHECK-F16-NEXT: max.NaN.f16x2 %r3, %r1, %r2;
; CHECK-F16-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-F16-NEXT: ret;
;
@@ -1466,41 +1449,38 @@ define <2 x half> @maximum_v2half(<2 x half> %a, <2 x half> %b) {
; CHECK-SM80-NOF16: {
; CHECK-SM80-NOF16-NEXT: .reg .pred %p<11>;
; CHECK-SM80-NOF16-NEXT: .reg .b16 %rs<15>;
-; CHECK-SM80-NOF16-NEXT: .reg .b32 %r<10>;
+; CHECK-SM80-NOF16-NEXT: .reg .b32 %r<7>;
; CHECK-SM80-NOF16-EMPTY:
; CHECK-SM80-NOF16-NEXT: // %bb.0:
-; CHECK-SM80-NOF16-NEXT: ld.param.b32 %r1, [maximum_v2half_param_0];
-; CHECK-SM80-NOF16-NEXT: ld.param.b32 %r2, [maximum_v2half_param_1];
-; CHECK-SM80-NOF16-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r3, %rs2;
-; CHECK-SM80-NOF16-NEXT: mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r4, %rs4;
-; CHECK-SM80-NOF16-NEXT: setp.gt.f32 %p1, %r4, %r3;
-; CHECK-SM80-NOF16-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; CHECK-SM80-NOF16-NEXT: setp.nan.f32 %p2, %r4, %r3;
+; CHECK-SM80-NOF16-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [maximum_v2half_param_0];
+; CHECK-SM80-NOF16-NEXT: ld.param.v2.b16 {%rs3, %rs4}, [maximum_v2half_param_1];
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r1, %rs4;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r2, %rs2;
+; CHECK-SM80-NOF16-NEXT: setp.gt.f32 %p1, %r2, %r1;
+; CHECK-SM80-NOF16-NEXT: selp.b16 %rs5, %rs2, %rs4, %p1;
+; CHECK-SM80-NOF16-NEXT: setp.nan.f32 %p2, %r2, %r1;
; CHECK-SM80-NOF16-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p3, %rs4, 0;
-; CHECK-SM80-NOF16-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p4, %rs2, 0;
-; CHECK-SM80-NOF16-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r5, %rs6;
-; CHECK-SM80-NOF16-NEXT: setp.eq.f32 %p5, %r5, 0f00000000;
+; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p3, %rs2, 0;
+; CHECK-SM80-NOF16-NEXT: selp.b16 %rs7, %rs2, %rs6, %p3;
+; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p4, %rs4, 0;
+; CHECK-SM80-NOF16-NEXT: selp.b16 %rs8, %rs4, %rs7, %p4;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r3, %rs6;
+; CHECK-SM80-NOF16-NEXT: setp.eq.f32 %p5, %r3, 0f00000000;
; CHECK-SM80-NOF16-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r6, %rs1;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r7, %rs3;
-; CHECK-SM80-NOF16-NEXT: setp.gt.f32 %p6, %r7, %r6;
-; CHECK-SM80-NOF16-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; CHECK-SM80-NOF16-NEXT: setp.nan.f32 %p7, %r7, %r6;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r4, %rs3;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r5, %rs1;
+; CHECK-SM80-NOF16-NEXT: setp.gt.f32 %p6, %r5, %r4;
+; CHECK-SM80-NOF16-NEXT: selp.b16 %rs10, %rs1, %rs3, %p6;
+; CHECK-SM80-NOF16-NEXT: setp.nan.f32 %p7, %r5, %r4;
; CHECK-SM80-NOF16-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
-; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p8, %rs3, 0;
-; CHECK-SM80-NOF16-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p9, %rs1, 0;
-; CHECK-SM80-NOF16-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r8, %rs11;
-; CHECK-SM80-NOF16-NEXT: setp.eq.f32 %p10, %r8, 0f00000000;
+; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p8, %rs1, 0;
+; CHECK-SM80-NOF16-NEXT: selp.b16 %rs12, %rs1, %rs11, %p8;
+; CHECK-SM80-NOF16-NEXT: setp.eq.s16 %p9, %rs3, 0;
+; CHECK-SM80-NOF16-NEXT: selp.b16 %rs13, %rs3, %rs12, %p9;
+; CHECK-SM80-NOF16-NEXT: cvt.f32.f16 %r6, %rs11;
+; CHECK-SM80-NOF16-NEXT: setp.eq.f32 %p10, %r6, 0f00000000;
; CHECK-SM80-NOF16-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; CHECK-SM80-NOF16-NEXT: mov.b32 %r9, {%rs14, %rs9};
-; CHECK-SM80-NOF16-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-SM80-NOF16-NEXT: st.param.v2.b16 [func_retval0], {%rs14, %rs9};
; CHECK-SM80-NOF16-NEXT: ret;
%x = call <2 x half> @llvm.maximum.v2f16(<2 x half> %a, <2 x half> %b)
ret <2 x half> %x
diff --git a/llvm/test/CodeGen/NVPTX/param-load-store.ll b/llvm/test/CodeGen/NVPTX/param-load-store.ll
index ce6707c4564bf..4bea710e6dd93 100644
--- a/llvm/test/CodeGen/NVPTX/param-load-store.ll
+++ b/llvm/test/CodeGen/NVPTX/param-load-store.ll
@@ -323,9 +323,8 @@ define signext i16 @test_i16s(i16 signext %a) {
; CHECK: .func (.param .align 8 .b8 func_retval0[8])
; CHECK-LABEL: test_v3i16(
; CHECK-NEXT: .param .align 8 .b8 test_v3i16_param_0[8]
-; CHECK-DAG: ld.param.b16 [[E2:%rs[0-9]+]], [test_v3i16_param_0+4];
-; CHECK-DAG: ld.param.b32 [[R:%r[0-9]+]], [test_v3i16_param_0];
-; CHECK-DAG: mov.b32 {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]]}, [[R]];
+; CHECK-DAG: ld.param.b16 [[E2:%rs[0-9]+]], [test_v3i16_param_0+4];
+; CHECK-DAG: ld.param.v2.b16 {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]]}, [test_v3i16_param_0];
; CHECK: .param .align 8 .b8 param0[8];
; CHECK: st.param.v2.b16 [param0], {[[E0]], [[E1]]};
; CHECK: st.param.b16 [param0+4], [[E2]];
@@ -452,8 +451,7 @@ define <2 x bfloat> @test_v2bf16(<2 x bfloat> %a) {
; CHECK:.func (.param .align 8 .b8 func_retval0[8])
; CHECK-LABEL: test_v3f16(
; CHECK: .param .align 8 .b8 test_v3f16_param_0[8]
-; CHECK-DAG: ld.param.b32 [[HH01:%r[0-9]+]], [test_v3f16_param_0];
-; CHECK-DAG: mov.b32 {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]]}, [[HH01]];
+; CHECK-DAG: ld.param.v2.b16 {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]]}, [test_v3f16_param_0];
; CHECK-DAG: ld.param.b16 [[E2:%rs[0-9]+]], [test_v3f16_param_0+4];
; CHECK: .param .align 8 .b8 param0[8];
; CHECK-DAG: st.param.v2.b16 [param0], {[[E0]], [[E1]]};
diff --git a/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll b/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
index d5b451dad7bc3..e10949f95fac4 100644
--- a/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
+++ b/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
@@ -23,19 +23,19 @@ define half @reduce_fadd_half(<8 x half> %in) {
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fadd_half_param_0];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: mov.b16 %rs3, 0x0000;
-; CHECK-NEXT: add.rn.f16 %rs4, %rs1, %rs3;
-; CHECK-NEXT: add.rn.f16 %rs5, %rs4, %rs2;
-; CHECK-NEXT: mov.b32 {%rs6, %rs7}, %r2;
-; CHECK-NEXT: add.rn.f16 %rs8, %rs5, %rs6;
-; CHECK-NEXT: add.rn.f16 %rs9, %rs8, %rs7;
-; CHECK-NEXT: mov.b32 {%rs10, %rs11}, %r3;
-; CHECK-NEXT: add.rn.f16 %rs12, %rs9, %rs10;
-; CHECK-NEXT: add.rn.f16 %rs13, %rs12, %rs11;
-; CHECK-NEXT: mov.b32 {%rs14, %rs15}, %r4;
-; CHECK-NEXT: add.rn.f16 %rs16, %rs13, %rs14;
-; CHECK-NEXT: add.rn.f16 %rs17, %rs16, %rs15;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; CHECK-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-NEXT: mov.b16 %rs9, 0x0000;
+; CHECK-NEXT: add.rn.f16 %rs10, %rs7, %rs9;
+; CHECK-NEXT: add.rn.f16 %rs11, %rs10, %rs8;
+; CHECK-NEXT: add.rn.f16 %rs12, %rs11, %rs5;
+; CHECK-NEXT: add.rn.f16 %rs13, %rs12, %rs6;
+; CHECK-NEXT: add.rn.f16 %rs14, %rs13, %rs3;
+; CHECK-NEXT: add.rn.f16 %rs15, %rs14, %rs4;
+; CHECK-NEXT: add.rn.f16 %rs16, %rs15, %rs1;
+; CHECK-NEXT: add.rn.f16 %rs17, %rs16, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs17;
; CHECK-NEXT: ret;
%res = call half @llvm.vector.reduce.fadd(half 0.0, <8 x half> %in)
@@ -90,12 +90,10 @@ define half @reduce_fadd_half_reassoc_nonpow2(<7 x half> %in) {
; CHECK-LABEL: reduce_fadd_half_reassoc_nonpow2(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<16>;
-; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [reduce_fadd_half_reassoc_nonpow2_param_0+8];
-; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r1;
; CHECK-NEXT: ld.param.b16 %rs7, [reduce_fadd_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [reduce_fadd_half_reassoc_nonpow2_param_0+8];
; CHECK-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fadd_half_reassoc_nonpow2_param_0];
; CHECK-NEXT: mov.b16 %rs8, 0x0000;
; CHECK-NEXT: add.rn.f16 %rs9, %rs1, %rs8;
@@ -187,17 +185,17 @@ define half @reduce_fmul_half(<8 x half> %in) {
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmul_half_param_0];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT: mul.rn.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT: mov.b32 {%rs4, %rs5}, %r2;
-; CHECK-NEXT: mul.rn.f16 %rs6, %rs3, %rs4;
-; CHECK-NEXT: mul.rn.f16 %rs7, %rs6, %rs5;
-; CHECK-NEXT: mov.b32 {%rs8, %rs9}, %r3;
-; CHECK-NEXT: mul.rn.f16 %rs10, %rs7, %rs8;
-; CHECK-NEXT: mul.rn.f16 %rs11, %rs10, %rs9;
-; CHECK-NEXT: mov.b32 {%rs12, %rs13}, %r4;
-; CHECK-NEXT: mul.rn.f16 %rs14, %rs11, %rs12;
-; CHECK-NEXT: mul.rn.f16 %rs15, %rs14, %rs13;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; CHECK-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-NEXT: mul.rn.f16 %rs9, %rs7, %rs8;
+; CHECK-NEXT: mul.rn.f16 %rs10, %rs9, %rs5;
+; CHECK-NEXT: mul.rn.f16 %rs11, %rs10, %rs6;
+; CHECK-NEXT: mul.rn.f16 %rs12, %rs11, %rs3;
+; CHECK-NEXT: mul.rn.f16 %rs13, %rs12, %rs4;
+; CHECK-NEXT: mul.rn.f16 %rs14, %rs13, %rs1;
+; CHECK-NEXT: mul.rn.f16 %rs15, %rs14, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs15;
; CHECK-NEXT: ret;
%res = call half @llvm.vector.reduce.fmul(half 1.0, <8 x half> %in)
@@ -253,15 +251,15 @@ define half @reduce_fmul_half_reassoc_nonpow2(<7 x half> %in) {
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [reduce_fmul_half_reassoc_nonpow2_param_0+8];
; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fmul_half_reassoc_nonpow2_param_0];
-; CHECK-NEXT: mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT: mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT: ld.param.v2.b32 {%r2, %r3}, [reduce_fmul_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; CHECK-NEXT: ld.param.b16 %rs7, [reduce_fmul_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT: mul.rn.f16x2 %r4, %r2, %r1;
; CHECK-NEXT: mov.b16 %rs8, 0x3C00;
-; CHECK-NEXT: mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT: mul.rn.f16x2 %r5, %r3, %r4;
-; CHECK-NEXT: mul.rn.f16x2 %r6, %r2, %r1;
-; CHECK-NEXT: mul.rn.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT: mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT: mul.rn.f16x2 %r6, %r3, %r5;
+; CHECK-NEXT: mul.rn.f16x2 %r7, %r4, %r6;
; CHECK-NEXT: mov.b32 {%rs9, %rs10}, %r7;
; CHECK-NEXT: mul.rn.f16 %rs11, %rs9, %rs10;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs11;
@@ -382,15 +380,15 @@ define half @reduce_fmax_half_reassoc_nonpow2(<7 x half> %in) {
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [reduce_fmax_half_reassoc_nonpow2_param_0+8];
; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fmax_half_reassoc_nonpow2_param_0];
-; CHECK-NEXT: mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT: mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT: ld.param.v2.b32 {%r2, %r3}, [reduce_fmax_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; CHECK-NEXT: ld.param.b16 %rs7, [reduce_fmax_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT: max.f16x2 %r4, %r2, %r1;
; CHECK-NEXT: mov.b16 %rs8, 0xFE00;
-; CHECK-NEXT: mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT: max.f16x2 %r5, %r3, %r4;
-; CHECK-NEXT: max.f16x2 %r6, %r2, %r1;
-; CHECK-NEXT: max.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT: mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT: max.f16x2 %r6, %r3, %r5;
+; CHECK-NEXT: max.f16x2 %r7, %r4, %r6;
; CHECK-NEXT: mov.b32 {%rs9, %rs10}, %r7;
; CHECK-NEXT: max.f16 %rs11, %rs9, %rs10;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs11;
@@ -514,15 +512,15 @@ define half @reduce_fmin_half_reassoc_nonpow2(<7 x half> %in) {
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [reduce_fmin_half_reassoc_nonpow2_param_0+8];
; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fmin_half_reassoc_nonpow2_param_0];
-; CHECK-NEXT: mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT: mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT: ld.param.v2.b32 {%r2, %r3}, [reduce_fmin_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; CHECK-NEXT: ld.param.b16 %rs7, [reduce_fmin_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT: min.f16x2 %r4, %r2, %r1;
; CHECK-NEXT: mov.b16 %rs8, 0x7E00;
-; CHECK-NEXT: mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT: min.f16x2 %r5, %r3, %r4;
-; CHECK-NEXT: min.f16x2 %r6, %r2, %r1;
-; CHECK-NEXT: min.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT: mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT: min.f16x2 %r6, %r3, %r5;
+; CHECK-NEXT: min.f16x2 %r7, %r4, %r6;
; CHECK-NEXT: mov.b32 {%rs9, %rs10}, %r7;
; CHECK-NEXT: min.f16 %rs11, %rs9, %rs10;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs11;
@@ -646,15 +644,15 @@ define half @reduce_fmaximum_half_reassoc_nonpow2(<7 x half> %in) {
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [reduce_fmaximum_half_reassoc_nonpow2_param_0+8];
; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fmaximum_half_reassoc_nonpow2_param_0];
-; CHECK-NEXT: mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT: mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT: ld.param.v2.b32 {%r2, %r3}, [reduce_fmaximum_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; CHECK-NEXT: ld.param.b16 %rs7, [reduce_fmaximum_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT: max.NaN.f16x2 %r4, %r2, %r1;
; CHECK-NEXT: mov.b16 %rs8, 0xFC00;
-; CHECK-NEXT: mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT: max.NaN.f16x2 %r5, %r3, %r4;
-; CHECK-NEXT: max.NaN.f16x2 %r6, %r2, %r1;
-; CHECK-NEXT: max.NaN.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT: mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT: max.NaN.f16x2 %r6, %r3, %r5;
+; CHECK-NEXT: max.NaN.f16x2 %r7, %r4, %r6;
; CHECK-NEXT: mov.b32 {%rs9, %rs10}, %r7;
; CHECK-NEXT: max.NaN.f16 %rs11, %rs9, %rs10;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs11;
@@ -778,15 +776,15 @@ define half @reduce_fminimum_half_reassoc_nonpow2(<7 x half> %in) {
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [reduce_fminimum_half_reassoc_nonpow2_param_0+8];
; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fminimum_half_reassoc_nonpow2_param_0];
-; CHECK-NEXT: mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT: mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT: ld.param.v2.b32 {%r2, %r3}, [reduce_fminimum_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; CHECK-NEXT: ld.param.b16 %rs7, [reduce_fminimum_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT: min.NaN.f16x2 %r4, %r2, %r1;
; CHECK-NEXT: mov.b16 %rs8, 0x7C00;
-; CHECK-NEXT: mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT: min.NaN.f16x2 %r5, %r3, %r4;
-; CHECK-NEXT: min.NaN.f16x2 %r6, %r2, %r1;
-; CHECK-NEXT: min.NaN.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT: mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT: min.NaN.f16x2 %r6, %r3, %r5;
+; CHECK-NEXT: min.NaN.f16x2 %r7, %r4, %r6;
; CHECK-NEXT: mov.b32 {%rs9, %rs10}, %r7;
; CHECK-NEXT: min.NaN.f16 %rs11, %rs9, %rs10;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs11;
@@ -911,12 +909,11 @@ define i16 @reduce_add_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM80-LABEL: reduce_add_i16_nonpow2(
; CHECK-SM80: {
; CHECK-SM80-NEXT: .reg .b16 %rs<14>;
-; CHECK-SM80-NEXT: .reg .b32 %r<3>;
+; CHECK-SM80-NEXT: .reg .b32 %r<2>;
; CHECK-SM80-EMPTY:
; CHECK-SM80-NEXT: // %bb.0:
-; CHECK-SM80-NEXT: ld.param.b32 %r1, [reduce_add_i16_nonpow2_param_0+8];
-; CHECK-SM80-NEXT: mov.b32 {%rs5, %rs6}, %r1;
; CHECK-SM80-NEXT: ld.param.b16 %rs7, [reduce_add_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [reduce_add_i16_nonpow2_param_0+8];
; CHECK-SM80-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_add_i16_nonpow2_param_0];
; CHECK-SM80-NEXT: add.s16 %rs8, %rs3, %rs7;
; CHECK-SM80-NEXT: add.s16 %rs9, %rs1, %rs5;
@@ -924,8 +921,8 @@ define i16 @reduce_add_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM80-NEXT: add.s16 %rs11, %rs2, %rs6;
; CHECK-SM80-NEXT: add.s16 %rs12, %rs11, %rs4;
; CHECK-SM80-NEXT: add.s16 %rs13, %rs10, %rs12;
-; CHECK-SM80-NEXT: cvt.u32.u16 %r2, %rs13;
-; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT: cvt.u32.u16 %r1, %rs13;
+; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r1;
; CHECK-SM80-NEXT: ret;
;
; CHECK-SM100-LABEL: reduce_add_i16_nonpow2(
@@ -936,15 +933,15 @@ define i16 @reduce_add_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM100-NEXT: // %bb.0:
; CHECK-SM100-NEXT: ld.param.b32 %r1, [reduce_add_i16_nonpow2_param_0+8];
; CHECK-SM100-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM100-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_add_i16_nonpow2_param_0];
-; CHECK-SM100-NEXT: mov.b32 %r2, {%rs1, %rs2};
-; CHECK-SM100-NEXT: mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT: ld.param.v2.b32 {%r2, %r3}, [reduce_add_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-SM100-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; CHECK-SM100-NEXT: ld.param.b16 %rs7, [reduce_add_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT: add.s16x2 %r4, %r2, %r1;
; CHECK-SM100-NEXT: mov.b16 %rs8, 0;
-; CHECK-SM100-NEXT: mov.b32 %r4, {%rs7, %rs8};
-; CHECK-SM100-NEXT: add.s16x2 %r5, %r3, %r4;
-; CHECK-SM100-NEXT: add.s16x2 %r6, %r2, %r1;
-; CHECK-SM100-NEXT: add.s16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT: mov.b32 %r5, {%rs7, %rs8};
+; CHECK-SM100-NEXT: add.s16x2 %r6, %r3, %r5;
+; CHECK-SM100-NEXT: add.s16x2 %r7, %r4, %r6;
; CHECK-SM100-NEXT: mov.b32 {%rs9, %rs10}, %r7;
; CHECK-SM100-NEXT: add.s16 %rs11, %rs9, %rs10;
; CHECK-SM100-NEXT: cvt.u32.u16 %r8, %rs11;
@@ -1026,12 +1023,11 @@ define i16 @reduce_mul_i16_nonpow2(<7 x i16> %in) {
; CHECK-LABEL: reduce_mul_i16_nonpow2(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<14>;
-; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [reduce_mul_i16_nonpow2_param_0+8];
-; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r1;
; CHECK-NEXT: ld.param.b16 %rs7, [reduce_mul_i16_nonpow2_param_0+12];
+; CHECK-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [reduce_mul_i16_nonpow2_param_0+8];
; CHECK-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_mul_i16_nonpow2_param_0];
; CHECK-NEXT: mul.lo.s16 %rs8, %rs3, %rs7;
; CHECK-NEXT: mul.lo.s16 %rs9, %rs1, %rs5;
@@ -1039,8 +1035,8 @@ define i16 @reduce_mul_i16_nonpow2(<7 x i16> %in) {
; CHECK-NEXT: mul.lo.s16 %rs11, %rs2, %rs6;
; CHECK-NEXT: mul.lo.s16 %rs12, %rs4, %rs11;
; CHECK-NEXT: mul.lo.s16 %rs13, %rs10, %rs12;
-; CHECK-NEXT: cvt.u32.u16 %r2, %rs13;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs13;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
; CHECK-NEXT: ret;
%res = call i16 @llvm.vector.reduce.mul(<7 x i16> %in)
ret i16 %res
@@ -1137,12 +1133,11 @@ define i16 @reduce_umax_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM80-LABEL: reduce_umax_i16_nonpow2(
; CHECK-SM80: {
; CHECK-SM80-NEXT: .reg .b16 %rs<14>;
-; CHECK-SM80-NEXT: .reg .b32 %r<3>;
+; CHECK-SM80-NEXT: .reg .b32 %r<2>;
; CHECK-SM80-EMPTY:
; CHECK-SM80-NEXT: // %bb.0:
-; CHECK-SM80-NEXT: ld.param.b32 %r1, [reduce_umax_i16_nonpow2_param_0+8];
-; CHECK-SM80-NEXT: mov.b32 {%rs5, %rs6}, %r1;
; CHECK-SM80-NEXT: ld.param.b16 %rs7, [reduce_umax_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [reduce_umax_i16_nonpow2_param_0+8];
; CHECK-SM80-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_umax_i16_nonpow2_param_0];
; CHECK-SM80-NEXT: max.u16 %rs8, %rs3, %rs7;
; CHECK-SM80-NEXT: max.u16 %rs9, %rs1, %rs5;
@@ -1150,8 +1145,8 @@ define i16 @reduce_umax_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM80-NEXT: max.u16 %rs11, %rs2, %rs6;
; CHECK-SM80-NEXT: max.u16 %rs12, %rs4, %rs11;
; CHECK-SM80-NEXT: max.u16 %rs13, %rs10, %rs12;
-; CHECK-SM80-NEXT: cvt.u32.u16 %r2, %rs13;
-; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT: cvt.u32.u16 %r1, %rs13;
+; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r1;
; CHECK-SM80-NEXT: ret;
;
; CHECK-SM100-LABEL: reduce_umax_i16_nonpow2(
@@ -1162,15 +1157,15 @@ define i16 @reduce_umax_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM100-NEXT: // %bb.0:
; CHECK-SM100-NEXT: ld.param.b32 %r1, [reduce_umax_i16_nonpow2_param_0+8];
; CHECK-SM100-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM100-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_umax_i16_nonpow2_param_0];
-; CHECK-SM100-NEXT: mov.b32 %r2, {%rs1, %rs2};
-; CHECK-SM100-NEXT: mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT: ld.param.v2.b32 {%r2, %r3}, [reduce_umax_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-SM100-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; CHECK-SM100-NEXT: ld.param.b16 %rs7, [reduce_umax_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT: max.u16x2 %r4, %r2, %r1;
; CHECK-SM100-NEXT: mov.b16 %rs8, 0;
-; CHECK-SM100-NEXT: mov.b32 %r4, {%rs7, %rs8};
-; CHECK-SM100-NEXT: max.u16x2 %r5, %r3, %r4;
-; CHECK-SM100-NEXT: max.u16x2 %r6, %r2, %r1;
-; CHECK-SM100-NEXT: max.u16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT: mov.b32 %r5, {%rs7, %rs8};
+; CHECK-SM100-NEXT: max.u16x2 %r6, %r3, %r5;
+; CHECK-SM100-NEXT: max.u16x2 %r7, %r4, %r6;
; CHECK-SM100-NEXT: mov.b32 {%rs9, %rs10}, %r7;
; CHECK-SM100-NEXT: max.u16 %rs11, %rs9, %rs10;
; CHECK-SM100-NEXT: cvt.u32.u16 %r8, %rs11;
@@ -1271,12 +1266,11 @@ define i16 @reduce_umin_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM80-LABEL: reduce_umin_i16_nonpow2(
; CHECK-SM80: {
; CHECK-SM80-NEXT: .reg .b16 %rs<14>;
-; CHECK-SM80-NEXT: .reg .b32 %r<3>;
+; CHECK-SM80-NEXT: .reg .b32 %r<2>;
; CHECK-SM80-EMPTY:
; CHECK-SM80-NEXT: // %bb.0:
-; CHECK-SM80-NEXT: ld.param.b32 %r1, [reduce_umin_i16_nonpow2_param_0+8];
-; CHECK-SM80-NEXT: mov.b32 {%rs5, %rs6}, %r1;
; CHECK-SM80-NEXT: ld.param.b16 %rs7, [reduce_umin_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [reduce_umin_i16_nonpow2_param_0+8];
; CHECK-SM80-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_umin_i16_nonpow2_param_0];
; CHECK-SM80-NEXT: min.u16 %rs8, %rs3, %rs7;
; CHECK-SM80-NEXT: min.u16 %rs9, %rs1, %rs5;
@@ -1284,8 +1278,8 @@ define i16 @reduce_umin_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM80-NEXT: min.u16 %rs11, %rs2, %rs6;
; CHECK-SM80-NEXT: min.u16 %rs12, %rs4, %rs11;
; CHECK-SM80-NEXT: min.u16 %rs13, %rs10, %rs12;
-; CHECK-SM80-NEXT: cvt.u32.u16 %r2, %rs13;
-; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT: cvt.u32.u16 %r1, %rs13;
+; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r1;
; CHECK-SM80-NEXT: ret;
;
; CHECK-SM100-LABEL: reduce_umin_i16_nonpow2(
@@ -1296,15 +1290,15 @@ define i16 @reduce_umin_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM100-NEXT: // %bb.0:
; CHECK-SM100-NEXT: ld.param.b32 %r1, [reduce_umin_i16_nonpow2_param_0+8];
; CHECK-SM100-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM100-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_umin_i16_nonpow2_param_0];
-; CHECK-SM100-NEXT: mov.b32 %r2, {%rs1, %rs2};
-; CHECK-SM100-NEXT: mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT: ld.param.v2.b32 {%r2, %r3}, [reduce_umin_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-SM100-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; CHECK-SM100-NEXT: ld.param.b16 %rs7, [reduce_umin_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT: min.u16x2 %r4, %r2, %r1;
; CHECK-SM100-NEXT: mov.b16 %rs8, -1;
-; CHECK-SM100-NEXT: mov.b32 %r4, {%rs7, %rs8};
-; CHECK-SM100-NEXT: min.u16x2 %r5, %r3, %r4;
-; CHECK-SM100-NEXT: min.u16x2 %r6, %r2, %r1;
-; CHECK-SM100-NEXT: min.u16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT: mov.b32 %r5, {%rs7, %rs8};
+; CHECK-SM100-NEXT: min.u16x2 %r6, %r3, %r5;
+; CHECK-SM100-NEXT: min.u16x2 %r7, %r4, %r6;
; CHECK-SM100-NEXT: mov.b32 {%rs9, %rs10}, %r7;
; CHECK-SM100-NEXT: min.u16 %rs11, %rs9, %rs10;
; CHECK-SM100-NEXT: cvt.u32.u16 %r8, %rs11;
@@ -1405,12 +1399,11 @@ define i16 @reduce_smax_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM80-LABEL: reduce_smax_i16_nonpow2(
; CHECK-SM80: {
; CHECK-SM80-NEXT: .reg .b16 %rs<14>;
-; CHECK-SM80-NEXT: .reg .b32 %r<3>;
+; CHECK-SM80-NEXT: .reg .b32 %r<2>;
; CHECK-SM80-EMPTY:
; CHECK-SM80-NEXT: // %bb.0:
-; CHECK-SM80-NEXT: ld.param.b32 %r1, [reduce_smax_i16_nonpow2_param_0+8];
-; CHECK-SM80-NEXT: mov.b32 {%rs5, %rs6}, %r1;
; CHECK-SM80-NEXT: ld.param.b16 %rs7, [reduce_smax_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [reduce_smax_i16_nonpow2_param_0+8];
; CHECK-SM80-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_smax_i16_nonpow2_param_0];
; CHECK-SM80-NEXT: max.s16 %rs8, %rs3, %rs7;
; CHECK-SM80-NEXT: max.s16 %rs9, %rs1, %rs5;
@@ -1418,8 +1411,8 @@ define i16 @reduce_smax_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM80-NEXT: max.s16 %rs11, %rs2, %rs6;
; CHECK-SM80-NEXT: max.s16 %rs12, %rs4, %rs11;
; CHECK-SM80-NEXT: max.s16 %rs13, %rs10, %rs12;
-; CHECK-SM80-NEXT: cvt.u32.u16 %r2, %rs13;
-; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT: cvt.u32.u16 %r1, %rs13;
+; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r1;
; CHECK-SM80-NEXT: ret;
;
; CHECK-SM100-LABEL: reduce_smax_i16_nonpow2(
@@ -1430,15 +1423,15 @@ define i16 @reduce_smax_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM100-NEXT: // %bb.0:
; CHECK-SM100-NEXT: ld.param.b32 %r1, [reduce_smax_i16_nonpow2_param_0+8];
; CHECK-SM100-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM100-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_smax_i16_nonpow2_param_0];
-; CHECK-SM100-NEXT: mov.b32 %r2, {%rs1, %rs2};
-; CHECK-SM100-NEXT: mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT: ld.param.v2.b32 {%r2, %r3}, [reduce_smax_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-SM100-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; CHECK-SM100-NEXT: ld.param.b16 %rs7, [reduce_smax_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT: max.s16x2 %r4, %r2, %r1;
; CHECK-SM100-NEXT: mov.b16 %rs8, -32768;
-; CHECK-SM100-NEXT: mov.b32 %r4, {%rs7, %rs8};
-; CHECK-SM100-NEXT: max.s16x2 %r5, %r3, %r4;
-; CHECK-SM100-NEXT: max.s16x2 %r6, %r2, %r1;
-; CHECK-SM100-NEXT: max.s16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT: mov.b32 %r5, {%rs7, %rs8};
+; CHECK-SM100-NEXT: max.s16x2 %r6, %r3, %r5;
+; CHECK-SM100-NEXT: max.s16x2 %r7, %r4, %r6;
; CHECK-SM100-NEXT: mov.b32 {%rs9, %rs10}, %r7;
; CHECK-SM100-NEXT: max.s16 %rs11, %rs9, %rs10;
; CHECK-SM100-NEXT: cvt.u32.u16 %r8, %rs11;
@@ -1539,12 +1532,11 @@ define i16 @reduce_smin_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM80-LABEL: reduce_smin_i16_nonpow2(
; CHECK-SM80: {
; CHECK-SM80-NEXT: .reg .b16 %rs<14>;
-; CHECK-SM80-NEXT: .reg .b32 %r<3>;
+; CHECK-SM80-NEXT: .reg .b32 %r<2>;
; CHECK-SM80-EMPTY:
; CHECK-SM80-NEXT: // %bb.0:
-; CHECK-SM80-NEXT: ld.param.b32 %r1, [reduce_smin_i16_nonpow2_param_0+8];
-; CHECK-SM80-NEXT: mov.b32 {%rs5, %rs6}, %r1;
; CHECK-SM80-NEXT: ld.param.b16 %rs7, [reduce_smin_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT: ld.param.v2.b16 {%rs5, %rs6}, [reduce_smin_i16_nonpow2_param_0+8];
; CHECK-SM80-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_smin_i16_nonpow2_param_0];
; CHECK-SM80-NEXT: min.s16 %rs8, %rs3, %rs7;
; CHECK-SM80-NEXT: min.s16 %rs9, %rs1, %rs5;
@@ -1552,8 +1544,8 @@ define i16 @reduce_smin_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM80-NEXT: min.s16 %rs11, %rs2, %rs6;
; CHECK-SM80-NEXT: min.s16 %rs12, %rs4, %rs11;
; CHECK-SM80-NEXT: min.s16 %rs13, %rs10, %rs12;
-; CHECK-SM80-NEXT: cvt.u32.u16 %r2, %rs13;
-; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT: cvt.u32.u16 %r1, %rs13;
+; CHECK-SM80-NEXT: st.param.b32 [func_retval0], %r1;
; CHECK-SM80-NEXT: ret;
;
; CHECK-SM100-LABEL: reduce_smin_i16_nonpow2(
@@ -1564,15 +1556,15 @@ define i16 @reduce_smin_i16_nonpow2(<7 x i16> %in) {
; CHECK-SM100-NEXT: // %bb.0:
; CHECK-SM100-NEXT: ld.param.b32 %r1, [reduce_smin_i16_nonpow2_param_0+8];
; CHECK-SM100-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM100-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_smin_i16_nonpow2_param_0];
-; CHECK-SM100-NEXT: mov.b32 %r2, {%rs1, %rs2};
-; CHECK-SM100-NEXT: mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT: ld.param.v2.b32 {%r2, %r3}, [reduce_smin_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-SM100-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; CHECK-SM100-NEXT: ld.param.b16 %rs7, [reduce_smin_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT: min.s16x2 %r4, %r2, %r1;
; CHECK-SM100-NEXT: mov.b16 %rs8, 32767;
-; CHECK-SM100-NEXT: mov.b32 %r4, {%rs7, %rs8};
-; CHECK-SM100-NEXT: min.s16x2 %r5, %r3, %r4;
-; CHECK-SM100-NEXT: min.s16x2 %r6, %r2, %r1;
-; CHECK-SM100-NEXT: min.s16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT: mov.b32 %r5, {%rs7, %rs8};
+; CHECK-SM100-NEXT: min.s16x2 %r6, %r3, %r5;
+; CHECK-SM100-NEXT: min.s16x2 %r7, %r4, %r6;
; CHECK-SM100-NEXT: mov.b32 {%rs9, %rs10}, %r7;
; CHECK-SM100-NEXT: min.s16 %rs11, %rs9, %rs10;
; CHECK-SM100-NEXT: cvt.u32.u16 %r8, %rs11;
@@ -1675,15 +1667,15 @@ define i16 @reduce_and_i16_nonpow2(<7 x i16> %in) {
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [reduce_and_i16_nonpow2_param_0+8];
; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_and_i16_nonpow2_param_0];
-; CHECK-NEXT: mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT: mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT: ld.param.v2.b32 {%r2, %r3}, [reduce_and_i16_nonpow2_param_0];
+; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; CHECK-NEXT: ld.param.b16 %rs7, [reduce_and_i16_nonpow2_param_0+12];
+; CHECK-NEXT: and.b32 %r4, %r2, %r1;
; CHECK-NEXT: mov.b16 %rs8, -1;
-; CHECK-NEXT: mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT: and.b32 %r5, %r3, %r4;
-; CHECK-NEXT: and.b32 %r6, %r2, %r1;
-; CHECK-NEXT: and.b32 %r7, %r6, %r5;
+; CHECK-NEXT: mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT: and.b32 %r6, %r3, %r5;
+; CHECK-NEXT: and.b32 %r7, %r4, %r6;
; CHECK-NEXT: mov.b32 {%rs9, %rs10}, %r7;
; CHECK-NEXT: and.b16 %rs11, %rs9, %rs10;
; CHECK-NEXT: cvt.u32.u16 %r8, %rs11;
@@ -1786,15 +1778,15 @@ define i16 @reduce_or_i16_nonpow2(<7 x i16> %in) {
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [reduce_or_i16_nonpow2_param_0+8];
; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_or_i16_nonpow2_param_0];
-; CHECK-NEXT: mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT: mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT: ld.param.v2.b32 {%r2, %r3}, [reduce_or_i16_nonpow2_param_0];
+; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; CHECK-NEXT: ld.param.b16 %rs7, [reduce_or_i16_nonpow2_param_0+12];
+; CHECK-NEXT: or.b32 %r4, %r2, %r1;
; CHECK-NEXT: mov.b16 %rs8, 0;
-; CHECK-NEXT: mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT: or.b32 %r5, %r3, %r4;
-; CHECK-NEXT: or.b32 %r6, %r2, %r1;
-; CHECK-NEXT: or.b32 %r7, %r6, %r5;
+; CHECK-NEXT: mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT: or.b32 %r6, %r3, %r5;
+; CHECK-NEXT: or.b32 %r7, %r4, %r6;
; CHECK-NEXT: mov.b32 {%rs9, %rs10}, %r7;
; CHECK-NEXT: or.b16 %rs11, %rs9, %rs10;
; CHECK-NEXT: cvt.u32.u16 %r8, %rs11;
@@ -1897,15 +1889,15 @@ define i16 @reduce_xor_i16_nonpow2(<7 x i16> %in) {
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [reduce_xor_i16_nonpow2_param_0+8];
; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_xor_i16_nonpow2_param_0];
-; CHECK-NEXT: mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT: mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT: ld.param.v2.b32 {%r2, %r3}, [reduce_xor_i16_nonpow2_param_0];
+; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; CHECK-NEXT: ld.param.b16 %rs7, [reduce_xor_i16_nonpow2_param_0+12];
+; CHECK-NEXT: xor.b32 %r4, %r2, %r1;
; CHECK-NEXT: mov.b16 %rs8, 0;
-; CHECK-NEXT: mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT: xor.b32 %r5, %r3, %r4;
-; CHECK-NEXT: xor.b32 %r6, %r2, %r1;
-; CHECK-NEXT: xor.b32 %r7, %r6, %r5;
+; CHECK-NEXT: mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT: xor.b32 %r6, %r3, %r5;
+; CHECK-NEXT: xor.b32 %r7, %r4, %r6;
; CHECK-NEXT: mov.b32 {%rs9, %rs10}, %r7;
; CHECK-NEXT: xor.b16 %rs11, %rs9, %rs10;
; CHECK-NEXT: cvt.u32.u16 %r8, %rs11;
diff --git a/llvm/test/CodeGen/NVPTX/sext-setcc.ll b/llvm/test/CodeGen/NVPTX/sext-setcc.ll
index 802954bda6a9f..0af8190f20d18 100644
--- a/llvm/test/CodeGen/NVPTX/sext-setcc.ll
+++ b/llvm/test/CodeGen/NVPTX/sext-setcc.ll
@@ -7,19 +7,16 @@ define <2 x i16> @sext_setcc_v2i1_to_v2i16(ptr %p) {
; CHECK: {
; CHECK-NEXT: .reg .pred %p<3>;
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<3>;
; CHECK-NEXT: .reg .b64 %rd<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0: // %entry
; CHECK-NEXT: ld.param.b64 %rd1, [sext_setcc_v2i1_to_v2i16_param_0];
-; CHECK-NEXT: ld.b32 %r1, [%rd1];
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT: ld.v2.b16 {%rs1, %rs2}, [%rd1];
; CHECK-NEXT: setp.eq.s16 %p1, %rs1, 0;
; CHECK-NEXT: setp.eq.s16 %p2, %rs2, 0;
; CHECK-NEXT: selp.b16 %rs3, -1, 0, %p2;
; CHECK-NEXT: selp.b16 %rs4, -1, 0, %p1;
-; CHECK-NEXT: mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; CHECK-NEXT: ret;
entry:
%v = load <2 x i16>, ptr %p, align 4
diff --git a/llvm/test/CodeGen/NVPTX/shift-opt.ll b/llvm/test/CodeGen/NVPTX/shift-opt.ll
index 65bcbb8e67156..b165b4cb4b262 100644
--- a/llvm/test/CodeGen/NVPTX/shift-opt.ll
+++ b/llvm/test/CodeGen/NVPTX/shift-opt.ll
@@ -71,18 +71,17 @@ define <2 x i16> @test_vec(<2 x i16> %x, <2 x i8> %y) {
; CHECK-LABEL: test_vec(
; CHECK: {
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<6>;
+; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [test_vec_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [test_vec_param_1];
-; CHECK-NEXT: and.b32 %r3, %r2, 16711935;
-; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_vec_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [test_vec_param_1];
+; CHECK-NEXT: and.b32 %r2, %r1, 16711935;
; CHECK-NEXT: shr.u16 %rs3, %rs2, 5;
; CHECK-NEXT: shr.u16 %rs4, %rs1, 5;
-; CHECK-NEXT: mov.b32 %r4, {%rs4, %rs3};
-; CHECK-NEXT: or.b32 %r5, %r4, %r3;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r5;
+; CHECK-NEXT: mov.b32 %r3, {%rs4, %rs3};
+; CHECK-NEXT: or.b32 %r4, %r3, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
; CHECK-NEXT: ret;
%ext = zext <2 x i8> %y to <2 x i16>
%shl = shl <2 x i16> %ext, splat(i16 5)
diff --git a/llvm/test/CodeGen/NVPTX/unfold-masked-merge-vector-variablemask.ll b/llvm/test/CodeGen/NVPTX/unfold-masked-merge-vector-variablemask.ll
index bac8bbbf0b4de..c9a1072019dc3 100644
--- a/llvm/test/CodeGen/NVPTX/unfold-masked-merge-vector-variablemask.ll
+++ b/llvm/test/CodeGen/NVPTX/unfold-masked-merge-vector-variablemask.ll
@@ -61,13 +61,13 @@ define <4 x i8> @out_v4i8(<4 x i8> %x, <4 x i8> %y, <4 x i8> %mask) nounwind {
; CHECK-NEXT: .reg .b32 %r<8>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [out_v4i8_param_1];
-; CHECK-NEXT: ld.param.b32 %r2, [out_v4i8_param_0];
-; CHECK-NEXT: ld.param.b32 %r3, [out_v4i8_param_2];
-; CHECK-NEXT: and.b32 %r4, %r2, %r3;
-; CHECK-NEXT: xor.b32 %r5, %r3, -1;
-; CHECK-NEXT: and.b32 %r6, %r1, %r5;
-; CHECK-NEXT: or.b32 %r7, %r4, %r6;
+; CHECK-NEXT: ld.param.b32 %r1, [out_v4i8_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [out_v4i8_param_2];
+; CHECK-NEXT: and.b32 %r3, %r1, %r2;
+; CHECK-NEXT: ld.param.b32 %r4, [out_v4i8_param_1];
+; CHECK-NEXT: xor.b32 %r5, %r2, -1;
+; CHECK-NEXT: and.b32 %r6, %r4, %r5;
+; CHECK-NEXT: or.b32 %r7, %r3, %r6;
; CHECK-NEXT: st.param.b32 [func_retval0], %r7;
; CHECK-NEXT: ret;
%mx = and <4 x i8> %x, %mask
@@ -83,13 +83,13 @@ define <4 x i8> @out_v4i8_undef(<4 x i8> %x, <4 x i8> %y, <4 x i8> %mask) nounwi
; CHECK-NEXT: .reg .b32 %r<8>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [out_v4i8_undef_param_1];
-; CHECK-NEXT: ld.param.b32 %r2, [out_v4i8_undef_param_0];
-; CHECK-NEXT: ld.param.b32 %r3, [out_v4i8_undef_param_2];
-; CHECK-NEXT: and.b32 %r4, %r2, %r3;
-; CHECK-NEXT: xor.b32 %r5, %r3, -16711681;
-; CHECK-NEXT: and.b32 %r6, %r1, %r5;
-; CHECK-NEXT: or.b32 %r7, %r4, %r6;
+; CHECK-NEXT: ld.param.b32 %r1, [out_v4i8_undef_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [out_v4i8_undef_param_2];
+; CHECK-NEXT: and.b32 %r3, %r1, %r2;
+; CHECK-NEXT: ld.param.b32 %r4, [out_v4i8_undef_param_1];
+; CHECK-NEXT: xor.b32 %r5, %r2, -16711681;
+; CHECK-NEXT: and.b32 %r6, %r4, %r5;
+; CHECK-NEXT: or.b32 %r7, %r3, %r6;
; CHECK-NEXT: st.param.b32 [func_retval0], %r7;
; CHECK-NEXT: ret;
%mx = and <4 x i8> %x, %mask
@@ -105,13 +105,13 @@ define <2 x i16> @out_v2i16(<2 x i16> %x, <2 x i16> %y, <2 x i16> %mask) nounwin
; CHECK-NEXT: .reg .b32 %r<8>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [out_v2i16_param_1];
-; CHECK-NEXT: ld.param.b32 %r2, [out_v2i16_param_0];
-; CHECK-NEXT: ld.param.b32 %r3, [out_v2i16_param_2];
-; CHECK-NEXT: and.b32 %r4, %r2, %r3;
-; CHECK-NEXT: xor.b32 %r5, %r3, -1;
-; CHECK-NEXT: and.b32 %r6, %r1, %r5;
-; CHECK-NEXT: or.b32 %r7, %r4, %r6;
+; CHECK-NEXT: ld.param.b32 %r1, [out_v2i16_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [out_v2i16_param_2];
+; CHECK-NEXT: and.b32 %r3, %r1, %r2;
+; CHECK-NEXT: ld.param.b32 %r4, [out_v2i16_param_1];
+; CHECK-NEXT: xor.b32 %r5, %r2, -1;
+; CHECK-NEXT: and.b32 %r6, %r4, %r5;
+; CHECK-NEXT: or.b32 %r7, %r3, %r6;
; CHECK-NEXT: st.param.b32 [func_retval0], %r7;
; CHECK-NEXT: ret;
%mx = and <2 x i16> %x, %mask
@@ -551,14 +551,14 @@ define <8 x i8> @in_v8i8(<8 x i8> %x, <8 x i8> %y, <8 x i8> %mask) nounwind {
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.v2.b32 {%r1, %r2}, [in_v8i8_param_0];
; CHECK-NEXT: ld.param.v2.b32 {%r3, %r4}, [in_v8i8_param_1];
-; CHECK-NEXT: ld.param.v2.b32 {%r5, %r6}, [in_v8i8_param_2];
-; CHECK-NEXT: xor.b32 %r7, %r2, %r4;
-; CHECK-NEXT: and.b32 %r8, %r7, %r6;
-; CHECK-NEXT: xor.b32 %r9, %r8, %r4;
-; CHECK-NEXT: xor.b32 %r10, %r1, %r3;
-; CHECK-NEXT: and.b32 %r11, %r10, %r5;
-; CHECK-NEXT: xor.b32 %r12, %r11, %r3;
-; CHECK-NEXT: st.param.v2.b32 [func_retval0], {%r12, %r9};
+; CHECK-NEXT: xor.b32 %r5, %r2, %r4;
+; CHECK-NEXT: xor.b32 %r6, %r1, %r3;
+; CHECK-NEXT: ld.param.v2.b32 {%r7, %r8}, [in_v8i8_param_2];
+; CHECK-NEXT: and.b32 %r9, %r6, %r7;
+; CHECK-NEXT: and.b32 %r10, %r5, %r8;
+; CHECK-NEXT: xor.b32 %r11, %r10, %r4;
+; CHECK-NEXT: xor.b32 %r12, %r9, %r3;
+; CHECK-NEXT: st.param.v2.b32 [func_retval0], {%r12, %r11};
; CHECK-NEXT: ret;
%n0 = xor <8 x i8> %x, %y
%n1 = and <8 x i8> %n0, %mask
@@ -574,14 +574,14 @@ define <4 x i16> @in_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i16> %mask) nounwind
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.v2.b32 {%r1, %r2}, [in_v4i16_param_0];
; CHECK-NEXT: ld.param.v2.b32 {%r3, %r4}, [in_v4i16_param_1];
-; CHECK-NEXT: ld.param.v2.b32 {%r5, %r6}, [in_v4i16_param_2];
-; CHECK-NEXT: xor.b32 %r7, %r2, %r4;
-; CHECK-NEXT: and.b32 %r8, %r7, %r6;
-; CHECK-NEXT: xor.b32 %r9, %r8, %r4;
-; CHECK-NEXT: xor.b32 %r10, %r1, %r3;
-; CHECK-NEXT: and.b32 %r11, %r10, %r5;
-; CHECK-NEXT: xor.b32 %r12, %r11, %r3;
-; CHECK-NEXT: st.param.v2.b32 [func_retval0], {%r12, %r9};
+; CHECK-NEXT: xor.b32 %r5, %r2, %r4;
+; CHECK-NEXT: xor.b32 %r6, %r1, %r3;
+; CHECK-NEXT: ld.param.v2.b32 {%r7, %r8}, [in_v4i16_param_2];
+; CHECK-NEXT: and.b32 %r9, %r6, %r7;
+; CHECK-NEXT: and.b32 %r10, %r5, %r8;
+; CHECK-NEXT: xor.b32 %r11, %r10, %r4;
+; CHECK-NEXT: xor.b32 %r12, %r9, %r3;
+; CHECK-NEXT: st.param.v2.b32 [func_retval0], {%r12, %r11};
; CHECK-NEXT: ret;
%n0 = xor <4 x i16> %x, %y
%n1 = and <4 x i16> %n0, %mask
diff --git a/llvm/test/CodeGen/NVPTX/vector-loads.ll b/llvm/test/CodeGen/NVPTX/vector-loads.ll
index 88ff59407a143..1ae6f6bcd748f 100644
--- a/llvm/test/CodeGen/NVPTX/vector-loads.ll
+++ b/llvm/test/CodeGen/NVPTX/vector-loads.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 | FileCheck %s
; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
@@ -8,28 +9,67 @@
;
; which will load two floats at once into scalar registers.
-; CHECK-LABEL: foo
define void @foo(ptr %a) {
-; CHECK: ld.v2.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}}
+; CHECK-LABEL: foo(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [foo_param_0];
+; CHECK-NEXT: ld.v2.b32 {%r1, %r2}, [%rd1];
+; CHECK-NEXT: mul.rn.f32 %r3, %r2, %r2;
+; CHECK-NEXT: mul.rn.f32 %r4, %r1, %r1;
+; CHECK-NEXT: st.v2.b32 [%rd1], {%r4, %r3};
+; CHECK-NEXT: ret;
%t1 = load <2 x float>, ptr %a
%t2 = fmul <2 x float> %t1, %t1
store <2 x float> %t2, ptr %a
ret void
}
-; CHECK-LABEL: foo2
define void @foo2(ptr %a) {
-; CHECK: ld.v4.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}}
+; CHECK-LABEL: foo2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<9>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [foo2_param_0];
+; CHECK-NEXT: ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; CHECK-NEXT: mul.rn.f32 %r5, %r4, %r4;
+; CHECK-NEXT: mul.rn.f32 %r6, %r3, %r3;
+; CHECK-NEXT: mul.rn.f32 %r7, %r2, %r2;
+; CHECK-NEXT: mul.rn.f32 %r8, %r1, %r1;
+; CHECK-NEXT: st.v4.b32 [%rd1], {%r8, %r7, %r6, %r5};
+; CHECK-NEXT: ret;
%t1 = load <4 x float>, ptr %a
%t2 = fmul <4 x float> %t1, %t1
store <4 x float> %t2, ptr %a
ret void
}
-; CHECK-LABEL: foo3
define void @foo3(ptr %a) {
-; CHECK: ld.v4.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}}
-; CHECK-NEXT: ld.v4.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}}
+; CHECK-LABEL: foo3(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<17>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [foo3_param_0];
+; CHECK-NEXT: ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1+16];
+; CHECK-NEXT: ld.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1];
+; CHECK-NEXT: mul.rn.f32 %r9, %r8, %r8;
+; CHECK-NEXT: mul.rn.f32 %r10, %r7, %r7;
+; CHECK-NEXT: mul.rn.f32 %r11, %r6, %r6;
+; CHECK-NEXT: mul.rn.f32 %r12, %r5, %r5;
+; CHECK-NEXT: mul.rn.f32 %r13, %r4, %r4;
+; CHECK-NEXT: mul.rn.f32 %r14, %r3, %r3;
+; CHECK-NEXT: mul.rn.f32 %r15, %r2, %r2;
+; CHECK-NEXT: mul.rn.f32 %r16, %r1, %r1;
+; CHECK-NEXT: st.v4.b32 [%rd1+16], {%r16, %r15, %r14, %r13};
+; CHECK-NEXT: st.v4.b32 [%rd1], {%r12, %r11, %r10, %r9};
+; CHECK-NEXT: ret;
%t1 = load <8 x float>, ptr %a
%t2 = fmul <8 x float> %t1, %t1
store <8 x float> %t2, ptr %a
@@ -38,28 +78,67 @@ define void @foo3(ptr %a) {
-; CHECK-LABEL: foo4
define void @foo4(ptr %a) {
-; CHECK: ld.v2.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}}
+; CHECK-LABEL: foo4(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [foo4_param_0];
+; CHECK-NEXT: ld.v2.b32 {%r1, %r2}, [%rd1];
+; CHECK-NEXT: mul.lo.s32 %r3, %r2, %r2;
+; CHECK-NEXT: mul.lo.s32 %r4, %r1, %r1;
+; CHECK-NEXT: st.v2.b32 [%rd1], {%r4, %r3};
+; CHECK-NEXT: ret;
%t1 = load <2 x i32>, ptr %a
%t2 = mul <2 x i32> %t1, %t1
store <2 x i32> %t2, ptr %a
ret void
}
-; CHECK-LABEL: foo5
define void @foo5(ptr %a) {
-; CHECK: ld.v4.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}}
+; CHECK-LABEL: foo5(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<9>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [foo5_param_0];
+; CHECK-NEXT: ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; CHECK-NEXT: mul.lo.s32 %r5, %r4, %r4;
+; CHECK-NEXT: mul.lo.s32 %r6, %r3, %r3;
+; CHECK-NEXT: mul.lo.s32 %r7, %r2, %r2;
+; CHECK-NEXT: mul.lo.s32 %r8, %r1, %r1;
+; CHECK-NEXT: st.v4.b32 [%rd1], {%r8, %r7, %r6, %r5};
+; CHECK-NEXT: ret;
%t1 = load <4 x i32>, ptr %a
%t2 = mul <4 x i32> %t1, %t1
store <4 x i32> %t2, ptr %a
ret void
}
-; CHECK-LABEL: foo6
define void @foo6(ptr %a) {
-; CHECK: ld.v4.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}}
-; CHECK-NEXT: ld.v4.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}}
+; CHECK-LABEL: foo6(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<17>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [foo6_param_0];
+; CHECK-NEXT: ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1+16];
+; CHECK-NEXT: ld.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1];
+; CHECK-NEXT: mul.lo.s32 %r9, %r8, %r8;
+; CHECK-NEXT: mul.lo.s32 %r10, %r7, %r7;
+; CHECK-NEXT: mul.lo.s32 %r11, %r6, %r6;
+; CHECK-NEXT: mul.lo.s32 %r12, %r5, %r5;
+; CHECK-NEXT: mul.lo.s32 %r13, %r4, %r4;
+; CHECK-NEXT: mul.lo.s32 %r14, %r3, %r3;
+; CHECK-NEXT: mul.lo.s32 %r15, %r2, %r2;
+; CHECK-NEXT: mul.lo.s32 %r16, %r1, %r1;
+; CHECK-NEXT: st.v4.b32 [%rd1+16], {%r16, %r15, %r14, %r13};
+; CHECK-NEXT: st.v4.b32 [%rd1], {%r12, %r11, %r10, %r9};
+; CHECK-NEXT: ret;
%t1 = load <8 x i32>, ptr %a
%t2 = mul <8 x i32> %t1, %t1
store <8 x i32> %t2, ptr %a
@@ -70,8 +149,30 @@ define void @foo6(ptr %a) {
; computation was still too complex when LSV was called.
declare i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() #0
declare i32 @llvm.nvvm.read.ptx.sreg.tid.x() #0
-; CHECK-LABEL: foo_complex
define void @foo_complex(ptr nocapture readonly align 16 dereferenceable(134217728) %alloc0) {
+; CHECK-LABEL: foo_complex(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-NEXT: .reg .b32 %r<8>;
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [foo_complex_param_0];
+; CHECK-NEXT: mov.u32 %r1, %tid.x;
+; CHECK-NEXT: mov.u32 %r2, %ctaid.x;
+; CHECK-NEXT: shr.u32 %r3, %r2, 8;
+; CHECK-NEXT: shl.b32 %r4, %r2, 9;
+; CHECK-NEXT: and.b32 %r5, %r4, 130560;
+; CHECK-NEXT: shl.b32 %r6, %r1, 1;
+; CHECK-NEXT: or.b32 %r7, %r5, %r6;
+; CHECK-NEXT: cvt.u64.u32 %rd2, %r7;
+; CHECK-NEXT: mul.wide.u32 %rd3, %r3, 131072;
+; CHECK-NEXT: add.s64 %rd4, %rd1, %rd3;
+; CHECK-NEXT: add.s64 %rd5, %rd4, %rd2;
+; CHECK-NEXT: ld.v2.b8 {%rs1, %rs2}, [%rd5+128];
+; CHECK-NEXT: max.u16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.b8 [%rd5+129], %rs3;
+; CHECK-NEXT: ret;
%t0 = tail call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !1
%t1 = tail call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x()
%t2 = lshr i32 %t1, 8
@@ -86,112 +187,138 @@ define void @foo_complex(ptr nocapture readonly align 16 dereferenceable(1342177
%t11 = zext i32 %t10 to i64
%t20 = zext i32 %t2 to i64
%t27 = getelementptr inbounds [1024 x [131072 x i8]], ptr %alloc0, i64 0, i64 %t20, i64 %t9
-; CHECK: ld.v2.b8
%t28 = load i8, ptr %t27, align 2
%t31 = getelementptr inbounds [1024 x [131072 x i8]], ptr %alloc0, i64 0, i64 %t20, i64 %t11
%t32 = load i8, ptr %t31, align 1
%t33 = icmp ult i8 %t28, %t32
%t34 = select i1 %t33, i8 %t32, i8 %t28
store i8 %t34, ptr %t31
-; CHECK: ret
ret void
}
-; CHECK-LABEL: extv8f16_global_a16(
define void @extv8f16_global_a16(ptr addrspace(1) noalias readonly align 16 %dst, ptr addrspace(1) noalias readonly align 16 %src) #0 {
-; CHECK: ld.global.v4.b32 {%r
+; CHECK-LABEL: extv8f16_global_a16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<9>;
+; CHECK-NEXT: .reg .b32 %r<13>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [extv8f16_global_a16_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [extv8f16_global_a16_param_1];
+; CHECK-NEXT: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; CHECK-NEXT: cvt.f32.f16 %r5, %rs8;
+; CHECK-NEXT: cvt.f32.f16 %r6, %rs7;
+; CHECK-NEXT: cvt.f32.f16 %r7, %rs6;
+; CHECK-NEXT: cvt.f32.f16 %r8, %rs5;
+; CHECK-NEXT: cvt.f32.f16 %r9, %rs4;
+; CHECK-NEXT: cvt.f32.f16 %r10, %rs3;
+; CHECK-NEXT: cvt.f32.f16 %r11, %rs2;
+; CHECK-NEXT: cvt.f32.f16 %r12, %rs1;
+; CHECK-NEXT: st.global.v4.b32 [%rd1+16], {%r12, %r11, %r10, %r9};
+; CHECK-NEXT: st.global.v4.b32 [%rd1], {%r8, %r7, %r6, %r5};
+; CHECK-NEXT: ret;
%v = load <8 x half>, ptr addrspace(1) %src, align 16
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
%ext = fpext <8 x half> %v to <8 x float>
-; CHECK: st.global.v4.b32
-; CHECK: st.global.v4.b32
store <8 x float> %ext, ptr addrspace(1) %dst, align 16
ret void
}
-; CHECK-LABEL: extv8f16_global_a4(
define void @extv8f16_global_a4(ptr addrspace(1) noalias readonly align 16 %dst, ptr addrspace(1) noalias readonly align 16 %src) #0 {
-; CHECK: ld.global.b32 %r
-; CHECK: ld.global.b32 %r
-; CHECK: ld.global.b32 %r
-; CHECK: ld.global.b32 %r
+; CHECK-LABEL: extv8f16_global_a4(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<9>;
+; CHECK-NEXT: .reg .b32 %r<9>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [extv8f16_global_a4_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [extv8f16_global_a4_param_1];
+; CHECK-NEXT: ld.global.v2.b16 {%rs1, %rs2}, [%rd2+8];
+; CHECK-NEXT: ld.global.v2.b16 {%rs3, %rs4}, [%rd2+12];
+; CHECK-NEXT: ld.global.v2.b16 {%rs5, %rs6}, [%rd2];
+; CHECK-NEXT: ld.global.v2.b16 {%rs7, %rs8}, [%rd2+4];
+; CHECK-NEXT: cvt.f32.f16 %r1, %rs8;
+; CHECK-NEXT: cvt.f32.f16 %r2, %rs7;
+; CHECK-NEXT: cvt.f32.f16 %r3, %rs6;
+; CHECK-NEXT: cvt.f32.f16 %r4, %rs5;
+; CHECK-NEXT: cvt.f32.f16 %r5, %rs4;
+; CHECK-NEXT: cvt.f32.f16 %r6, %rs3;
+; CHECK-NEXT: cvt.f32.f16 %r7, %rs2;
+; CHECK-NEXT: cvt.f32.f16 %r8, %rs1;
+; CHECK-NEXT: st.global.v4.b32 [%rd1+16], {%r8, %r7, %r6, %r5};
+; CHECK-NEXT: st.global.v4.b32 [%rd1], {%r4, %r3, %r2, %r1};
+; CHECK-NEXT: ret;
%v = load <8 x half>, ptr addrspace(1) %src, align 4
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
%ext = fpext <8 x half> %v to <8 x float>
-; CHECK: st.global.v4.b32
-; CHECK: st.global.v4.b32
store <8 x float> %ext, ptr addrspace(1) %dst, align 16
ret void
}
-; CHECK-LABEL: extv8f16_generic_a16(
define void @extv8f16_generic_a16(ptr noalias readonly align 16 %dst, ptr noalias readonly align 16 %src) #0 {
-; CHECK: ld.v4.b32 {%r
+; CHECK-LABEL: extv8f16_generic_a16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<9>;
+; CHECK-NEXT: .reg .b32 %r<13>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [extv8f16_generic_a16_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [extv8f16_generic_a16_param_1];
+; CHECK-NEXT: ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; CHECK-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; CHECK-NEXT: cvt.f32.f16 %r5, %rs8;
+; CHECK-NEXT: cvt.f32.f16 %r6, %rs7;
+; CHECK-NEXT: cvt.f32.f16 %r7, %rs6;
+; CHECK-NEXT: cvt.f32.f16 %r8, %rs5;
+; CHECK-NEXT: cvt.f32.f16 %r9, %rs4;
+; CHECK-NEXT: cvt.f32.f16 %r10, %rs3;
+; CHECK-NEXT: cvt.f32.f16 %r11, %rs2;
+; CHECK-NEXT: cvt.f32.f16 %r12, %rs1;
+; CHECK-NEXT: st.v4.b32 [%rd1+16], {%r12, %r11, %r10, %r9};
+; CHECK-NEXT: st.v4.b32 [%rd1], {%r8, %r7, %r6, %r5};
+; CHECK-NEXT: ret;
%v = load <8 x half>, ptr %src, align 16
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
%ext = fpext <8 x half> %v to <8 x float>
-; CHECK: st.v4.b32
-; CHECK: st.v4.b32
store <8 x float> %ext, ptr %dst, align 16
ret void
}
-; CHECK-LABEL: extv8f16_generic_a4(
define void @extv8f16_generic_a4(ptr noalias readonly align 16 %dst, ptr noalias readonly align 16 %src) #0 {
-; CHECK: ld.b32 %r
-; CHECK: ld.b32 %r
-; CHECK: ld.b32 %r
-; CHECK: ld.b32 %r
+; CHECK-LABEL: extv8f16_generic_a4(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<9>;
+; CHECK-NEXT: .reg .b32 %r<9>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [extv8f16_generic_a4_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [extv8f16_generic_a4_param_1];
+; CHECK-NEXT: ld.v2.b16 {%rs1, %rs2}, [%rd2+8];
+; CHECK-NEXT: ld.v2.b16 {%rs3, %rs4}, [%rd2+12];
+; CHECK-NEXT: ld.v2.b16 {%rs5, %rs6}, [%rd2];
+; CHECK-NEXT: ld.v2.b16 {%rs7, %rs8}, [%rd2+4];
+; CHECK-NEXT: cvt.f32.f16 %r1, %rs8;
+; CHECK-NEXT: cvt.f32.f16 %r2, %rs7;
+; CHECK-NEXT: cvt.f32.f16 %r3, %rs6;
+; CHECK-NEXT: cvt.f32.f16 %r4, %rs5;
+; CHECK-NEXT: cvt.f32.f16 %r5, %rs4;
+; CHECK-NEXT: cvt.f32.f16 %r6, %rs3;
+; CHECK-NEXT: cvt.f32.f16 %r7, %rs2;
+; CHECK-NEXT: cvt.f32.f16 %r8, %rs1;
+; CHECK-NEXT: st.v4.b32 [%rd1+16], {%r8, %r7, %r6, %r5};
+; CHECK-NEXT: st.v4.b32 [%rd1], {%r4, %r3, %r2, %r1};
+; CHECK-NEXT: ret;
%v = load <8 x half>, ptr %src, align 4
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
%ext = fpext <8 x half> %v to <8 x float>
-; CHECK: st.v4.b32
-; CHECK: st.v4.b32
store <8 x float> %ext, ptr %dst, align 16
ret void
}
@@ -199,10 +326,18 @@ define void @extv8f16_generic_a4(ptr noalias readonly align 16 %dst, ptr noalias
!1 = !{i32 0, i32 64}
-; CHECK-LABEL: bf16_v4_align_load_store
define dso_local void @bf16_v4_align_load_store(ptr noundef %0, ptr noundef %1) #0 {
- ; CHECK: ld.v4.b16
- ; CHECK: st.v4.b16
+; CHECK-LABEL: bf16_v4_align_load_store(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [bf16_v4_align_load_store_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [bf16_v4_align_load_store_param_1];
+; CHECK-NEXT: ld.v2.b32 {%r1, %r2}, [%rd2];
+; CHECK-NEXT: st.v2.b32 [%rd1], {%r1, %r2};
+; CHECK-NEXT: ret;
%3 = load <4 x bfloat>, ptr %1, align 8
store <4 x bfloat> %3, ptr %0, align 8
ret void
More information about the llvm-commits
mailing list