[llvm] [AArch64][GlobalISel] Improve lowering of vector fp16 fptrunc and fpext (PR #163398)
Ryan Cowan via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 27 09:54:36 PDT 2025
https://github.com/HolyMolyCowMan updated https://github.com/llvm/llvm-project/pull/163398
>From ce5625b35d6e81ebbafc8b06f2c71e03e5abb9d0 Mon Sep 17 00:00:00 2001
From: Ryan Cowan <ryan.cowan at arm.com>
Date: Mon, 13 Oct 2025 12:14:00 +0000
Subject: [PATCH 01/12] [AArch64][GlobalISel] Improve lowering of vector fp16
fptrunc and fpext
---
llvm/lib/Target/AArch64/AArch64Combine.td | 9 +-
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 62 +++-
.../AArch64/GISel/AArch64LegalizerInfo.h | 2 +
.../GISel/AArch64PostLegalizerLowering.cpp | 194 ++++++++++++
.../GlobalISel/legalizer-info-validation.mir | 8 +-
llvm/test/CodeGen/AArch64/arm64-fp128.ll | 24 +-
llvm/test/CodeGen/AArch64/fmla.ll | 48 +--
.../CodeGen/AArch64/fp16-v4-instructions.ll | 73 +----
.../CodeGen/AArch64/fp16-v8-instructions.ll | 100 ++-----
llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll | 186 +++++-------
llvm/test/CodeGen/AArch64/fpext.ll | 49 ++-
llvm/test/CodeGen/AArch64/fptoi.ll | 278 ++++++------------
.../test/CodeGen/AArch64/fptosi-sat-vector.ll | 85 ++----
.../test/CodeGen/AArch64/fptoui-sat-vector.ll | 85 ++----
llvm/test/CodeGen/AArch64/fptrunc.ll | 101 +++----
15 files changed, 592 insertions(+), 712 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index b3ec65cab51fa..fe3ba82ceb3cd 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -332,6 +332,13 @@ def combine_mul_cmlt : GICombineRule<
(apply [{ applyCombineMulCMLT(*${root}, MRI, B, ${matchinfo}); }])
>;
+def lower_fptrunc_fptrunc: GICombineRule<
+ (defs root:$root),
+ (match (wip_match_opcode G_FPTRUNC):$root,
+ [{ return matchFpTruncFpTrunc(*${root}, MRI); }]),
+ (apply [{ applyFpTruncFpTrunc(*${root}, MRI, B); }])
+>;
+
// Post-legalization combines which should happen at all optimization levels.
// (E.g. ones that facilitate matching for the selector) For example, matching
// pseudos.
@@ -340,7 +347,7 @@ def AArch64PostLegalizerLowering
[shuffle_vector_lowering, vashr_vlshr_imm,
icmp_lowering, build_vector_lowering,
lower_vector_fcmp, form_truncstore, fconstant_to_constant,
- vector_sext_inreg_to_shift,
+ vector_sext_inreg_to_shift, lower_fptrunc_fptrunc,
unmerge_ext_to_unmerge, lower_mulv2s64,
vector_unmerge_lowering, insertelt_nonconst,
unmerge_duplanes]> {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 5f93847bc680e..9153694817676 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -21,6 +21,7 @@
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/IR/DerivedTypes.h"
@@ -817,14 +818,31 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
.legalFor(
{{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}})
.libcallFor({{s16, s128}, {s32, s128}, {s64, s128}})
- .clampNumElements(0, v4s16, v4s16)
- .clampNumElements(0, v2s32, v2s32)
+ .moreElementsToNextPow2(1)
+ .customIf([](const LegalityQuery &Q) {
+ LLT DstTy = Q.Types[0];
+ LLT SrcTy = Q.Types[1];
+ return SrcTy.isFixedVector() && DstTy.isFixedVector() &&
+ SrcTy.getScalarSizeInBits() == 64 &&
+ DstTy.getScalarSizeInBits() == 16;
+ })
+ // Clamp based on input
+ .clampNumElements(1, v4s32, v4s32)
+ .clampNumElements(1, v2s64, v2s64)
.scalarize(0);
getActionDefinitionsBuilder(G_FPEXT)
.legalFor(
{{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}})
.libcallFor({{s128, s64}, {s128, s32}, {s128, s16}})
+ .moreElementsToNextPow2(0)
+ .customIf([](const LegalityQuery &Q) {
+ LLT DstTy = Q.Types[0];
+ LLT SrcTy = Q.Types[1];
+ return SrcTy.isVector() && DstTy.isVector() &&
+ SrcTy.getScalarSizeInBits() == 16 &&
+ DstTy.getScalarSizeInBits() == 64;
+ })
.clampNumElements(0, v4s32, v4s32)
.clampNumElements(0, v2s64, v2s64)
.scalarize(0);
@@ -1464,6 +1482,12 @@ bool AArch64LegalizerInfo::legalizeCustom(
return legalizeICMP(MI, MRI, MIRBuilder);
case TargetOpcode::G_BITCAST:
return legalizeBitcast(MI, Helper);
+ case TargetOpcode::G_FPEXT:
+ // In order to vectorise f16 to f64 properly, we need to use f32 as an
+ // intermediary
+ return legalizeViaF32(MI, MIRBuilder, MRI, TargetOpcode::G_FPEXT);
+ case TargetOpcode::G_FPTRUNC:
+ return legalizeViaF32(MI, MIRBuilder, MRI, TargetOpcode::G_FPTRUNC);
}
llvm_unreachable("expected switch to return");
@@ -2390,3 +2414,37 @@ bool AArch64LegalizerInfo::legalizePrefetch(MachineInstr &MI,
MI.eraseFromParent();
return true;
}
+
+bool AArch64LegalizerInfo::legalizeViaF32(MachineInstr &MI,
+ MachineIRBuilder &MIRBuilder,
+ MachineRegisterInfo &MRI,
+ unsigned Opcode) const {
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(1).getReg();
+ LLT DstTy = MRI.getType(Dst);
+ LLT SrcTy = MRI.getType(Src);
+
+ LLT MidTy = LLT::fixed_vector(SrcTy.getNumElements(), LLT::scalar(32));
+
+ MachineInstrBuilder Mid;
+ MachineInstrBuilder Fin;
+ MIRBuilder.setInstrAndDebugLoc(MI);
+ switch (Opcode) {
+ default:
+ return false;
+ case TargetOpcode::G_FPEXT: {
+ Mid = MIRBuilder.buildFPExt(MidTy, Src);
+ Fin = MIRBuilder.buildFPExt(DstTy, Mid.getReg(0));
+ break;
+ }
+ case TargetOpcode::G_FPTRUNC: {
+ Mid = MIRBuilder.buildFPTrunc(MidTy, Src);
+ Fin = MIRBuilder.buildFPTrunc(DstTy, Mid.getReg(0));
+ break;
+ }
+ }
+
+ MRI.replaceRegWith(Dst, Fin.getReg(0));
+ MI.eraseFromParent();
+ return true;
+}
\ No newline at end of file
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
index bcb294326fa92..049808d66f983 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
@@ -67,6 +67,8 @@ class AArch64LegalizerInfo : public LegalizerInfo {
bool legalizeDynStackAlloc(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizePrefetch(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizeBitcast(MachineInstr &MI, LegalizerHelper &Helper) const;
+ bool legalizeViaF32(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
+ MachineRegisterInfo &MRI, unsigned Opcode) const;
const AArch64Subtarget *ST;
};
} // End llvm namespace.
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
index 23dcaea2ac1a4..e675fac0f13ac 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
@@ -901,6 +901,200 @@ unsigned getCmpOperandFoldingProfit(Register CmpOp, MachineRegisterInfo &MRI) {
return 0;
}
+// Helper function for matchFpTruncFpTrunc.
+// Checks that the given definition belongs to an FPTRUNC and that the source is
+// not an integer, as no rounding is necessary due to the range of values
+bool checkTruncSrc(MachineRegisterInfo &MRI, MachineInstr *MaybeFpTrunc) {
+ if (!MaybeFpTrunc || MaybeFpTrunc->getOpcode() != TargetOpcode::G_FPTRUNC)
+ return false;
+
+ // Check the source is 64 bits as we only want to match a very specific
+ // pattern
+ Register FpTruncSrc = MaybeFpTrunc->getOperand(1).getReg();
+ LLT SrcTy = MRI.getType(FpTruncSrc);
+ if (SrcTy.getScalarSizeInBits() != 64)
+ return false;
+
+ // Need to check the float didn't come from an int as no rounding is
+ // neccessary
+ MachineInstr *FpTruncSrcDef = getDefIgnoringCopies(FpTruncSrc, MRI);
+ if (FpTruncSrcDef->getOpcode() == TargetOpcode::G_SITOFP ||
+ FpTruncSrcDef->getOpcode() == TargetOpcode::G_UITOFP)
+ return false;
+
+ return true;
+}
+
+// To avoid double rounding issues we need to lower FPTRUNC(FPTRUNC) to an odd
+// rounding truncate and a normal truncate. When
+// truncating an FP that came from an integer this is not a problem as the range
+// of values is lower in the int
+bool matchFpTruncFpTrunc(MachineInstr &MI, MachineRegisterInfo &MRI) {
+ if (MI.getOpcode() != TargetOpcode::G_FPTRUNC)
+ return false;
+
+ // Check the destination is 16 bits as we only want to match a very specific
+ // pattern
+ Register Dst = MI.getOperand(0).getReg();
+ LLT DstTy = MRI.getType(Dst);
+ if (DstTy.getScalarSizeInBits() != 16)
+ return false;
+
+ Register Src = MI.getOperand(1).getReg();
+
+ MachineInstr *ParentDef = getDefIgnoringCopies(Src, MRI);
+ if (!ParentDef)
+ return false;
+
+ MachineInstr *FpTruncDef;
+ switch (ParentDef->getOpcode()) {
+ default:
+ return false;
+ case TargetOpcode::G_CONCAT_VECTORS: {
+ // Expecting exactly two FPTRUNCs
+ if (ParentDef->getNumOperands() != 3)
+ return false;
+
+ // All operands need to be FPTRUNC
+ for (unsigned OpIdx = 1, NumOperands = ParentDef->getNumOperands();
+ OpIdx != NumOperands; ++OpIdx) {
+ Register FpTruncDst = ParentDef->getOperand(OpIdx).getReg();
+
+ FpTruncDef = getDefIgnoringCopies(FpTruncDst, MRI);
+
+ if (!checkTruncSrc(MRI, FpTruncDef))
+ return false;
+ }
+
+ return true;
+ }
+ // This is to match cases in which vectors are widened to a larger size
+ case TargetOpcode::G_INSERT_VECTOR_ELT: {
+ Register VecExtractDst = ParentDef->getOperand(2).getReg();
+ MachineInstr *VecExtractDef = getDefIgnoringCopies(VecExtractDst, MRI);
+
+ Register FpTruncDst = VecExtractDef->getOperand(1).getReg();
+ FpTruncDef = getDefIgnoringCopies(FpTruncDst, MRI);
+
+ if (!checkTruncSrc(MRI, FpTruncDef))
+ return false;
+ break;
+ }
+ case TargetOpcode::G_FPTRUNC: {
+ Register FpTruncDst = ParentDef->getOperand(1).getReg();
+ FpTruncDef = getDefIgnoringCopies(FpTruncDst, MRI);
+
+ if (!checkTruncSrc(MRI, FpTruncDef))
+ return false;
+ break;
+ }
+ }
+
+ return true;
+}
+
+void applyFpTruncFpTrunc(MachineInstr &MI, MachineRegisterInfo &MRI,
+ MachineIRBuilder &B) {
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(1).getReg();
+
+ LLT V2F32 = LLT::fixed_vector(2, LLT::scalar(32));
+ LLT V4F32 = LLT::fixed_vector(4, LLT::scalar(32));
+ LLT V4F16 = LLT::fixed_vector(4, LLT::scalar(16));
+
+ B.setInstrAndDebugLoc(MI);
+
+ MachineInstr *ParentDef = getDefIgnoringCopies(Src, MRI);
+ if (!ParentDef)
+ return;
+
+ switch (ParentDef->getOpcode()) {
+ default:
+ return;
+ case TargetOpcode::G_INSERT_VECTOR_ELT: {
+ Register VecExtractDst = ParentDef->getOperand(2).getReg();
+ MachineInstr *VecExtractDef = getDefIgnoringCopies(VecExtractDst, MRI);
+
+ Register FpTruncDst = VecExtractDef->getOperand(1).getReg();
+ MachineInstr *FpTruncDef = getDefIgnoringCopies(FpTruncDst, MRI);
+
+ Register FpTruncSrc = FpTruncDef->getOperand(1).getReg();
+ MRI.setRegClass(FpTruncSrc, &AArch64::FPR128RegClass);
+
+ Register Fp32 = MRI.createGenericVirtualRegister(V2F32);
+ MRI.setRegClass(Fp32, &AArch64::FPR64RegClass);
+
+ B.buildInstr(AArch64::FCVTXNv2f32, {Fp32}, {FpTruncSrc});
+
+ // Only 4f32 -> 4f16 is legal so we need to mimic that situation
+ Register Fp32Padding = B.buildUndef(V2F32).getReg(0);
+ MRI.setRegClass(Fp32Padding, &AArch64::FPR64RegClass);
+
+ Register Fp32Full = MRI.createGenericVirtualRegister(V4F32);
+ MRI.setRegClass(Fp32Full, &AArch64::FPR128RegClass);
+ B.buildConcatVectors(Fp32Full, {Fp32, Fp32Padding});
+
+ Register Fp16 = MRI.createGenericVirtualRegister(V4F16);
+ MRI.setRegClass(Fp16, &AArch64::FPR64RegClass);
+ B.buildFPTrunc(Fp16, Fp32Full);
+
+ MRI.replaceRegWith(Dst, Fp16);
+ MI.eraseFromParent();
+ break;
+ }
+ case TargetOpcode::G_CONCAT_VECTORS: {
+ // Get the two FP Truncs that are being concatenated
+ Register FpTrunc1Dst = ParentDef->getOperand(1).getReg();
+ Register FpTrunc2Dst = ParentDef->getOperand(2).getReg();
+
+ MachineInstr *FpTrunc1Def = getDefIgnoringCopies(FpTrunc1Dst, MRI);
+ MachineInstr *FpTrunc2Def = getDefIgnoringCopies(FpTrunc2Dst, MRI);
+
+ // Make the registers 128bit to store the 2 doubles
+ Register LoFp64 = FpTrunc1Def->getOperand(1).getReg();
+ MRI.setRegClass(LoFp64, &AArch64::FPR128RegClass);
+ Register HiFp64 = FpTrunc2Def->getOperand(1).getReg();
+ MRI.setRegClass(HiFp64, &AArch64::FPR128RegClass);
+
+ B.setInstrAndDebugLoc(MI);
+
+ // Convert the lower half
+ Register LoFp32 = MRI.createGenericVirtualRegister(V2F32);
+ MRI.setRegClass(LoFp32, &AArch64::FPR64RegClass);
+ B.buildInstr(AArch64::FCVTXNv2f32, {LoFp32}, {LoFp64});
+
+ // Create a register for the high half to use
+ Register AccUndef = MRI.createGenericVirtualRegister(V4F32);
+ MRI.setRegClass(AccUndef, &AArch64::FPR128RegClass);
+ B.buildUndef(AccUndef);
+
+ Register Acc = MRI.createGenericVirtualRegister(V4F32);
+ MRI.setRegClass(Acc, &AArch64::FPR128RegClass);
+ B.buildInstr(TargetOpcode::INSERT_SUBREG)
+ .addDef(Acc)
+ .addUse(AccUndef)
+ .addUse(LoFp32)
+ .addImm(AArch64::dsub);
+
+ // Convert the high half
+ Register AccOut = MRI.createGenericVirtualRegister(V4F32);
+ MRI.setRegClass(AccOut, &AArch64::FPR128RegClass);
+ B.buildInstr(AArch64::FCVTXNv4f32)
+ .addDef(AccOut)
+ .addUse(Acc)
+ .addUse(HiFp64);
+
+ Register Fp16 = MRI.createGenericVirtualRegister(V4F16);
+ MRI.setRegClass(Fp16, &AArch64::FPR64RegClass);
+ B.buildFPTrunc(Fp16, AccOut);
+
+ MRI.replaceRegWith(Dst, Fp16);
+ MI.eraseFromParent();
+ break;
+ }
+ }
+}
+
/// \returns true if it would be profitable to swap the LHS and RHS of a G_ICMP
/// instruction \p MI.
bool trySwapICmpOperands(MachineInstr &MI, MachineRegisterInfo &MRI) {
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 896603d6eb20d..0561f91b6e015 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -555,11 +555,11 @@
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FPEXT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FPTRUNC (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FPTOSI (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/CodeGen/AArch64/arm64-fp128.ll b/llvm/test/CodeGen/AArch64/arm64-fp128.ll
index 3e4b887fed55d..b8b8d20b9a17b 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fp128.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fp128.ll
@@ -1197,30 +1197,22 @@ define <2 x half> @vec_round_f16(<2 x fp128> %val) {
;
; CHECK-GI-LABEL: vec_round_f16:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #64
-; CHECK-GI-NEXT: str x30, [sp, #48] // 8-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 64
+; CHECK-GI-NEXT: sub sp, sp, #48
+; CHECK-GI-NEXT: str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 48
; CHECK-GI-NEXT: .cfi_offset w30, -16
-; CHECK-GI-NEXT: mov v2.d[0], x8
; CHECK-GI-NEXT: str q1, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT: mov v2.d[1], x8
-; CHECK-GI-NEXT: str q2, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT: bl __trunctfhf2
; CHECK-GI-NEXT: // kill: def $h0 killed $h0 def $q0
; CHECK-GI-NEXT: str q0, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: bl __trunctfhf2
+; CHECK-GI-NEXT: ldr q1, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: // kill: def $h0 killed $h0 def $q0
-; CHECK-GI-NEXT: str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT: ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: bl __trunctfhf2
-; CHECK-GI-NEXT: ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: bl __trunctfhf2
-; CHECK-GI-NEXT: ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-GI-NEXT: ldr x30, [sp, #48] // 8-byte Folded Reload
-; CHECK-GI-NEXT: mov v0.h[1], v1.h[0]
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: add sp, sp, #64
+; CHECK-GI-NEXT: ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[1], v0.h[0]
+; CHECK-GI-NEXT: fmov d0, d1
+; CHECK-GI-NEXT: add sp, sp, #48
; CHECK-GI-NEXT: ret
%dst = fptrunc <2 x fp128> %val to <2 x half>
ret <2 x half> %dst
diff --git a/llvm/test/CodeGen/AArch64/fmla.ll b/llvm/test/CodeGen/AArch64/fmla.ll
index a37aabb0b5384..12b6562b5cf0c 100644
--- a/llvm/test/CodeGen/AArch64/fmla.ll
+++ b/llvm/test/CodeGen/AArch64/fmla.ll
@@ -865,22 +865,22 @@ define <7 x half> @fmuladd_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v3.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v2.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v5.4h
-; CHECK-GI-NOFP16-NEXT: mov v5.h[0], v2.h[4]
; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v4.4h
; CHECK-GI-NOFP16-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT: mov v5.h[1], v2.h[5]
-; CHECK-GI-NOFP16-NEXT: fmul v1.4s, v3.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT: mov v5.h[2], v2.h[6]
-; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT: mov v0.h[0], v3.h[0]
-; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v5.4h
+; CHECK-GI-NOFP16-NEXT: mov v1.h[0], v2.h[4]
+; CHECK-GI-NOFP16-NEXT: fmul v3.4s, v3.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT: mov v1.h[1], v2.h[5]
+; CHECK-GI-NOFP16-NEXT: fcvtn v4.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v3.4s
+; CHECK-GI-NOFP16-NEXT: mov v1.h[2], v2.h[6]
+; CHECK-GI-NOFP16-NEXT: mov v0.h[0], v4.h[0]
+; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v3.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT: mov v0.h[1], v3.h[1]
-; CHECK-GI-NOFP16-NEXT: fadd v1.4s, v1.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT: mov v0.h[2], v3.h[2]
+; CHECK-GI-NOFP16-NEXT: mov v0.h[1], v4.h[1]
+; CHECK-GI-NOFP16-NEXT: fadd v1.4s, v2.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT: mov v0.h[2], v4.h[2]
; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT: mov v0.h[3], v3.h[3]
+; CHECK-GI-NOFP16-NEXT: mov v0.h[3], v4.h[3]
; CHECK-GI-NOFP16-NEXT: mov v0.h[4], v1.h[0]
; CHECK-GI-NOFP16-NEXT: mov v0.h[5], v1.h[1]
; CHECK-GI-NOFP16-NEXT: mov v0.h[6], v1.h[2]
@@ -1350,22 +1350,22 @@ define <7 x half> @fmul_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v3.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v2.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v5.4h
-; CHECK-GI-NOFP16-NEXT: mov v5.h[0], v2.h[4]
; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v4.4h
; CHECK-GI-NOFP16-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT: mov v5.h[1], v2.h[5]
-; CHECK-GI-NOFP16-NEXT: fmul v1.4s, v3.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT: mov v5.h[2], v2.h[6]
-; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT: mov v0.h[0], v3.h[0]
-; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v5.4h
+; CHECK-GI-NOFP16-NEXT: mov v1.h[0], v2.h[4]
+; CHECK-GI-NOFP16-NEXT: fmul v3.4s, v3.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT: mov v1.h[1], v2.h[5]
+; CHECK-GI-NOFP16-NEXT: fcvtn v4.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v3.4s
+; CHECK-GI-NOFP16-NEXT: mov v1.h[2], v2.h[6]
+; CHECK-GI-NOFP16-NEXT: mov v0.h[0], v4.h[0]
+; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v3.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT: mov v0.h[1], v3.h[1]
-; CHECK-GI-NOFP16-NEXT: fadd v1.4s, v1.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT: mov v0.h[2], v3.h[2]
+; CHECK-GI-NOFP16-NEXT: mov v0.h[1], v4.h[1]
+; CHECK-GI-NOFP16-NEXT: fadd v1.4s, v2.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT: mov v0.h[2], v4.h[2]
; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT: mov v0.h[3], v3.h[3]
+; CHECK-GI-NOFP16-NEXT: mov v0.h[3], v4.h[3]
; CHECK-GI-NOFP16-NEXT: mov v0.h[4], v1.h[0]
; CHECK-GI-NOFP16-NEXT: mov v0.h[5], v1.h[1]
; CHECK-GI-NOFP16-NEXT: mov v0.h[6], v1.h[2]
diff --git a/llvm/test/CodeGen/AArch64/fp16-v4-instructions.ll b/llvm/test/CodeGen/AArch64/fp16-v4-instructions.ll
index 6233ce743b706..760742a4efad7 100644
--- a/llvm/test/CodeGen/AArch64/fp16-v4-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/fp16-v4-instructions.ll
@@ -170,47 +170,12 @@ define <4 x half> @s_to_h(<4 x float> %a) {
}
define <4 x half> @d_to_h(<4 x double> %a) {
-; CHECK-CVT-SD-LABEL: d_to_h:
-; CHECK-CVT-SD: // %bb.0:
-; CHECK-CVT-SD-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-CVT-SD-NEXT: fcvtxn2 v0.4s, v1.2d
-; CHECK-CVT-SD-NEXT: fcvtn v0.4h, v0.4s
-; CHECK-CVT-SD-NEXT: ret
-;
-; CHECK-FP16-SD-LABEL: d_to_h:
-; CHECK-FP16-SD: // %bb.0:
-; CHECK-FP16-SD-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-FP16-SD-NEXT: fcvtxn2 v0.4s, v1.2d
-; CHECK-FP16-SD-NEXT: fcvtn v0.4h, v0.4s
-; CHECK-FP16-SD-NEXT: ret
-;
-; CHECK-CVT-GI-LABEL: d_to_h:
-; CHECK-CVT-GI: // %bb.0:
-; CHECK-CVT-GI-NEXT: mov d2, v0.d[1]
-; CHECK-CVT-GI-NEXT: fcvt h0, d0
-; CHECK-CVT-GI-NEXT: mov d3, v1.d[1]
-; CHECK-CVT-GI-NEXT: fcvt h1, d1
-; CHECK-CVT-GI-NEXT: fcvt h2, d2
-; CHECK-CVT-GI-NEXT: mov v0.h[1], v2.h[0]
-; CHECK-CVT-GI-NEXT: fcvt h2, d3
-; CHECK-CVT-GI-NEXT: mov v0.h[2], v1.h[0]
-; CHECK-CVT-GI-NEXT: mov v0.h[3], v2.h[0]
-; CHECK-CVT-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-CVT-GI-NEXT: ret
-;
-; CHECK-FP16-GI-LABEL: d_to_h:
-; CHECK-FP16-GI: // %bb.0:
-; CHECK-FP16-GI-NEXT: mov d2, v0.d[1]
-; CHECK-FP16-GI-NEXT: fcvt h0, d0
-; CHECK-FP16-GI-NEXT: mov d3, v1.d[1]
-; CHECK-FP16-GI-NEXT: fcvt h1, d1
-; CHECK-FP16-GI-NEXT: fcvt h2, d2
-; CHECK-FP16-GI-NEXT: mov v0.h[1], v2.h[0]
-; CHECK-FP16-GI-NEXT: fcvt h2, d3
-; CHECK-FP16-GI-NEXT: mov v0.h[2], v1.h[0]
-; CHECK-FP16-GI-NEXT: mov v0.h[3], v2.h[0]
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-FP16-GI-NEXT: ret
+; CHECK-LABEL: d_to_h:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-NEXT: fcvtxn2 v0.4s, v1.2d
+; CHECK-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-NEXT: ret
%1 = fptrunc <4 x double> %a to <4 x half>
ret <4 x half> %1
}
@@ -241,30 +206,16 @@ define <4 x double> @h_to_d(<4 x half> %a) {
;
; CHECK-CVT-GI-LABEL: h_to_d:
; CHECK-CVT-GI: // %bb.0:
-; CHECK-CVT-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-CVT-GI-NEXT: mov h1, v0.h[1]
-; CHECK-CVT-GI-NEXT: mov h2, v0.h[2]
-; CHECK-CVT-GI-NEXT: mov h3, v0.h[3]
-; CHECK-CVT-GI-NEXT: fcvt d0, h0
-; CHECK-CVT-GI-NEXT: fcvt d4, h1
-; CHECK-CVT-GI-NEXT: fcvt d1, h2
-; CHECK-CVT-GI-NEXT: fcvt d2, h3
-; CHECK-CVT-GI-NEXT: mov v0.d[1], v4.d[0]
-; CHECK-CVT-GI-NEXT: mov v1.d[1], v2.d[0]
+; CHECK-CVT-GI-NEXT: fcvtl v1.4s, v0.4h
+; CHECK-CVT-GI-NEXT: fcvtl v0.2d, v1.2s
+; CHECK-CVT-GI-NEXT: fcvtl2 v1.2d, v1.4s
; CHECK-CVT-GI-NEXT: ret
;
; CHECK-FP16-GI-LABEL: h_to_d:
; CHECK-FP16-GI: // %bb.0:
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d4, h1
-; CHECK-FP16-GI-NEXT: fcvt d1, h2
-; CHECK-FP16-GI-NEXT: fcvt d2, h3
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v4.d[0]
-; CHECK-FP16-GI-NEXT: mov v1.d[1], v2.d[0]
+; CHECK-FP16-GI-NEXT: fcvtl v1.4s, v0.4h
+; CHECK-FP16-GI-NEXT: fcvtl v0.2d, v1.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v1.2d, v1.4s
; CHECK-FP16-GI-NEXT: ret
%1 = fpext <4 x half> %a to <4 x double>
ret <4 x double> %1
diff --git a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
index 86763eb5f9e3b..4d8505679c71c 100644
--- a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
@@ -198,48 +198,22 @@ define <8 x half> @d_to_h(<8 x double> %a) {
;
; CHECK-CVT-GI-LABEL: d_to_h:
; CHECK-CVT-GI: // %bb.0:
-; CHECK-CVT-GI-NEXT: mov d4, v0.d[1]
-; CHECK-CVT-GI-NEXT: fcvt h0, d0
-; CHECK-CVT-GI-NEXT: mov d5, v1.d[1]
-; CHECK-CVT-GI-NEXT: fcvt h1, d1
-; CHECK-CVT-GI-NEXT: fcvt h4, d4
-; CHECK-CVT-GI-NEXT: mov v0.h[1], v4.h[0]
-; CHECK-CVT-GI-NEXT: fcvt h4, d5
-; CHECK-CVT-GI-NEXT: mov v0.h[2], v1.h[0]
-; CHECK-CVT-GI-NEXT: mov d1, v2.d[1]
-; CHECK-CVT-GI-NEXT: fcvt h2, d2
-; CHECK-CVT-GI-NEXT: mov v0.h[3], v4.h[0]
-; CHECK-CVT-GI-NEXT: fcvt h1, d1
-; CHECK-CVT-GI-NEXT: mov v0.h[4], v2.h[0]
-; CHECK-CVT-GI-NEXT: mov d2, v3.d[1]
-; CHECK-CVT-GI-NEXT: fcvt h3, d3
-; CHECK-CVT-GI-NEXT: mov v0.h[5], v1.h[0]
-; CHECK-CVT-GI-NEXT: fcvt h1, d2
-; CHECK-CVT-GI-NEXT: mov v0.h[6], v3.h[0]
-; CHECK-CVT-GI-NEXT: mov v0.h[7], v1.h[0]
+; CHECK-CVT-GI-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-CVT-GI-NEXT: fcvtxn2 v0.4s, v1.2d
+; CHECK-CVT-GI-NEXT: fcvtxn v1.2s, v2.2d
+; CHECK-CVT-GI-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-CVT-GI-NEXT: fcvtxn2 v1.4s, v3.2d
+; CHECK-CVT-GI-NEXT: fcvtn2 v0.8h, v1.4s
; CHECK-CVT-GI-NEXT: ret
;
; CHECK-FP16-GI-LABEL: d_to_h:
; CHECK-FP16-GI: // %bb.0:
-; CHECK-FP16-GI-NEXT: mov d4, v0.d[1]
-; CHECK-FP16-GI-NEXT: fcvt h0, d0
-; CHECK-FP16-GI-NEXT: mov d5, v1.d[1]
-; CHECK-FP16-GI-NEXT: fcvt h1, d1
-; CHECK-FP16-GI-NEXT: fcvt h4, d4
-; CHECK-FP16-GI-NEXT: mov v0.h[1], v4.h[0]
-; CHECK-FP16-GI-NEXT: fcvt h4, d5
-; CHECK-FP16-GI-NEXT: mov v0.h[2], v1.h[0]
-; CHECK-FP16-GI-NEXT: mov d1, v2.d[1]
-; CHECK-FP16-GI-NEXT: fcvt h2, d2
-; CHECK-FP16-GI-NEXT: mov v0.h[3], v4.h[0]
-; CHECK-FP16-GI-NEXT: fcvt h1, d1
-; CHECK-FP16-GI-NEXT: mov v0.h[4], v2.h[0]
-; CHECK-FP16-GI-NEXT: mov d2, v3.d[1]
-; CHECK-FP16-GI-NEXT: fcvt h3, d3
-; CHECK-FP16-GI-NEXT: mov v0.h[5], v1.h[0]
-; CHECK-FP16-GI-NEXT: fcvt h1, d2
-; CHECK-FP16-GI-NEXT: mov v0.h[6], v3.h[0]
-; CHECK-FP16-GI-NEXT: mov v0.h[7], v1.h[0]
+; CHECK-FP16-GI-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtxn2 v0.4s, v1.2d
+; CHECK-FP16-GI-NEXT: fcvtxn v1.2s, v2.2d
+; CHECK-FP16-GI-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtxn2 v1.4s, v3.2d
+; CHECK-FP16-GI-NEXT: fcvtn2 v0.8h, v1.4s
; CHECK-FP16-GI-NEXT: ret
%1 = fptrunc <8 x double> %a to <8 x half>
ret <8 x half> %1
@@ -298,48 +272,22 @@ define <8 x double> @h_to_d(<8 x half> %a) {
;
; CHECK-CVT-GI-LABEL: h_to_d:
; CHECK-CVT-GI: // %bb.0:
-; CHECK-CVT-GI-NEXT: mov h1, v0.h[1]
-; CHECK-CVT-GI-NEXT: mov h2, v0.h[2]
-; CHECK-CVT-GI-NEXT: mov h3, v0.h[3]
-; CHECK-CVT-GI-NEXT: mov h4, v0.h[4]
-; CHECK-CVT-GI-NEXT: mov h5, v0.h[5]
-; CHECK-CVT-GI-NEXT: mov h6, v0.h[6]
-; CHECK-CVT-GI-NEXT: mov h7, v0.h[7]
-; CHECK-CVT-GI-NEXT: fcvt d0, h0
-; CHECK-CVT-GI-NEXT: fcvt d16, h1
-; CHECK-CVT-GI-NEXT: fcvt d1, h2
-; CHECK-CVT-GI-NEXT: fcvt d17, h3
-; CHECK-CVT-GI-NEXT: fcvt d2, h4
-; CHECK-CVT-GI-NEXT: fcvt d4, h5
-; CHECK-CVT-GI-NEXT: fcvt d3, h6
-; CHECK-CVT-GI-NEXT: fcvt d5, h7
-; CHECK-CVT-GI-NEXT: mov v0.d[1], v16.d[0]
-; CHECK-CVT-GI-NEXT: mov v1.d[1], v17.d[0]
-; CHECK-CVT-GI-NEXT: mov v2.d[1], v4.d[0]
-; CHECK-CVT-GI-NEXT: mov v3.d[1], v5.d[0]
+; CHECK-CVT-GI-NEXT: fcvtl v1.4s, v0.4h
+; CHECK-CVT-GI-NEXT: fcvtl2 v3.4s, v0.8h
+; CHECK-CVT-GI-NEXT: fcvtl v0.2d, v1.2s
+; CHECK-CVT-GI-NEXT: fcvtl2 v1.2d, v1.4s
+; CHECK-CVT-GI-NEXT: fcvtl v2.2d, v3.2s
+; CHECK-CVT-GI-NEXT: fcvtl2 v3.2d, v3.4s
; CHECK-CVT-GI-NEXT: ret
;
; CHECK-FP16-GI-LABEL: h_to_d:
; CHECK-FP16-GI: // %bb.0:
-; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
-; CHECK-FP16-GI-NEXT: mov h4, v0.h[4]
-; CHECK-FP16-GI-NEXT: mov h5, v0.h[5]
-; CHECK-FP16-GI-NEXT: mov h6, v0.h[6]
-; CHECK-FP16-GI-NEXT: mov h7, v0.h[7]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d16, h1
-; CHECK-FP16-GI-NEXT: fcvt d1, h2
-; CHECK-FP16-GI-NEXT: fcvt d17, h3
-; CHECK-FP16-GI-NEXT: fcvt d2, h4
-; CHECK-FP16-GI-NEXT: fcvt d4, h5
-; CHECK-FP16-GI-NEXT: fcvt d3, h6
-; CHECK-FP16-GI-NEXT: fcvt d5, h7
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v16.d[0]
-; CHECK-FP16-GI-NEXT: mov v1.d[1], v17.d[0]
-; CHECK-FP16-GI-NEXT: mov v2.d[1], v4.d[0]
-; CHECK-FP16-GI-NEXT: mov v3.d[1], v5.d[0]
+; CHECK-FP16-GI-NEXT: fcvtl v1.4s, v0.4h
+; CHECK-FP16-GI-NEXT: fcvtl2 v3.4s, v0.8h
+; CHECK-FP16-GI-NEXT: fcvtl v0.2d, v1.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v1.2d, v1.4s
+; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v3.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v3.2d, v3.4s
; CHECK-FP16-GI-NEXT: ret
%1 = fpext <8 x half> %a to <8 x double>
ret <8 x double> %1
diff --git a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
index 637c02875b84e..b075a8b6f70ee 100644
--- a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
@@ -285,31 +285,24 @@ define <4 x i32> @stest_f16i32(<4 x half> %x) {
;
; CHECK-FP16-GI-LABEL: stest_f16i32:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-FP16-GI-NEXT: adrp x8, .LCPI6_1
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d1, h1
-; CHECK-FP16-GI-NEXT: fcvt d2, h2
-; CHECK-FP16-GI-NEXT: fcvt d3, h3
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
-; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v2.2d
; CHECK-FP16-GI-NEXT: ldr q2, [x8, :lo12:.LCPI6_1]
; CHECK-FP16-GI-NEXT: adrp x8, .LCPI6_0
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v2.2d, v0.2d
-; CHECK-FP16-GI-NEXT: cmgt v4.2d, v2.2d, v1.2d
-; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT: fcvtl v1.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v2.2d, v1.2d
+; CHECK-FP16-GI-NEXT: cmgt v4.2d, v2.2d, v0.2d
+; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v4.16b
; CHECK-FP16-GI-NEXT: ldr q2, [x8, :lo12:.LCPI6_0]
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v0.2d, v2.2d
-; CHECK-FP16-GI-NEXT: cmgt v4.2d, v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v4.16b
-; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: cmgt v4.2d, v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v1.4s, v0.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%conv = fptosi <4 x half> %x to <4 x i64>
@@ -351,24 +344,17 @@ define <4 x i32> @utest_f16i32(<4 x half> %x) {
;
; CHECK-FP16-GI-LABEL: utest_f16i32:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[2]
-; CHECK-FP16-GI-NEXT: mov h4, v0.h[3]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-FP16-GI-NEXT: movi v1.2d, #0x000000ffffffff
-; CHECK-FP16-GI-NEXT: fcvt d2, h2
-; CHECK-FP16-GI-NEXT: fcvt d3, h3
-; CHECK-FP16-GI-NEXT: fcvt d4, h4
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v2.d[0]
-; CHECK-FP16-GI-NEXT: mov v3.d[1], v4.d[0]
+; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v2.2d
; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v3.2d
-; CHECK-FP16-GI-NEXT: cmhi v3.2d, v1.2d, v0.2d
-; CHECK-FP16-GI-NEXT: cmhi v4.2d, v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bit v1.16b, v2.16b, v4.16b
-; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT: cmhi v3.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: cmhi v4.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT: bif v2.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v4.16b
+; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v2.4s, v0.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%conv = fptoui <4 x half> %x to <4 x i64>
@@ -412,28 +398,21 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) {
;
; CHECK-FP16-GI-LABEL: ustest_f16i32:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[2]
-; CHECK-FP16-GI-NEXT: mov h4, v0.h[3]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-FP16-GI-NEXT: movi v1.2d, #0x000000ffffffff
-; CHECK-FP16-GI-NEXT: fcvt d2, h2
-; CHECK-FP16-GI-NEXT: fcvt d3, h3
-; CHECK-FP16-GI-NEXT: fcvt d4, h4
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v2.d[0]
-; CHECK-FP16-GI-NEXT: mov v3.d[1], v4.d[0]
+; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v2.2d
; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v3.2d
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, v0.2d
-; CHECK-FP16-GI-NEXT: cmgt v4.2d, v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bit v1.16b, v2.16b, v4.16b
-; CHECK-FP16-GI-NEXT: cmgt v2.2d, v0.2d, #0
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, #0
-; CHECK-FP16-GI-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-FP16-GI-NEXT: and v1.16b, v1.16b, v3.16b
-; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: cmgt v4.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT: bif v2.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v4.16b
+; CHECK-FP16-GI-NEXT: cmgt v1.2d, v2.2d, #0
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v0.2d, #0
+; CHECK-FP16-GI-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-FP16-GI-NEXT: and v0.16b, v0.16b, v3.16b
+; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v1.4s, v0.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%conv = fptosi <4 x half> %x to <4 x i64>
@@ -2273,31 +2252,24 @@ define <4 x i32> @stest_f16i32_mm(<4 x half> %x) {
;
; CHECK-FP16-GI-LABEL: stest_f16i32_mm:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-FP16-GI-NEXT: adrp x8, .LCPI33_1
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d1, h1
-; CHECK-FP16-GI-NEXT: fcvt d2, h2
-; CHECK-FP16-GI-NEXT: fcvt d3, h3
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
-; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v2.2d
; CHECK-FP16-GI-NEXT: ldr q2, [x8, :lo12:.LCPI33_1]
; CHECK-FP16-GI-NEXT: adrp x8, .LCPI33_0
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v2.2d, v0.2d
-; CHECK-FP16-GI-NEXT: cmgt v4.2d, v2.2d, v1.2d
-; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT: fcvtl v1.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v2.2d, v1.2d
+; CHECK-FP16-GI-NEXT: cmgt v4.2d, v2.2d, v0.2d
+; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v4.16b
; CHECK-FP16-GI-NEXT: ldr q2, [x8, :lo12:.LCPI33_0]
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v0.2d, v2.2d
-; CHECK-FP16-GI-NEXT: cmgt v4.2d, v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v4.16b
-; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: cmgt v4.2d, v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v1.4s, v0.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%conv = fptosi <4 x half> %x to <4 x i64>
@@ -2337,24 +2309,17 @@ define <4 x i32> @utest_f16i32_mm(<4 x half> %x) {
;
; CHECK-FP16-GI-LABEL: utest_f16i32_mm:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[2]
-; CHECK-FP16-GI-NEXT: mov h4, v0.h[3]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-FP16-GI-NEXT: movi v1.2d, #0x000000ffffffff
-; CHECK-FP16-GI-NEXT: fcvt d2, h2
-; CHECK-FP16-GI-NEXT: fcvt d3, h3
-; CHECK-FP16-GI-NEXT: fcvt d4, h4
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v2.d[0]
-; CHECK-FP16-GI-NEXT: mov v3.d[1], v4.d[0]
+; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v2.2d
; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v3.2d
-; CHECK-FP16-GI-NEXT: cmhi v3.2d, v1.2d, v0.2d
-; CHECK-FP16-GI-NEXT: cmhi v4.2d, v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bit v1.16b, v2.16b, v4.16b
-; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT: cmhi v3.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: cmhi v4.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT: bif v2.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v4.16b
+; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v2.4s, v0.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%conv = fptoui <4 x half> %x to <4 x i64>
@@ -2397,28 +2362,21 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) {
;
; CHECK-FP16-GI-LABEL: ustest_f16i32_mm:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[2]
-; CHECK-FP16-GI-NEXT: mov h4, v0.h[3]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-FP16-GI-NEXT: movi v1.2d, #0x000000ffffffff
-; CHECK-FP16-GI-NEXT: fcvt d2, h2
-; CHECK-FP16-GI-NEXT: fcvt d3, h3
-; CHECK-FP16-GI-NEXT: fcvt d4, h4
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v2.d[0]
-; CHECK-FP16-GI-NEXT: mov v3.d[1], v4.d[0]
+; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v2.2d
; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v3.2d
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, v0.2d
-; CHECK-FP16-GI-NEXT: cmgt v4.2d, v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bit v1.16b, v2.16b, v4.16b
-; CHECK-FP16-GI-NEXT: cmgt v2.2d, v0.2d, #0
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, #0
-; CHECK-FP16-GI-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-FP16-GI-NEXT: and v1.16b, v1.16b, v3.16b
-; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: cmgt v4.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT: bif v2.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v4.16b
+; CHECK-FP16-GI-NEXT: cmgt v1.2d, v2.2d, #0
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v0.2d, #0
+; CHECK-FP16-GI-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-FP16-GI-NEXT: and v0.16b, v0.16b, v3.16b
+; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v1.4s, v0.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%conv = fptosi <4 x half> %x to <4 x i64>
diff --git a/llvm/test/CodeGen/AArch64/fpext.ll b/llvm/test/CodeGen/AArch64/fpext.ll
index df90f9d5f0910..8980340a447de 100644
--- a/llvm/test/CodeGen/AArch64/fpext.ll
+++ b/llvm/test/CodeGen/AArch64/fpext.ll
@@ -82,11 +82,12 @@ define <3 x double> @fpext_v3f32_v3f64(<3 x float> %a) {
;
; CHECK-GI-LABEL: fpext_v3f32_v3f64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov s1, v0.s[2]
+; CHECK-GI-NEXT: mov v1.s[0], v0.s[2]
; CHECK-GI-NEXT: fcvtl v0.2d, v0.2s
-; CHECK-GI-NEXT: fcvt d2, s1
+; CHECK-GI-NEXT: fcvtl v2.2d, v1.2s
; CHECK-GI-NEXT: mov d1, v0.d[1]
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: // kill: def $d2 killed $d2 killed $q2
; CHECK-GI-NEXT: ret
entry:
%c = fpext <3 x float> %a to <3 x double>
@@ -320,20 +321,11 @@ entry:
}
define <2 x double> @fpext_v2f16_v2f64(<2 x half> %a) {
-; CHECK-SD-LABEL: fpext_v2f16_v2f64:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-SD-NEXT: fcvtl v0.2d, v0.2s
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: fpext_v2f16_v2f64:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: mov h1, v0.h[1]
-; CHECK-GI-NEXT: fcvt d0, h0
-; CHECK-GI-NEXT: fcvt d1, h1
-; CHECK-GI-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: fpext_v2f16_v2f64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-NEXT: fcvtl v0.2d, v0.2s
+; CHECK-NEXT: ret
entry:
%c = fpext <2 x half> %a to <2 x double>
ret <2 x double> %c
@@ -353,12 +345,12 @@ define <3 x double> @fpext_v3f16_v3f64(<3 x half> %a) {
;
; CHECK-GI-LABEL: fpext_v3f16_v3f64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: mov h1, v0.h[1]
-; CHECK-GI-NEXT: mov h2, v0.h[2]
-; CHECK-GI-NEXT: fcvt d0, h0
-; CHECK-GI-NEXT: fcvt d1, h1
-; CHECK-GI-NEXT: fcvt d2, h2
+; CHECK-GI-NEXT: fcvtl v1.4s, v0.4h
+; CHECK-GI-NEXT: fcvtl v0.2d, v1.2s
+; CHECK-GI-NEXT: fcvtl2 v2.2d, v1.4s
+; CHECK-GI-NEXT: // kill: def $d2 killed $d2 killed $q2
+; CHECK-GI-NEXT: mov d1, v0.d[1]
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT: ret
entry:
%c = fpext <3 x half> %a to <3 x double>
@@ -375,16 +367,9 @@ define <4 x double> @fpext_v4f16_v4f64(<4 x half> %a) {
;
; CHECK-GI-LABEL: fpext_v4f16_v4f64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: mov h1, v0.h[1]
-; CHECK-GI-NEXT: mov h2, v0.h[2]
-; CHECK-GI-NEXT: mov h3, v0.h[3]
-; CHECK-GI-NEXT: fcvt d0, h0
-; CHECK-GI-NEXT: fcvt d4, h1
-; CHECK-GI-NEXT: fcvt d1, h2
-; CHECK-GI-NEXT: fcvt d2, h3
-; CHECK-GI-NEXT: mov v0.d[1], v4.d[0]
-; CHECK-GI-NEXT: mov v1.d[1], v2.d[0]
+; CHECK-GI-NEXT: fcvtl v1.4s, v0.4h
+; CHECK-GI-NEXT: fcvtl v0.2d, v1.2s
+; CHECK-GI-NEXT: fcvtl2 v1.2d, v1.4s
; CHECK-GI-NEXT: ret
entry:
%c = fpext <4 x half> %a to <4 x double>
diff --git a/llvm/test/CodeGen/AArch64/fptoi.ll b/llvm/test/CodeGen/AArch64/fptoi.ll
index f6053cee50dae..3dafabe0b69d7 100644
--- a/llvm/test/CodeGen/AArch64/fptoi.ll
+++ b/llvm/test/CodeGen/AArch64/fptoi.ll
@@ -4610,11 +4610,8 @@ define <2 x i64> @fptos_v2f16_v2i64(<2 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptos_v2f16_v2i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d1, h1
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT: fcvtl v0.2d, v0.2s
; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
; CHECK-FP16-GI-NEXT: ret
entry:
@@ -4654,11 +4651,8 @@ define <2 x i64> @fptou_v2f16_v2i64(<2 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptou_v2f16_v2i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d1, h1
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT: fcvtl v0.2d, v0.2s
; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v0.2d
; CHECK-FP16-GI-NEXT: ret
entry:
@@ -4710,20 +4704,14 @@ define <3 x i64> @fptos_v3f16_v3i64(<3 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptos_v3f16_v3i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[1]
-; CHECK-FP16-GI-NEXT: fcvt d1, h0
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[2]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d2, h2
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v2.d[0]
-; CHECK-FP16-GI-NEXT: fcvt d2, h3
-; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: mov v2.d[1], v1.d[0]
-; CHECK-FP16-GI-NEXT: mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT: fcvtl v1.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v2.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v1.2d
; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v2.2d
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-FP16-GI-NEXT: // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT: mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-FP16-GI-NEXT: ret
entry:
%c = fptosi <3 x half> %a to <3 x i64>
@@ -4774,20 +4762,14 @@ define <3 x i64> @fptou_v3f16_v3i64(<3 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptou_v3f16_v3i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[1]
-; CHECK-FP16-GI-NEXT: fcvt d1, h0
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[2]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d2, h2
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v2.d[0]
-; CHECK-FP16-GI-NEXT: fcvt d2, h3
-; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: mov v2.d[1], v1.d[0]
-; CHECK-FP16-GI-NEXT: mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT: fcvtl v1.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v2.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v1.2d
; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v2.2d
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-FP16-GI-NEXT: // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT: mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-FP16-GI-NEXT: ret
entry:
%c = fptoui <3 x half> %a to <3 x i64>
@@ -4842,17 +4824,10 @@ define <4 x i64> @fptos_v4f16_v4i64(<4 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptos_v4f16_v4i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d1, h1
-; CHECK-FP16-GI-NEXT: fcvt d2, h2
-; CHECK-FP16-GI-NEXT: fcvt d3, h3
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
-; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT: fcvtl v1.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v2.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v1.2d
; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v2.2d
; CHECK-FP16-GI-NEXT: ret
entry:
@@ -4908,17 +4883,10 @@ define <4 x i64> @fptou_v4f16_v4i64(<4 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptou_v4f16_v4i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d1, h1
-; CHECK-FP16-GI-NEXT: fcvt d2, h2
-; CHECK-FP16-GI-NEXT: fcvt d3, h3
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
-; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT: fcvtl v1.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v2.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v1.2d
; CHECK-FP16-GI-NEXT: fcvtzu v1.2d, v2.2d
; CHECK-FP16-GI-NEXT: ret
entry:
@@ -5005,29 +4973,16 @@ define <8 x i64> @fptos_v8f16_v8i64(<8 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptos_v8f16_v8i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
-; CHECK-FP16-GI-NEXT: mov h4, v0.h[4]
-; CHECK-FP16-GI-NEXT: mov h5, v0.h[5]
-; CHECK-FP16-GI-NEXT: mov h6, v0.h[6]
-; CHECK-FP16-GI-NEXT: mov h7, v0.h[7]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d1, h1
-; CHECK-FP16-GI-NEXT: fcvt d2, h2
-; CHECK-FP16-GI-NEXT: fcvt d3, h3
-; CHECK-FP16-GI-NEXT: fcvt d4, h4
-; CHECK-FP16-GI-NEXT: fcvt d5, h5
-; CHECK-FP16-GI-NEXT: fcvt d6, h6
-; CHECK-FP16-GI-NEXT: fcvt d7, h7
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
-; CHECK-FP16-GI-NEXT: mov v4.d[1], v5.d[0]
-; CHECK-FP16-GI-NEXT: mov v6.d[1], v7.d[0]
-; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v4.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v3.2d, v6.2d
+; CHECK-FP16-GI-NEXT: fcvtl v1.4s, v0.4h
+; CHECK-FP16-GI-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v1.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v1.2d, v1.4s
+; CHECK-FP16-GI-NEXT: fcvtl v3.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v4.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v3.2d, v4.2d
; CHECK-FP16-GI-NEXT: ret
entry:
%c = fptosi <8 x half> %a to <8 x i64>
@@ -5113,29 +5068,16 @@ define <8 x i64> @fptou_v8f16_v8i64(<8 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptou_v8f16_v8i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
-; CHECK-FP16-GI-NEXT: mov h4, v0.h[4]
-; CHECK-FP16-GI-NEXT: mov h5, v0.h[5]
-; CHECK-FP16-GI-NEXT: mov h6, v0.h[6]
-; CHECK-FP16-GI-NEXT: mov h7, v0.h[7]
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d1, h1
-; CHECK-FP16-GI-NEXT: fcvt d2, h2
-; CHECK-FP16-GI-NEXT: fcvt d3, h3
-; CHECK-FP16-GI-NEXT: fcvt d4, h4
-; CHECK-FP16-GI-NEXT: fcvt d5, h5
-; CHECK-FP16-GI-NEXT: fcvt d6, h6
-; CHECK-FP16-GI-NEXT: fcvt d7, h7
-; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
-; CHECK-FP16-GI-NEXT: mov v4.d[1], v5.d[0]
-; CHECK-FP16-GI-NEXT: mov v6.d[1], v7.d[0]
-; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v4.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v3.2d, v6.2d
+; CHECK-FP16-GI-NEXT: fcvtl v1.4s, v0.4h
+; CHECK-FP16-GI-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v1.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v1.2d, v1.4s
+; CHECK-FP16-GI-NEXT: fcvtl v3.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v4.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v3.2d, v4.2d
; CHECK-FP16-GI-NEXT: ret
entry:
%c = fptoui <8 x half> %a to <8 x i64>
@@ -5285,52 +5227,26 @@ define <16 x i64> @fptos_v16f16_v16i64(<16 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptos_v16f16_v16i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h4, v0.h[2]
-; CHECK-FP16-GI-NEXT: mov h5, v0.h[3]
-; CHECK-FP16-GI-NEXT: fcvt d2, h0
-; CHECK-FP16-GI-NEXT: mov h6, v0.h[4]
-; CHECK-FP16-GI-NEXT: mov h7, v0.h[5]
-; CHECK-FP16-GI-NEXT: mov h16, v0.h[6]
-; CHECK-FP16-GI-NEXT: mov h0, v0.h[7]
-; CHECK-FP16-GI-NEXT: mov h17, v1.h[1]
-; CHECK-FP16-GI-NEXT: mov h18, v1.h[2]
-; CHECK-FP16-GI-NEXT: mov h19, v1.h[3]
-; CHECK-FP16-GI-NEXT: mov h20, v1.h[4]
-; CHECK-FP16-GI-NEXT: mov h21, v1.h[5]
-; CHECK-FP16-GI-NEXT: mov h22, v1.h[6]
-; CHECK-FP16-GI-NEXT: mov h23, v1.h[7]
-; CHECK-FP16-GI-NEXT: fcvt d3, h3
-; CHECK-FP16-GI-NEXT: fcvt d4, h4
-; CHECK-FP16-GI-NEXT: fcvt d5, h5
-; CHECK-FP16-GI-NEXT: fcvt d6, h6
-; CHECK-FP16-GI-NEXT: fcvt d7, h7
-; CHECK-FP16-GI-NEXT: fcvt d16, h16
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d24, h1
-; CHECK-FP16-GI-NEXT: fcvt d1, h17
-; CHECK-FP16-GI-NEXT: fcvt d17, h18
-; CHECK-FP16-GI-NEXT: fcvt d18, h19
-; CHECK-FP16-GI-NEXT: fcvt d19, h20
-; CHECK-FP16-GI-NEXT: fcvt d20, h21
-; CHECK-FP16-GI-NEXT: fcvt d21, h22
-; CHECK-FP16-GI-NEXT: fcvt d22, h23
-; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
-; CHECK-FP16-GI-NEXT: mov v4.d[1], v5.d[0]
-; CHECK-FP16-GI-NEXT: mov v6.d[1], v7.d[0]
-; CHECK-FP16-GI-NEXT: mov v16.d[1], v0.d[0]
-; CHECK-FP16-GI-NEXT: mov v24.d[1], v1.d[0]
-; CHECK-FP16-GI-NEXT: mov v17.d[1], v18.d[0]
-; CHECK-FP16-GI-NEXT: mov v19.d[1], v20.d[0]
-; CHECK-FP16-GI-NEXT: mov v21.d[1], v22.d[0]
-; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v2.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v4.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v6.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v3.2d, v16.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v4.2d, v24.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v5.2d, v17.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v6.2d, v19.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v7.2d, v21.2d
+; CHECK-FP16-GI-NEXT: fcvtl v2.4s, v0.4h
+; CHECK-FP16-GI-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-FP16-GI-NEXT: fcvtl v3.4s, v1.4h
+; CHECK-FP16-GI-NEXT: fcvtl2 v1.4s, v1.8h
+; CHECK-FP16-GI-NEXT: fcvtl v4.2d, v2.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v2.2d, v2.4s
+; CHECK-FP16-GI-NEXT: fcvtl v5.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v6.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtl v7.2d, v3.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v16.2d, v3.4s
+; CHECK-FP16-GI-NEXT: fcvtl v17.2d, v1.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v18.2d, v1.4s
+; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v4.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v5.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v3.2d, v6.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v4.2d, v7.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v5.2d, v16.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v6.2d, v17.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v7.2d, v18.2d
; CHECK-FP16-GI-NEXT: ret
entry:
%c = fptosi <16 x half> %a to <16 x i64>
@@ -5480,52 +5396,26 @@ define <16 x i64> @fptou_v16f16_v16i64(<16 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptou_v16f16_v16i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: mov h3, v0.h[1]
-; CHECK-FP16-GI-NEXT: mov h4, v0.h[2]
-; CHECK-FP16-GI-NEXT: mov h5, v0.h[3]
-; CHECK-FP16-GI-NEXT: fcvt d2, h0
-; CHECK-FP16-GI-NEXT: mov h6, v0.h[4]
-; CHECK-FP16-GI-NEXT: mov h7, v0.h[5]
-; CHECK-FP16-GI-NEXT: mov h16, v0.h[6]
-; CHECK-FP16-GI-NEXT: mov h0, v0.h[7]
-; CHECK-FP16-GI-NEXT: mov h17, v1.h[1]
-; CHECK-FP16-GI-NEXT: mov h18, v1.h[2]
-; CHECK-FP16-GI-NEXT: mov h19, v1.h[3]
-; CHECK-FP16-GI-NEXT: mov h20, v1.h[4]
-; CHECK-FP16-GI-NEXT: mov h21, v1.h[5]
-; CHECK-FP16-GI-NEXT: mov h22, v1.h[6]
-; CHECK-FP16-GI-NEXT: mov h23, v1.h[7]
-; CHECK-FP16-GI-NEXT: fcvt d3, h3
-; CHECK-FP16-GI-NEXT: fcvt d4, h4
-; CHECK-FP16-GI-NEXT: fcvt d5, h5
-; CHECK-FP16-GI-NEXT: fcvt d6, h6
-; CHECK-FP16-GI-NEXT: fcvt d7, h7
-; CHECK-FP16-GI-NEXT: fcvt d16, h16
-; CHECK-FP16-GI-NEXT: fcvt d0, h0
-; CHECK-FP16-GI-NEXT: fcvt d24, h1
-; CHECK-FP16-GI-NEXT: fcvt d1, h17
-; CHECK-FP16-GI-NEXT: fcvt d17, h18
-; CHECK-FP16-GI-NEXT: fcvt d18, h19
-; CHECK-FP16-GI-NEXT: fcvt d19, h20
-; CHECK-FP16-GI-NEXT: fcvt d20, h21
-; CHECK-FP16-GI-NEXT: fcvt d21, h22
-; CHECK-FP16-GI-NEXT: fcvt d22, h23
-; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
-; CHECK-FP16-GI-NEXT: mov v4.d[1], v5.d[0]
-; CHECK-FP16-GI-NEXT: mov v6.d[1], v7.d[0]
-; CHECK-FP16-GI-NEXT: mov v16.d[1], v0.d[0]
-; CHECK-FP16-GI-NEXT: mov v24.d[1], v1.d[0]
-; CHECK-FP16-GI-NEXT: mov v17.d[1], v18.d[0]
-; CHECK-FP16-GI-NEXT: mov v19.d[1], v20.d[0]
-; CHECK-FP16-GI-NEXT: mov v21.d[1], v22.d[0]
-; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v2.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v1.2d, v4.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v6.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v3.2d, v16.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v4.2d, v24.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v5.2d, v17.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v6.2d, v19.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v7.2d, v21.2d
+; CHECK-FP16-GI-NEXT: fcvtl v2.4s, v0.4h
+; CHECK-FP16-GI-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-FP16-GI-NEXT: fcvtl v3.4s, v1.4h
+; CHECK-FP16-GI-NEXT: fcvtl2 v1.4s, v1.8h
+; CHECK-FP16-GI-NEXT: fcvtl v4.2d, v2.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v2.2d, v2.4s
+; CHECK-FP16-GI-NEXT: fcvtl v5.2d, v0.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v6.2d, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtl v7.2d, v3.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v16.2d, v3.4s
+; CHECK-FP16-GI-NEXT: fcvtl v17.2d, v1.2s
+; CHECK-FP16-GI-NEXT: fcvtl2 v18.2d, v1.4s
+; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v4.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v5.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v3.2d, v6.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v4.2d, v7.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v5.2d, v16.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v6.2d, v17.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v7.2d, v18.2d
; CHECK-FP16-GI-NEXT: ret
entry:
%c = fptoui <16 x half> %a to <16 x i64>
diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
index b963acd8cb2a1..dbcfaff8aee05 100644
--- a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
@@ -3088,30 +3088,14 @@ define <4 x i64> @test_signed_v4f16_v4i64(<4 x half> %f) {
; CHECK-SD-FP16-NEXT: mov v1.d[1], x11
; CHECK-SD-FP16-NEXT: ret
;
-; CHECK-GI-CVT-LABEL: test_signed_v4f16_v4i64:
-; CHECK-GI-CVT: // %bb.0:
-; CHECK-GI-CVT-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-GI-CVT-NEXT: fcvtl v1.2d, v0.2s
-; CHECK-GI-CVT-NEXT: fcvtl2 v2.2d, v0.4s
-; CHECK-GI-CVT-NEXT: fcvtzs v0.2d, v1.2d
-; CHECK-GI-CVT-NEXT: fcvtzs v1.2d, v2.2d
-; CHECK-GI-CVT-NEXT: ret
-;
-; CHECK-GI-FP16-LABEL: test_signed_v4f16_v4i64:
-; CHECK-GI-FP16: // %bb.0:
-; CHECK-GI-FP16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT: mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT: mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT: mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT: fcvt d0, h0
-; CHECK-GI-FP16-NEXT: fcvt d1, h1
-; CHECK-GI-FP16-NEXT: fcvt d2, h2
-; CHECK-GI-FP16-NEXT: fcvt d3, h3
-; CHECK-GI-FP16-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT: mov v2.d[1], v3.d[0]
-; CHECK-GI-FP16-NEXT: fcvtzs v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT: fcvtzs v1.2d, v2.2d
-; CHECK-GI-FP16-NEXT: ret
+; CHECK-GI-LABEL: test_signed_v4f16_v4i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-GI-NEXT: fcvtl v1.2d, v0.2s
+; CHECK-GI-NEXT: fcvtl2 v2.2d, v0.4s
+; CHECK-GI-NEXT: fcvtzs v0.2d, v1.2d
+; CHECK-GI-NEXT: fcvtzs v1.2d, v2.2d
+; CHECK-GI-NEXT: ret
%x = call <4 x i64> @llvm.fptosi.sat.v4f16.v4i64(<4 x half> %f)
ret <4 x i64> %x
}
@@ -3797,46 +3781,19 @@ define <8 x i64> @test_signed_v8f16_v8i64(<8 x half> %f) {
; CHECK-SD-FP16-NEXT: mov v3.d[1], x14
; CHECK-SD-FP16-NEXT: ret
;
-; CHECK-GI-CVT-LABEL: test_signed_v8f16_v8i64:
-; CHECK-GI-CVT: // %bb.0:
-; CHECK-GI-CVT-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-GI-CVT-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-GI-CVT-NEXT: fcvtl v2.2d, v1.2s
-; CHECK-GI-CVT-NEXT: fcvtl2 v1.2d, v1.4s
-; CHECK-GI-CVT-NEXT: fcvtl v3.2d, v0.2s
-; CHECK-GI-CVT-NEXT: fcvtl2 v4.2d, v0.4s
-; CHECK-GI-CVT-NEXT: fcvtzs v0.2d, v2.2d
-; CHECK-GI-CVT-NEXT: fcvtzs v1.2d, v1.2d
-; CHECK-GI-CVT-NEXT: fcvtzs v2.2d, v3.2d
-; CHECK-GI-CVT-NEXT: fcvtzs v3.2d, v4.2d
-; CHECK-GI-CVT-NEXT: ret
-;
-; CHECK-GI-FP16-LABEL: test_signed_v8f16_v8i64:
-; CHECK-GI-FP16: // %bb.0:
-; CHECK-GI-FP16-NEXT: mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT: mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT: mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT: mov h4, v0.h[4]
-; CHECK-GI-FP16-NEXT: mov h5, v0.h[5]
-; CHECK-GI-FP16-NEXT: mov h6, v0.h[6]
-; CHECK-GI-FP16-NEXT: mov h7, v0.h[7]
-; CHECK-GI-FP16-NEXT: fcvt d0, h0
-; CHECK-GI-FP16-NEXT: fcvt d1, h1
-; CHECK-GI-FP16-NEXT: fcvt d2, h2
-; CHECK-GI-FP16-NEXT: fcvt d3, h3
-; CHECK-GI-FP16-NEXT: fcvt d4, h4
-; CHECK-GI-FP16-NEXT: fcvt d5, h5
-; CHECK-GI-FP16-NEXT: fcvt d6, h6
-; CHECK-GI-FP16-NEXT: fcvt d7, h7
-; CHECK-GI-FP16-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT: mov v2.d[1], v3.d[0]
-; CHECK-GI-FP16-NEXT: mov v4.d[1], v5.d[0]
-; CHECK-GI-FP16-NEXT: mov v6.d[1], v7.d[0]
-; CHECK-GI-FP16-NEXT: fcvtzs v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT: fcvtzs v1.2d, v2.2d
-; CHECK-GI-FP16-NEXT: fcvtzs v2.2d, v4.2d
-; CHECK-GI-FP16-NEXT: fcvtzs v3.2d, v6.2d
-; CHECK-GI-FP16-NEXT: ret
+; CHECK-GI-LABEL: test_signed_v8f16_v8i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: fcvtl v1.4s, v0.4h
+; CHECK-GI-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NEXT: fcvtl v2.2d, v1.2s
+; CHECK-GI-NEXT: fcvtl2 v1.2d, v1.4s
+; CHECK-GI-NEXT: fcvtl v3.2d, v0.2s
+; CHECK-GI-NEXT: fcvtl2 v4.2d, v0.4s
+; CHECK-GI-NEXT: fcvtzs v0.2d, v2.2d
+; CHECK-GI-NEXT: fcvtzs v1.2d, v1.2d
+; CHECK-GI-NEXT: fcvtzs v2.2d, v3.2d
+; CHECK-GI-NEXT: fcvtzs v3.2d, v4.2d
+; CHECK-GI-NEXT: ret
%x = call <8 x i64> @llvm.fptosi.sat.v8f16.v8i64(<8 x half> %f)
ret <8 x i64> %x
}
diff --git a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
index 5a66b68af8e96..44e6e9415263b 100644
--- a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
@@ -2506,30 +2506,14 @@ define <4 x i64> @test_unsigned_v4f16_v4i64(<4 x half> %f) {
; CHECK-SD-FP16-NEXT: mov v1.d[1], x11
; CHECK-SD-FP16-NEXT: ret
;
-; CHECK-GI-CVT-LABEL: test_unsigned_v4f16_v4i64:
-; CHECK-GI-CVT: // %bb.0:
-; CHECK-GI-CVT-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-GI-CVT-NEXT: fcvtl v1.2d, v0.2s
-; CHECK-GI-CVT-NEXT: fcvtl2 v2.2d, v0.4s
-; CHECK-GI-CVT-NEXT: fcvtzu v0.2d, v1.2d
-; CHECK-GI-CVT-NEXT: fcvtzu v1.2d, v2.2d
-; CHECK-GI-CVT-NEXT: ret
-;
-; CHECK-GI-FP16-LABEL: test_unsigned_v4f16_v4i64:
-; CHECK-GI-FP16: // %bb.0:
-; CHECK-GI-FP16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT: mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT: mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT: mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT: fcvt d0, h0
-; CHECK-GI-FP16-NEXT: fcvt d1, h1
-; CHECK-GI-FP16-NEXT: fcvt d2, h2
-; CHECK-GI-FP16-NEXT: fcvt d3, h3
-; CHECK-GI-FP16-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT: mov v2.d[1], v3.d[0]
-; CHECK-GI-FP16-NEXT: fcvtzu v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT: fcvtzu v1.2d, v2.2d
-; CHECK-GI-FP16-NEXT: ret
+; CHECK-GI-LABEL: test_unsigned_v4f16_v4i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-GI-NEXT: fcvtl v1.2d, v0.2s
+; CHECK-GI-NEXT: fcvtl2 v2.2d, v0.4s
+; CHECK-GI-NEXT: fcvtzu v0.2d, v1.2d
+; CHECK-GI-NEXT: fcvtzu v1.2d, v2.2d
+; CHECK-GI-NEXT: ret
%x = call <4 x i64> @llvm.fptoui.sat.v4f16.v4i64(<4 x half> %f)
ret <4 x i64> %x
}
@@ -3114,46 +3098,19 @@ define <8 x i64> @test_unsigned_v8f16_v8i64(<8 x half> %f) {
; CHECK-SD-FP16-NEXT: mov v3.d[1], x14
; CHECK-SD-FP16-NEXT: ret
;
-; CHECK-GI-CVT-LABEL: test_unsigned_v8f16_v8i64:
-; CHECK-GI-CVT: // %bb.0:
-; CHECK-GI-CVT-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-GI-CVT-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-GI-CVT-NEXT: fcvtl v2.2d, v1.2s
-; CHECK-GI-CVT-NEXT: fcvtl2 v1.2d, v1.4s
-; CHECK-GI-CVT-NEXT: fcvtl v3.2d, v0.2s
-; CHECK-GI-CVT-NEXT: fcvtl2 v4.2d, v0.4s
-; CHECK-GI-CVT-NEXT: fcvtzu v0.2d, v2.2d
-; CHECK-GI-CVT-NEXT: fcvtzu v1.2d, v1.2d
-; CHECK-GI-CVT-NEXT: fcvtzu v2.2d, v3.2d
-; CHECK-GI-CVT-NEXT: fcvtzu v3.2d, v4.2d
-; CHECK-GI-CVT-NEXT: ret
-;
-; CHECK-GI-FP16-LABEL: test_unsigned_v8f16_v8i64:
-; CHECK-GI-FP16: // %bb.0:
-; CHECK-GI-FP16-NEXT: mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT: mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT: mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT: mov h4, v0.h[4]
-; CHECK-GI-FP16-NEXT: mov h5, v0.h[5]
-; CHECK-GI-FP16-NEXT: mov h6, v0.h[6]
-; CHECK-GI-FP16-NEXT: mov h7, v0.h[7]
-; CHECK-GI-FP16-NEXT: fcvt d0, h0
-; CHECK-GI-FP16-NEXT: fcvt d1, h1
-; CHECK-GI-FP16-NEXT: fcvt d2, h2
-; CHECK-GI-FP16-NEXT: fcvt d3, h3
-; CHECK-GI-FP16-NEXT: fcvt d4, h4
-; CHECK-GI-FP16-NEXT: fcvt d5, h5
-; CHECK-GI-FP16-NEXT: fcvt d6, h6
-; CHECK-GI-FP16-NEXT: fcvt d7, h7
-; CHECK-GI-FP16-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT: mov v2.d[1], v3.d[0]
-; CHECK-GI-FP16-NEXT: mov v4.d[1], v5.d[0]
-; CHECK-GI-FP16-NEXT: mov v6.d[1], v7.d[0]
-; CHECK-GI-FP16-NEXT: fcvtzu v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT: fcvtzu v1.2d, v2.2d
-; CHECK-GI-FP16-NEXT: fcvtzu v2.2d, v4.2d
-; CHECK-GI-FP16-NEXT: fcvtzu v3.2d, v6.2d
-; CHECK-GI-FP16-NEXT: ret
+; CHECK-GI-LABEL: test_unsigned_v8f16_v8i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: fcvtl v1.4s, v0.4h
+; CHECK-GI-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NEXT: fcvtl v2.2d, v1.2s
+; CHECK-GI-NEXT: fcvtl2 v1.2d, v1.4s
+; CHECK-GI-NEXT: fcvtl v3.2d, v0.2s
+; CHECK-GI-NEXT: fcvtl2 v4.2d, v0.4s
+; CHECK-GI-NEXT: fcvtzu v0.2d, v2.2d
+; CHECK-GI-NEXT: fcvtzu v1.2d, v1.2d
+; CHECK-GI-NEXT: fcvtzu v2.2d, v3.2d
+; CHECK-GI-NEXT: fcvtzu v3.2d, v4.2d
+; CHECK-GI-NEXT: ret
%x = call <8 x i64> @llvm.fptoui.sat.v8f16.v8i64(<8 x half> %f)
ret <8 x i64> %x
}
diff --git a/llvm/test/CodeGen/AArch64/fptrunc.ll b/llvm/test/CodeGen/AArch64/fptrunc.ll
index 1f84c944d7c16..de780bf475138 100644
--- a/llvm/test/CodeGen/AArch64/fptrunc.ll
+++ b/llvm/test/CodeGen/AArch64/fptrunc.ll
@@ -112,30 +112,22 @@ define <2 x half> @fptrunc_v2f128_v2f16(<2 x fp128> %a) {
;
; CHECK-GI-LABEL: fptrunc_v2f128_v2f16:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: sub sp, sp, #64
-; CHECK-GI-NEXT: str x30, [sp, #48] // 8-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 64
+; CHECK-GI-NEXT: sub sp, sp, #48
+; CHECK-GI-NEXT: str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 48
; CHECK-GI-NEXT: .cfi_offset w30, -16
-; CHECK-GI-NEXT: mov v2.d[0], x8
; CHECK-GI-NEXT: str q1, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT: mov v2.d[1], x8
-; CHECK-GI-NEXT: str q2, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT: bl __trunctfhf2
; CHECK-GI-NEXT: // kill: def $h0 killed $h0 def $q0
; CHECK-GI-NEXT: str q0, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT: bl __trunctfhf2
+; CHECK-GI-NEXT: ldr q1, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT: // kill: def $h0 killed $h0 def $q0
-; CHECK-GI-NEXT: str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT: ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: bl __trunctfhf2
-; CHECK-GI-NEXT: ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: bl __trunctfhf2
-; CHECK-GI-NEXT: ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-GI-NEXT: ldr x30, [sp, #48] // 8-byte Folded Reload
-; CHECK-GI-NEXT: mov v0.h[1], v1.h[0]
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: add sp, sp, #64
+; CHECK-GI-NEXT: ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-GI-NEXT: mov v1.h[1], v0.h[0]
+; CHECK-GI-NEXT: fmov d0, d1
+; CHECK-GI-NEXT: add sp, sp, #48
; CHECK-GI-NEXT: ret
entry:
%c = fptrunc <2 x fp128> %a to <2 x half>
@@ -260,8 +252,9 @@ define <3 x float> @fptrunc_v3f64_v3f32(<3 x double> %a) {
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT: fcvt s2, d2
+; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2
; CHECK-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT: fcvtn v2.2s, v2.2d
; CHECK-GI-NEXT: fcvtn v1.2s, v0.2d
; CHECK-GI-NEXT: mov v0.s[0], v1.s[0]
; CHECK-GI-NEXT: mov v0.s[1], v1.s[1]
@@ -284,61 +277,49 @@ entry:
}
define <2 x half> @fptrunc_v2f64_v2f16(<2 x double> %a) {
-; CHECK-SD-LABEL: fptrunc_v2f64_v2f16:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-SD-NEXT: fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: fptrunc_v2f64_v2f16:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov d1, v0.d[1]
-; CHECK-GI-NEXT: fcvt h0, d0
-; CHECK-GI-NEXT: fcvt h1, d1
-; CHECK-GI-NEXT: mov v0.h[1], v1.h[0]
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: fptrunc_v2f64_v2f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-NEXT: ret
entry:
%c = fptrunc <2 x double> %a to <2 x half>
ret <2 x half> %c
}
define <3 x half> @fptrunc_v3f64_v3f16(<3 x double> %a) {
-; CHECK-LABEL: fptrunc_v3f64_v3f16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: fcvt h0, d0
-; CHECK-NEXT: fcvt h1, d1
-; CHECK-NEXT: fcvt h2, d2
-; CHECK-NEXT: mov v0.h[1], v1.h[0]
-; CHECK-NEXT: mov v0.h[2], v2.h[0]
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: fptrunc_v3f64_v3f16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: fcvt h0, d0
+; CHECK-SD-NEXT: fcvt h1, d1
+; CHECK-SD-NEXT: fcvt h2, d2
+; CHECK-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-SD-NEXT: mov v0.h[2], v2.h[0]
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: fptrunc_v3f64_v3f16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-GI-NEXT: fcvtxn2 v0.4s, v2.2d
+; CHECK-GI-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-GI-NEXT: ret
entry:
%c = fptrunc <3 x double> %a to <3 x half>
ret <3 x half> %c
}
define <4 x half> @fptrunc_v4f64_v4f16(<4 x double> %a) {
-; CHECK-SD-LABEL: fptrunc_v4f64_v4f16:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-SD-NEXT: fcvtxn2 v0.4s, v1.2d
-; CHECK-SD-NEXT: fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: fptrunc_v4f64_v4f16:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov d2, v0.d[1]
-; CHECK-GI-NEXT: fcvt h0, d0
-; CHECK-GI-NEXT: mov d3, v1.d[1]
-; CHECK-GI-NEXT: fcvt h1, d1
-; CHECK-GI-NEXT: fcvt h2, d2
-; CHECK-GI-NEXT: mov v0.h[1], v2.h[0]
-; CHECK-GI-NEXT: fcvt h2, d3
-; CHECK-GI-NEXT: mov v0.h[2], v1.h[0]
-; CHECK-GI-NEXT: mov v0.h[3], v2.h[0]
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: fptrunc_v4f64_v4f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-NEXT: fcvtxn2 v0.4s, v1.2d
+; CHECK-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-NEXT: ret
entry:
%c = fptrunc <4 x double> %a to <4 x half>
ret <4 x half> %c
>From 040146e44d8e34fdc453fe6034edd96f16c488ed Mon Sep 17 00:00:00 2001
From: Ryan Cowan <ryan.cowan at arm.com>
Date: Fri, 17 Oct 2025 13:04:54 +0000
Subject: [PATCH 02/12] Address review comments
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 14 ++++----
.../AArch64/GISel/AArch64LegalizerInfo.h | 4 +--
.../GISel/AArch64PostLegalizerLowering.cpp | 32 +++++++++----------
3 files changed, 23 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 9153694817676..e188e32b87e04 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1483,11 +1483,10 @@ bool AArch64LegalizerInfo::legalizeCustom(
case TargetOpcode::G_BITCAST:
return legalizeBitcast(MI, Helper);
case TargetOpcode::G_FPEXT:
+ case TargetOpcode::G_FPTRUNC:
// In order to vectorise f16 to f64 properly, we need to use f32 as an
// intermediary
- return legalizeViaF32(MI, MIRBuilder, MRI, TargetOpcode::G_FPEXT);
- case TargetOpcode::G_FPTRUNC:
- return legalizeViaF32(MI, MIRBuilder, MRI, TargetOpcode::G_FPTRUNC);
+ return legalizeFpextFptrunc(MI, MIRBuilder, MRI);
}
llvm_unreachable("expected switch to return");
@@ -2415,10 +2414,9 @@ bool AArch64LegalizerInfo::legalizePrefetch(MachineInstr &MI,
return true;
}
-bool AArch64LegalizerInfo::legalizeViaF32(MachineInstr &MI,
- MachineIRBuilder &MIRBuilder,
- MachineRegisterInfo &MRI,
- unsigned Opcode) const {
+bool AArch64LegalizerInfo::legalizeFpextFptrunc(
+ MachineInstr &MI, MachineIRBuilder &MIRBuilder,
+ MachineRegisterInfo &MRI) const {
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
LLT DstTy = MRI.getType(Dst);
@@ -2429,7 +2427,7 @@ bool AArch64LegalizerInfo::legalizeViaF32(MachineInstr &MI,
MachineInstrBuilder Mid;
MachineInstrBuilder Fin;
MIRBuilder.setInstrAndDebugLoc(MI);
- switch (Opcode) {
+ switch (MI.getOpcode()) {
default:
return false;
case TargetOpcode::G_FPEXT: {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
index 049808d66f983..15999a44174d3 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
@@ -67,8 +67,8 @@ class AArch64LegalizerInfo : public LegalizerInfo {
bool legalizeDynStackAlloc(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizePrefetch(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizeBitcast(MachineInstr &MI, LegalizerHelper &Helper) const;
- bool legalizeViaF32(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
- MachineRegisterInfo &MRI, unsigned Opcode) const;
+ bool legalizeFpextFptrunc(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
+ MachineRegisterInfo &MRI) const;
const AArch64Subtarget *ST;
};
} // End llvm namespace.
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
index e675fac0f13ac..2fa1b86a8a9c6 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
@@ -904,7 +904,7 @@ unsigned getCmpOperandFoldingProfit(Register CmpOp, MachineRegisterInfo &MRI) {
// Helper function for matchFpTruncFpTrunc.
// Checks that the given definition belongs to an FPTRUNC and that the source is
// not an integer, as no rounding is necessary due to the range of values
-bool checkTruncSrc(MachineRegisterInfo &MRI, MachineInstr *MaybeFpTrunc) {
+bool isFPTruncFromDouble(MachineRegisterInfo &MRI, MachineInstr *MaybeFpTrunc) {
if (!MaybeFpTrunc || MaybeFpTrunc->getOpcode() != TargetOpcode::G_FPTRUNC)
return false;
@@ -930,8 +930,7 @@ bool checkTruncSrc(MachineRegisterInfo &MRI, MachineInstr *MaybeFpTrunc) {
// truncating an FP that came from an integer this is not a problem as the range
// of values is lower in the int
bool matchFpTruncFpTrunc(MachineInstr &MI, MachineRegisterInfo &MRI) {
- if (MI.getOpcode() != TargetOpcode::G_FPTRUNC)
- return false;
+ assert(MI.getOpcode() == TargetOpcode::G_FPTRUNC && "Expected G_FPTRUNC");
// Check the destination is 16 bits as we only want to match a very specific
// pattern
@@ -959,10 +958,9 @@ bool matchFpTruncFpTrunc(MachineInstr &MI, MachineRegisterInfo &MRI) {
for (unsigned OpIdx = 1, NumOperands = ParentDef->getNumOperands();
OpIdx != NumOperands; ++OpIdx) {
Register FpTruncDst = ParentDef->getOperand(OpIdx).getReg();
-
FpTruncDef = getDefIgnoringCopies(FpTruncDst, MRI);
- if (!checkTruncSrc(MRI, FpTruncDef))
+ if (!isFPTruncFromDouble(MRI, FpTruncDef))
return false;
}
@@ -973,41 +971,43 @@ bool matchFpTruncFpTrunc(MachineInstr &MI, MachineRegisterInfo &MRI) {
Register VecExtractDst = ParentDef->getOperand(2).getReg();
MachineInstr *VecExtractDef = getDefIgnoringCopies(VecExtractDst, MRI);
+ if (!VecExtractDef ||
+ VecExtractDef->getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
+ return false;
+
Register FpTruncDst = VecExtractDef->getOperand(1).getReg();
FpTruncDef = getDefIgnoringCopies(FpTruncDst, MRI);
-
- if (!checkTruncSrc(MRI, FpTruncDef))
- return false;
break;
}
case TargetOpcode::G_FPTRUNC: {
Register FpTruncDst = ParentDef->getOperand(1).getReg();
FpTruncDef = getDefIgnoringCopies(FpTruncDst, MRI);
-
- if (!checkTruncSrc(MRI, FpTruncDef))
- return false;
break;
}
}
+ if (!isFPTruncFromDouble(MRI, FpTruncDef))
+ return false;
+
return true;
}
void applyFpTruncFpTrunc(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) {
+ assert(MI.getOpcode() == TargetOpcode::G_FPTRUNC && "Expected G_FPTRUNC");
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
+ MachineInstr *ParentDef = getDefIgnoringCopies(Src, MRI);
+ if (!ParentDef)
+ return;
+
LLT V2F32 = LLT::fixed_vector(2, LLT::scalar(32));
LLT V4F32 = LLT::fixed_vector(4, LLT::scalar(32));
LLT V4F16 = LLT::fixed_vector(4, LLT::scalar(16));
B.setInstrAndDebugLoc(MI);
- MachineInstr *ParentDef = getDefIgnoringCopies(Src, MRI);
- if (!ParentDef)
- return;
-
switch (ParentDef->getOpcode()) {
default:
return;
@@ -1056,8 +1056,6 @@ void applyFpTruncFpTrunc(MachineInstr &MI, MachineRegisterInfo &MRI,
Register HiFp64 = FpTrunc2Def->getOperand(1).getReg();
MRI.setRegClass(HiFp64, &AArch64::FPR128RegClass);
- B.setInstrAndDebugLoc(MI);
-
// Convert the lower half
Register LoFp32 = MRI.createGenericVirtualRegister(V2F32);
MRI.setRegClass(LoFp32, &AArch64::FPR64RegClass);
>From ae3c7ae413152b787482e153cea432912ca61666 Mon Sep 17 00:00:00 2001
From: Ryan Cowan <ryan.cowan at arm.com>
Date: Mon, 20 Oct 2025 09:35:46 +0000
Subject: [PATCH 03/12] Separate FPEXT & FPTRUNC changes
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 18 +-
.../AArch64/GISel/AArch64LegalizerInfo.h | 2 +-
llvm/test/CodeGen/AArch64/fmla.ll | 48 +--
.../CodeGen/AArch64/fp16-v4-instructions.ll | 26 +-
.../CodeGen/AArch64/fp16-v8-instructions.ll | 50 +++-
llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll | 186 +++++++-----
llvm/test/CodeGen/AArch64/fpext.ll | 49 +--
llvm/test/CodeGen/AArch64/fptoi.ll | 278 ++++++++++++------
.../test/CodeGen/AArch64/fptosi-sat-vector.ll | 85 ++++--
.../test/CodeGen/AArch64/fptoui-sat-vector.ll | 85 ++++--
10 files changed, 553 insertions(+), 274 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index e188e32b87e04..4f3c8ab2e62e6 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -835,14 +835,6 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
.legalFor(
{{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}})
.libcallFor({{s128, s64}, {s128, s32}, {s128, s16}})
- .moreElementsToNextPow2(0)
- .customIf([](const LegalityQuery &Q) {
- LLT DstTy = Q.Types[0];
- LLT SrcTy = Q.Types[1];
- return SrcTy.isVector() && DstTy.isVector() &&
- SrcTy.getScalarSizeInBits() == 16 &&
- DstTy.getScalarSizeInBits() == 64;
- })
.clampNumElements(0, v4s32, v4s32)
.clampNumElements(0, v2s64, v2s64)
.scalarize(0);
@@ -1482,11 +1474,10 @@ bool AArch64LegalizerInfo::legalizeCustom(
return legalizeICMP(MI, MRI, MIRBuilder);
case TargetOpcode::G_BITCAST:
return legalizeBitcast(MI, Helper);
- case TargetOpcode::G_FPEXT:
case TargetOpcode::G_FPTRUNC:
// In order to vectorise f16 to f64 properly, we need to use f32 as an
// intermediary
- return legalizeFpextFptrunc(MI, MIRBuilder, MRI);
+ return legalizeFptrunc(MI, MIRBuilder, MRI);
}
llvm_unreachable("expected switch to return");
@@ -2414,7 +2405,7 @@ bool AArch64LegalizerInfo::legalizePrefetch(MachineInstr &MI,
return true;
}
-bool AArch64LegalizerInfo::legalizeFpextFptrunc(
+bool AArch64LegalizerInfo::legalizeFptrunc(
MachineInstr &MI, MachineIRBuilder &MIRBuilder,
MachineRegisterInfo &MRI) const {
Register Dst = MI.getOperand(0).getReg();
@@ -2430,11 +2421,6 @@ bool AArch64LegalizerInfo::legalizeFpextFptrunc(
switch (MI.getOpcode()) {
default:
return false;
- case TargetOpcode::G_FPEXT: {
- Mid = MIRBuilder.buildFPExt(MidTy, Src);
- Fin = MIRBuilder.buildFPExt(DstTy, Mid.getReg(0));
- break;
- }
case TargetOpcode::G_FPTRUNC: {
Mid = MIRBuilder.buildFPTrunc(MidTy, Src);
Fin = MIRBuilder.buildFPTrunc(DstTy, Mid.getReg(0));
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
index 15999a44174d3..630f31fd24f28 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
@@ -67,7 +67,7 @@ class AArch64LegalizerInfo : public LegalizerInfo {
bool legalizeDynStackAlloc(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizePrefetch(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizeBitcast(MachineInstr &MI, LegalizerHelper &Helper) const;
- bool legalizeFpextFptrunc(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
+ bool legalizeFptrunc(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
MachineRegisterInfo &MRI) const;
const AArch64Subtarget *ST;
};
diff --git a/llvm/test/CodeGen/AArch64/fmla.ll b/llvm/test/CodeGen/AArch64/fmla.ll
index 12b6562b5cf0c..a37aabb0b5384 100644
--- a/llvm/test/CodeGen/AArch64/fmla.ll
+++ b/llvm/test/CodeGen/AArch64/fmla.ll
@@ -865,22 +865,22 @@ define <7 x half> @fmuladd_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v3.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v2.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v5.4h
+; CHECK-GI-NOFP16-NEXT: mov v5.h[0], v2.h[4]
; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v4.4h
; CHECK-GI-NOFP16-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT: mov v1.h[0], v2.h[4]
-; CHECK-GI-NOFP16-NEXT: fmul v3.4s, v3.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT: mov v1.h[1], v2.h[5]
-; CHECK-GI-NOFP16-NEXT: fcvtn v4.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v3.4s
-; CHECK-GI-NOFP16-NEXT: mov v1.h[2], v2.h[6]
-; CHECK-GI-NOFP16-NEXT: mov v0.h[0], v4.h[0]
-; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT: mov v5.h[1], v2.h[5]
+; CHECK-GI-NOFP16-NEXT: fmul v1.4s, v3.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT: mov v5.h[2], v2.h[6]
+; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v1.4s
+; CHECK-GI-NOFP16-NEXT: mov v0.h[0], v3.h[0]
+; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v5.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT: mov v0.h[1], v4.h[1]
-; CHECK-GI-NOFP16-NEXT: fadd v1.4s, v2.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT: mov v0.h[2], v4.h[2]
+; CHECK-GI-NOFP16-NEXT: mov v0.h[1], v3.h[1]
+; CHECK-GI-NOFP16-NEXT: fadd v1.4s, v1.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT: mov v0.h[2], v3.h[2]
; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT: mov v0.h[3], v4.h[3]
+; CHECK-GI-NOFP16-NEXT: mov v0.h[3], v3.h[3]
; CHECK-GI-NOFP16-NEXT: mov v0.h[4], v1.h[0]
; CHECK-GI-NOFP16-NEXT: mov v0.h[5], v1.h[1]
; CHECK-GI-NOFP16-NEXT: mov v0.h[6], v1.h[2]
@@ -1350,22 +1350,22 @@ define <7 x half> @fmul_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v3.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v2.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v5.4h
+; CHECK-GI-NOFP16-NEXT: mov v5.h[0], v2.h[4]
; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v4.4h
; CHECK-GI-NOFP16-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT: mov v1.h[0], v2.h[4]
-; CHECK-GI-NOFP16-NEXT: fmul v3.4s, v3.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT: mov v1.h[1], v2.h[5]
-; CHECK-GI-NOFP16-NEXT: fcvtn v4.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v3.4s
-; CHECK-GI-NOFP16-NEXT: mov v1.h[2], v2.h[6]
-; CHECK-GI-NOFP16-NEXT: mov v0.h[0], v4.h[0]
-; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT: mov v5.h[1], v2.h[5]
+; CHECK-GI-NOFP16-NEXT: fmul v1.4s, v3.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT: mov v5.h[2], v2.h[6]
+; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v1.4s
+; CHECK-GI-NOFP16-NEXT: mov v0.h[0], v3.h[0]
+; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v5.4h
; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT: mov v0.h[1], v4.h[1]
-; CHECK-GI-NOFP16-NEXT: fadd v1.4s, v2.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT: mov v0.h[2], v4.h[2]
+; CHECK-GI-NOFP16-NEXT: mov v0.h[1], v3.h[1]
+; CHECK-GI-NOFP16-NEXT: fadd v1.4s, v1.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT: mov v0.h[2], v3.h[2]
; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT: mov v0.h[3], v4.h[3]
+; CHECK-GI-NOFP16-NEXT: mov v0.h[3], v3.h[3]
; CHECK-GI-NOFP16-NEXT: mov v0.h[4], v1.h[0]
; CHECK-GI-NOFP16-NEXT: mov v0.h[5], v1.h[1]
; CHECK-GI-NOFP16-NEXT: mov v0.h[6], v1.h[2]
diff --git a/llvm/test/CodeGen/AArch64/fp16-v4-instructions.ll b/llvm/test/CodeGen/AArch64/fp16-v4-instructions.ll
index 760742a4efad7..104fca4ef3989 100644
--- a/llvm/test/CodeGen/AArch64/fp16-v4-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/fp16-v4-instructions.ll
@@ -206,16 +206,30 @@ define <4 x double> @h_to_d(<4 x half> %a) {
;
; CHECK-CVT-GI-LABEL: h_to_d:
; CHECK-CVT-GI: // %bb.0:
-; CHECK-CVT-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-CVT-GI-NEXT: fcvtl v0.2d, v1.2s
-; CHECK-CVT-GI-NEXT: fcvtl2 v1.2d, v1.4s
+; CHECK-CVT-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-GI-NEXT: mov h1, v0.h[1]
+; CHECK-CVT-GI-NEXT: mov h2, v0.h[2]
+; CHECK-CVT-GI-NEXT: mov h3, v0.h[3]
+; CHECK-CVT-GI-NEXT: fcvt d0, h0
+; CHECK-CVT-GI-NEXT: fcvt d4, h1
+; CHECK-CVT-GI-NEXT: fcvt d1, h2
+; CHECK-CVT-GI-NEXT: fcvt d2, h3
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v4.d[0]
+; CHECK-CVT-GI-NEXT: mov v1.d[1], v2.d[0]
; CHECK-CVT-GI-NEXT: ret
;
; CHECK-FP16-GI-LABEL: h_to_d:
; CHECK-FP16-GI: // %bb.0:
-; CHECK-FP16-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl v0.2d, v1.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v1.2d, v1.4s
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d4, h1
+; CHECK-FP16-GI-NEXT: fcvt d1, h2
+; CHECK-FP16-GI-NEXT: fcvt d2, h3
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v4.d[0]
+; CHECK-FP16-GI-NEXT: mov v1.d[1], v2.d[0]
; CHECK-FP16-GI-NEXT: ret
%1 = fpext <4 x half> %a to <4 x double>
ret <4 x double> %1
diff --git a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
index 4d8505679c71c..78db49613f1e6 100644
--- a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
@@ -272,22 +272,48 @@ define <8 x double> @h_to_d(<8 x half> %a) {
;
; CHECK-CVT-GI-LABEL: h_to_d:
; CHECK-CVT-GI: // %bb.0:
-; CHECK-CVT-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-CVT-GI-NEXT: fcvtl2 v3.4s, v0.8h
-; CHECK-CVT-GI-NEXT: fcvtl v0.2d, v1.2s
-; CHECK-CVT-GI-NEXT: fcvtl2 v1.2d, v1.4s
-; CHECK-CVT-GI-NEXT: fcvtl v2.2d, v3.2s
-; CHECK-CVT-GI-NEXT: fcvtl2 v3.2d, v3.4s
+; CHECK-CVT-GI-NEXT: mov h1, v0.h[1]
+; CHECK-CVT-GI-NEXT: mov h2, v0.h[2]
+; CHECK-CVT-GI-NEXT: mov h3, v0.h[3]
+; CHECK-CVT-GI-NEXT: mov h4, v0.h[4]
+; CHECK-CVT-GI-NEXT: mov h5, v0.h[5]
+; CHECK-CVT-GI-NEXT: mov h6, v0.h[6]
+; CHECK-CVT-GI-NEXT: mov h7, v0.h[7]
+; CHECK-CVT-GI-NEXT: fcvt d0, h0
+; CHECK-CVT-GI-NEXT: fcvt d16, h1
+; CHECK-CVT-GI-NEXT: fcvt d1, h2
+; CHECK-CVT-GI-NEXT: fcvt d17, h3
+; CHECK-CVT-GI-NEXT: fcvt d2, h4
+; CHECK-CVT-GI-NEXT: fcvt d4, h5
+; CHECK-CVT-GI-NEXT: fcvt d3, h6
+; CHECK-CVT-GI-NEXT: fcvt d5, h7
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v16.d[0]
+; CHECK-CVT-GI-NEXT: mov v1.d[1], v17.d[0]
+; CHECK-CVT-GI-NEXT: mov v2.d[1], v4.d[0]
+; CHECK-CVT-GI-NEXT: mov v3.d[1], v5.d[0]
; CHECK-CVT-GI-NEXT: ret
;
; CHECK-FP16-GI-LABEL: h_to_d:
; CHECK-FP16-GI: // %bb.0:
-; CHECK-FP16-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl2 v3.4s, v0.8h
-; CHECK-FP16-GI-NEXT: fcvtl v0.2d, v1.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v1.2d, v1.4s
-; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v3.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v3.2d, v3.4s
+; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT: mov h4, v0.h[4]
+; CHECK-FP16-GI-NEXT: mov h5, v0.h[5]
+; CHECK-FP16-GI-NEXT: mov h6, v0.h[6]
+; CHECK-FP16-GI-NEXT: mov h7, v0.h[7]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d16, h1
+; CHECK-FP16-GI-NEXT: fcvt d1, h2
+; CHECK-FP16-GI-NEXT: fcvt d17, h3
+; CHECK-FP16-GI-NEXT: fcvt d2, h4
+; CHECK-FP16-GI-NEXT: fcvt d4, h5
+; CHECK-FP16-GI-NEXT: fcvt d3, h6
+; CHECK-FP16-GI-NEXT: fcvt d5, h7
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v16.d[0]
+; CHECK-FP16-GI-NEXT: mov v1.d[1], v17.d[0]
+; CHECK-FP16-GI-NEXT: mov v2.d[1], v4.d[0]
+; CHECK-FP16-GI-NEXT: mov v3.d[1], v5.d[0]
; CHECK-FP16-GI-NEXT: ret
%1 = fpext <8 x half> %a to <8 x double>
ret <8 x double> %1
diff --git a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
index b075a8b6f70ee..637c02875b84e 100644
--- a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
@@ -285,24 +285,31 @@ define <4 x i32> @stest_f16i32(<4 x half> %x) {
;
; CHECK-FP16-GI-LABEL: stest_f16i32:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
; CHECK-FP16-GI-NEXT: adrp x8, .LCPI6_1
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d1, h1
+; CHECK-FP16-GI-NEXT: fcvt d2, h2
+; CHECK-FP16-GI-NEXT: fcvt d3, h3
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v2.2d
; CHECK-FP16-GI-NEXT: ldr q2, [x8, :lo12:.LCPI6_1]
; CHECK-FP16-GI-NEXT: adrp x8, .LCPI6_0
-; CHECK-FP16-GI-NEXT: fcvtl v1.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v1.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v2.2d, v1.2d
-; CHECK-FP16-GI-NEXT: cmgt v4.2d, v2.2d, v0.2d
-; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v2.2d, v0.2d
+; CHECK-FP16-GI-NEXT: cmgt v4.2d, v2.2d, v1.2d
+; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v4.16b
; CHECK-FP16-GI-NEXT: ldr q2, [x8, :lo12:.LCPI6_0]
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: cmgt v4.2d, v0.2d, v2.2d
-; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v4.16b
-; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT: cmgt v4.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%conv = fptosi <4 x half> %x to <4 x i64>
@@ -344,17 +351,24 @@ define <4 x i32> @utest_f16i32(<4 x half> %x) {
;
; CHECK-FP16-GI-LABEL: utest_f16i32:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT: mov h4, v0.h[3]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
; CHECK-FP16-GI-NEXT: movi v1.2d, #0x000000ffffffff
-; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT: fcvt d2, h2
+; CHECK-FP16-GI-NEXT: fcvt d3, h3
+; CHECK-FP16-GI-NEXT: fcvt d4, h4
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v2.d[0]
+; CHECK-FP16-GI-NEXT: mov v3.d[1], v4.d[0]
; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: cmhi v3.2d, v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: cmhi v4.2d, v1.2d, v0.2d
-; CHECK-FP16-GI-NEXT: bif v2.16b, v1.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v4.16b
-; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT: cmhi v3.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT: cmhi v4.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bit v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%conv = fptoui <4 x half> %x to <4 x i64>
@@ -398,21 +412,28 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) {
;
; CHECK-FP16-GI-LABEL: ustest_f16i32:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT: mov h4, v0.h[3]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
; CHECK-FP16-GI-NEXT: movi v1.2d, #0x000000ffffffff
-; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT: fcvt d2, h2
+; CHECK-FP16-GI-NEXT: fcvt d3, h3
+; CHECK-FP16-GI-NEXT: fcvt d4, h4
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v2.d[0]
+; CHECK-FP16-GI-NEXT: mov v3.d[1], v4.d[0]
; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: cmgt v4.2d, v1.2d, v0.2d
-; CHECK-FP16-GI-NEXT: bif v2.16b, v1.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v4.16b
-; CHECK-FP16-GI-NEXT: cmgt v1.2d, v2.2d, #0
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v0.2d, #0
-; CHECK-FP16-GI-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-FP16-GI-NEXT: and v0.16b, v0.16b, v3.16b
-; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT: cmgt v4.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bit v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT: cmgt v2.2d, v0.2d, #0
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, #0
+; CHECK-FP16-GI-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-FP16-GI-NEXT: and v1.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%conv = fptosi <4 x half> %x to <4 x i64>
@@ -2252,24 +2273,31 @@ define <4 x i32> @stest_f16i32_mm(<4 x half> %x) {
;
; CHECK-FP16-GI-LABEL: stest_f16i32_mm:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
; CHECK-FP16-GI-NEXT: adrp x8, .LCPI33_1
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d1, h1
+; CHECK-FP16-GI-NEXT: fcvt d2, h2
+; CHECK-FP16-GI-NEXT: fcvt d3, h3
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v2.2d
; CHECK-FP16-GI-NEXT: ldr q2, [x8, :lo12:.LCPI33_1]
; CHECK-FP16-GI-NEXT: adrp x8, .LCPI33_0
-; CHECK-FP16-GI-NEXT: fcvtl v1.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v1.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v2.2d, v1.2d
-; CHECK-FP16-GI-NEXT: cmgt v4.2d, v2.2d, v0.2d
-; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v2.2d, v0.2d
+; CHECK-FP16-GI-NEXT: cmgt v4.2d, v2.2d, v1.2d
+; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v4.16b
; CHECK-FP16-GI-NEXT: ldr q2, [x8, :lo12:.LCPI33_0]
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: cmgt v4.2d, v0.2d, v2.2d
-; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v4.16b
-; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT: cmgt v4.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: bif v0.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bif v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%conv = fptosi <4 x half> %x to <4 x i64>
@@ -2309,17 +2337,24 @@ define <4 x i32> @utest_f16i32_mm(<4 x half> %x) {
;
; CHECK-FP16-GI-LABEL: utest_f16i32_mm:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT: mov h4, v0.h[3]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
; CHECK-FP16-GI-NEXT: movi v1.2d, #0x000000ffffffff
-; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT: fcvt d2, h2
+; CHECK-FP16-GI-NEXT: fcvt d3, h3
+; CHECK-FP16-GI-NEXT: fcvt d4, h4
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v2.d[0]
+; CHECK-FP16-GI-NEXT: mov v3.d[1], v4.d[0]
; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: cmhi v3.2d, v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: cmhi v4.2d, v1.2d, v0.2d
-; CHECK-FP16-GI-NEXT: bif v2.16b, v1.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v4.16b
-; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT: cmhi v3.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT: cmhi v4.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bit v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%conv = fptoui <4 x half> %x to <4 x i64>
@@ -2362,21 +2397,28 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) {
;
; CHECK-FP16-GI-LABEL: ustest_f16i32_mm:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT: mov h4, v0.h[3]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
; CHECK-FP16-GI-NEXT: movi v1.2d, #0x000000ffffffff
-; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT: fcvt d2, h2
+; CHECK-FP16-GI-NEXT: fcvt d3, h3
+; CHECK-FP16-GI-NEXT: fcvt d4, h4
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v2.d[0]
+; CHECK-FP16-GI-NEXT: mov v3.d[1], v4.d[0]
; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: cmgt v4.2d, v1.2d, v0.2d
-; CHECK-FP16-GI-NEXT: bif v2.16b, v1.16b, v3.16b
-; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v4.16b
-; CHECK-FP16-GI-NEXT: cmgt v1.2d, v2.2d, #0
-; CHECK-FP16-GI-NEXT: cmgt v3.2d, v0.2d, #0
-; CHECK-FP16-GI-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-FP16-GI-NEXT: and v0.16b, v0.16b, v3.16b
-; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT: cmgt v4.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT: bit v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT: cmgt v2.2d, v0.2d, #0
+; CHECK-FP16-GI-NEXT: cmgt v3.2d, v1.2d, #0
+; CHECK-FP16-GI-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-FP16-GI-NEXT: and v1.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%conv = fptosi <4 x half> %x to <4 x i64>
diff --git a/llvm/test/CodeGen/AArch64/fpext.ll b/llvm/test/CodeGen/AArch64/fpext.ll
index 8980340a447de..df90f9d5f0910 100644
--- a/llvm/test/CodeGen/AArch64/fpext.ll
+++ b/llvm/test/CodeGen/AArch64/fpext.ll
@@ -82,12 +82,11 @@ define <3 x double> @fpext_v3f32_v3f64(<3 x float> %a) {
;
; CHECK-GI-LABEL: fpext_v3f32_v3f64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: mov v1.s[0], v0.s[2]
+; CHECK-GI-NEXT: mov s1, v0.s[2]
; CHECK-GI-NEXT: fcvtl v0.2d, v0.2s
-; CHECK-GI-NEXT: fcvtl v2.2d, v1.2s
+; CHECK-GI-NEXT: fcvt d2, s1
; CHECK-GI-NEXT: mov d1, v0.d[1]
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: // kill: def $d2 killed $d2 killed $q2
; CHECK-GI-NEXT: ret
entry:
%c = fpext <3 x float> %a to <3 x double>
@@ -321,11 +320,20 @@ entry:
}
define <2 x double> @fpext_v2f16_v2f64(<2 x half> %a) {
-; CHECK-LABEL: fpext_v2f16_v2f64:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-NEXT: fcvtl v0.2d, v0.2s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: fpext_v2f16_v2f64:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-SD-NEXT: fcvtl v0.2d, v0.2s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: fpext_v2f16_v2f64:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: mov h1, v0.h[1]
+; CHECK-GI-NEXT: fcvt d0, h0
+; CHECK-GI-NEXT: fcvt d1, h1
+; CHECK-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT: ret
entry:
%c = fpext <2 x half> %a to <2 x double>
ret <2 x double> %c
@@ -345,12 +353,12 @@ define <3 x double> @fpext_v3f16_v3f64(<3 x half> %a) {
;
; CHECK-GI-LABEL: fpext_v3f16_v3f64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-GI-NEXT: fcvtl v0.2d, v1.2s
-; CHECK-GI-NEXT: fcvtl2 v2.2d, v1.4s
-; CHECK-GI-NEXT: // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NEXT: mov d1, v0.d[1]
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: mov h1, v0.h[1]
+; CHECK-GI-NEXT: mov h2, v0.h[2]
+; CHECK-GI-NEXT: fcvt d0, h0
+; CHECK-GI-NEXT: fcvt d1, h1
+; CHECK-GI-NEXT: fcvt d2, h2
; CHECK-GI-NEXT: ret
entry:
%c = fpext <3 x half> %a to <3 x double>
@@ -367,9 +375,16 @@ define <4 x double> @fpext_v4f16_v4f64(<4 x half> %a) {
;
; CHECK-GI-LABEL: fpext_v4f16_v4f64:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-GI-NEXT: fcvtl v0.2d, v1.2s
-; CHECK-GI-NEXT: fcvtl2 v1.2d, v1.4s
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: mov h1, v0.h[1]
+; CHECK-GI-NEXT: mov h2, v0.h[2]
+; CHECK-GI-NEXT: mov h3, v0.h[3]
+; CHECK-GI-NEXT: fcvt d0, h0
+; CHECK-GI-NEXT: fcvt d4, h1
+; CHECK-GI-NEXT: fcvt d1, h2
+; CHECK-GI-NEXT: fcvt d2, h3
+; CHECK-GI-NEXT: mov v0.d[1], v4.d[0]
+; CHECK-GI-NEXT: mov v1.d[1], v2.d[0]
; CHECK-GI-NEXT: ret
entry:
%c = fpext <4 x half> %a to <4 x double>
diff --git a/llvm/test/CodeGen/AArch64/fptoi.ll b/llvm/test/CodeGen/AArch64/fptoi.ll
index 3dafabe0b69d7..f6053cee50dae 100644
--- a/llvm/test/CodeGen/AArch64/fptoi.ll
+++ b/llvm/test/CodeGen/AArch64/fptoi.ll
@@ -4610,8 +4610,11 @@ define <2 x i64> @fptos_v2f16_v2i64(<2 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptos_v2f16_v2i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl v0.2d, v0.2s
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d1, h1
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
; CHECK-FP16-GI-NEXT: ret
entry:
@@ -4651,8 +4654,11 @@ define <2 x i64> @fptou_v2f16_v2i64(<2 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptou_v2f16_v2i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl v0.2d, v0.2s
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d1, h1
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v0.2d
; CHECK-FP16-GI-NEXT: ret
entry:
@@ -4704,14 +4710,20 @@ define <3 x i64> @fptos_v3f16_v3i64(<3 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptos_v3f16_v3i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl v1.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v2.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v1.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v2.2d
-; CHECK-FP16-GI-NEXT: // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT: fcvt d1, h0
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d2, h2
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v2.d[0]
+; CHECK-FP16-GI-NEXT: fcvt d2, h3
+; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: mov v2.d[1], v1.d[0]
; CHECK-FP16-GI-NEXT: mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v2.2d
; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT: // kill: def $d2 killed $d2 killed $q2
; CHECK-FP16-GI-NEXT: ret
entry:
%c = fptosi <3 x half> %a to <3 x i64>
@@ -4762,14 +4774,20 @@ define <3 x i64> @fptou_v3f16_v3i64(<3 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptou_v3f16_v3i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl v1.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v2.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v1.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v2.2d
-; CHECK-FP16-GI-NEXT: // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT: fcvt d1, h0
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d2, h2
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v2.d[0]
+; CHECK-FP16-GI-NEXT: fcvt d2, h3
+; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: mov v2.d[1], v1.d[0]
; CHECK-FP16-GI-NEXT: mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v2.2d
; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT: // kill: def $d2 killed $d2 killed $q2
; CHECK-FP16-GI-NEXT: ret
entry:
%c = fptoui <3 x half> %a to <3 x i64>
@@ -4824,10 +4842,17 @@ define <4 x i64> @fptos_v4f16_v4i64(<4 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptos_v4f16_v4i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl v1.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v2.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v1.2d
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d1, h1
+; CHECK-FP16-GI-NEXT: fcvt d2, h2
+; CHECK-FP16-GI-NEXT: fcvt d3, h3
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v2.2d
; CHECK-FP16-GI-NEXT: ret
entry:
@@ -4883,10 +4908,17 @@ define <4 x i64> @fptou_v4f16_v4i64(<4 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptou_v4f16_v4i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl v1.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v2.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v1.2d
+; CHECK-FP16-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d1, h1
+; CHECK-FP16-GI-NEXT: fcvt d2, h2
+; CHECK-FP16-GI-NEXT: fcvt d3, h3
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v0.2d
; CHECK-FP16-GI-NEXT: fcvtzu v1.2d, v2.2d
; CHECK-FP16-GI-NEXT: ret
entry:
@@ -4973,16 +5005,29 @@ define <8 x i64> @fptos_v8f16_v8i64(<8 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptos_v8f16_v8i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v1.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v1.2d, v1.4s
-; CHECK-FP16-GI-NEXT: fcvtl v3.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v4.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v2.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v1.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v3.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v3.2d, v4.2d
+; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT: mov h4, v0.h[4]
+; CHECK-FP16-GI-NEXT: mov h5, v0.h[5]
+; CHECK-FP16-GI-NEXT: mov h6, v0.h[6]
+; CHECK-FP16-GI-NEXT: mov h7, v0.h[7]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d1, h1
+; CHECK-FP16-GI-NEXT: fcvt d2, h2
+; CHECK-FP16-GI-NEXT: fcvt d3, h3
+; CHECK-FP16-GI-NEXT: fcvt d4, h4
+; CHECK-FP16-GI-NEXT: fcvt d5, h5
+; CHECK-FP16-GI-NEXT: fcvt d6, h6
+; CHECK-FP16-GI-NEXT: fcvt d7, h7
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT: mov v4.d[1], v5.d[0]
+; CHECK-FP16-GI-NEXT: mov v6.d[1], v7.d[0]
+; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v4.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v3.2d, v6.2d
; CHECK-FP16-GI-NEXT: ret
entry:
%c = fptosi <8 x half> %a to <8 x i64>
@@ -5068,16 +5113,29 @@ define <8 x i64> @fptou_v8f16_v8i64(<8 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptou_v8f16_v8i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-FP16-GI-NEXT: fcvtl v2.2d, v1.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v1.2d, v1.4s
-; CHECK-FP16-GI-NEXT: fcvtl v3.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v4.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v2.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v1.2d, v1.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v3.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v3.2d, v4.2d
+; CHECK-FP16-GI-NEXT: mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT: mov h4, v0.h[4]
+; CHECK-FP16-GI-NEXT: mov h5, v0.h[5]
+; CHECK-FP16-GI-NEXT: mov h6, v0.h[6]
+; CHECK-FP16-GI-NEXT: mov h7, v0.h[7]
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d1, h1
+; CHECK-FP16-GI-NEXT: fcvt d2, h2
+; CHECK-FP16-GI-NEXT: fcvt d3, h3
+; CHECK-FP16-GI-NEXT: fcvt d4, h4
+; CHECK-FP16-GI-NEXT: fcvt d5, h5
+; CHECK-FP16-GI-NEXT: fcvt d6, h6
+; CHECK-FP16-GI-NEXT: fcvt d7, h7
+; CHECK-FP16-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT: mov v4.d[1], v5.d[0]
+; CHECK-FP16-GI-NEXT: mov v6.d[1], v7.d[0]
+; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v4.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v3.2d, v6.2d
; CHECK-FP16-GI-NEXT: ret
entry:
%c = fptoui <8 x half> %a to <8 x i64>
@@ -5227,26 +5285,52 @@ define <16 x i64> @fptos_v16f16_v16i64(<16 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptos_v16f16_v16i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v2.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-FP16-GI-NEXT: fcvtl v3.4s, v1.4h
-; CHECK-FP16-GI-NEXT: fcvtl2 v1.4s, v1.8h
-; CHECK-FP16-GI-NEXT: fcvtl v4.2d, v2.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v2.2d, v2.4s
-; CHECK-FP16-GI-NEXT: fcvtl v5.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v6.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtl v7.2d, v3.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v16.2d, v3.4s
-; CHECK-FP16-GI-NEXT: fcvtl v17.2d, v1.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v18.2d, v1.4s
-; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v4.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v5.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v3.2d, v6.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v4.2d, v7.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v5.2d, v16.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v6.2d, v17.2d
-; CHECK-FP16-GI-NEXT: fcvtzs v7.2d, v18.2d
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h4, v0.h[2]
+; CHECK-FP16-GI-NEXT: mov h5, v0.h[3]
+; CHECK-FP16-GI-NEXT: fcvt d2, h0
+; CHECK-FP16-GI-NEXT: mov h6, v0.h[4]
+; CHECK-FP16-GI-NEXT: mov h7, v0.h[5]
+; CHECK-FP16-GI-NEXT: mov h16, v0.h[6]
+; CHECK-FP16-GI-NEXT: mov h0, v0.h[7]
+; CHECK-FP16-GI-NEXT: mov h17, v1.h[1]
+; CHECK-FP16-GI-NEXT: mov h18, v1.h[2]
+; CHECK-FP16-GI-NEXT: mov h19, v1.h[3]
+; CHECK-FP16-GI-NEXT: mov h20, v1.h[4]
+; CHECK-FP16-GI-NEXT: mov h21, v1.h[5]
+; CHECK-FP16-GI-NEXT: mov h22, v1.h[6]
+; CHECK-FP16-GI-NEXT: mov h23, v1.h[7]
+; CHECK-FP16-GI-NEXT: fcvt d3, h3
+; CHECK-FP16-GI-NEXT: fcvt d4, h4
+; CHECK-FP16-GI-NEXT: fcvt d5, h5
+; CHECK-FP16-GI-NEXT: fcvt d6, h6
+; CHECK-FP16-GI-NEXT: fcvt d7, h7
+; CHECK-FP16-GI-NEXT: fcvt d16, h16
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d24, h1
+; CHECK-FP16-GI-NEXT: fcvt d1, h17
+; CHECK-FP16-GI-NEXT: fcvt d17, h18
+; CHECK-FP16-GI-NEXT: fcvt d18, h19
+; CHECK-FP16-GI-NEXT: fcvt d19, h20
+; CHECK-FP16-GI-NEXT: fcvt d20, h21
+; CHECK-FP16-GI-NEXT: fcvt d21, h22
+; CHECK-FP16-GI-NEXT: fcvt d22, h23
+; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT: mov v4.d[1], v5.d[0]
+; CHECK-FP16-GI-NEXT: mov v6.d[1], v7.d[0]
+; CHECK-FP16-GI-NEXT: mov v16.d[1], v0.d[0]
+; CHECK-FP16-GI-NEXT: mov v24.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT: mov v17.d[1], v18.d[0]
+; CHECK-FP16-GI-NEXT: mov v19.d[1], v20.d[0]
+; CHECK-FP16-GI-NEXT: mov v21.d[1], v22.d[0]
+; CHECK-FP16-GI-NEXT: fcvtzs v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v1.2d, v4.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v2.2d, v6.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v3.2d, v16.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v4.2d, v24.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v5.2d, v17.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v6.2d, v19.2d
+; CHECK-FP16-GI-NEXT: fcvtzs v7.2d, v21.2d
; CHECK-FP16-GI-NEXT: ret
entry:
%c = fptosi <16 x half> %a to <16 x i64>
@@ -5396,26 +5480,52 @@ define <16 x i64> @fptou_v16f16_v16i64(<16 x half> %a) {
;
; CHECK-FP16-GI-LABEL: fptou_v16f16_v16i64:
; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v2.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-FP16-GI-NEXT: fcvtl v3.4s, v1.4h
-; CHECK-FP16-GI-NEXT: fcvtl2 v1.4s, v1.8h
-; CHECK-FP16-GI-NEXT: fcvtl v4.2d, v2.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v2.2d, v2.4s
-; CHECK-FP16-GI-NEXT: fcvtl v5.2d, v0.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v6.2d, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtl v7.2d, v3.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v16.2d, v3.4s
-; CHECK-FP16-GI-NEXT: fcvtl v17.2d, v1.2s
-; CHECK-FP16-GI-NEXT: fcvtl2 v18.2d, v1.4s
-; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v4.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v1.2d, v2.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v5.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v3.2d, v6.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v4.2d, v7.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v5.2d, v16.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v6.2d, v17.2d
-; CHECK-FP16-GI-NEXT: fcvtzu v7.2d, v18.2d
+; CHECK-FP16-GI-NEXT: mov h3, v0.h[1]
+; CHECK-FP16-GI-NEXT: mov h4, v0.h[2]
+; CHECK-FP16-GI-NEXT: mov h5, v0.h[3]
+; CHECK-FP16-GI-NEXT: fcvt d2, h0
+; CHECK-FP16-GI-NEXT: mov h6, v0.h[4]
+; CHECK-FP16-GI-NEXT: mov h7, v0.h[5]
+; CHECK-FP16-GI-NEXT: mov h16, v0.h[6]
+; CHECK-FP16-GI-NEXT: mov h0, v0.h[7]
+; CHECK-FP16-GI-NEXT: mov h17, v1.h[1]
+; CHECK-FP16-GI-NEXT: mov h18, v1.h[2]
+; CHECK-FP16-GI-NEXT: mov h19, v1.h[3]
+; CHECK-FP16-GI-NEXT: mov h20, v1.h[4]
+; CHECK-FP16-GI-NEXT: mov h21, v1.h[5]
+; CHECK-FP16-GI-NEXT: mov h22, v1.h[6]
+; CHECK-FP16-GI-NEXT: mov h23, v1.h[7]
+; CHECK-FP16-GI-NEXT: fcvt d3, h3
+; CHECK-FP16-GI-NEXT: fcvt d4, h4
+; CHECK-FP16-GI-NEXT: fcvt d5, h5
+; CHECK-FP16-GI-NEXT: fcvt d6, h6
+; CHECK-FP16-GI-NEXT: fcvt d7, h7
+; CHECK-FP16-GI-NEXT: fcvt d16, h16
+; CHECK-FP16-GI-NEXT: fcvt d0, h0
+; CHECK-FP16-GI-NEXT: fcvt d24, h1
+; CHECK-FP16-GI-NEXT: fcvt d1, h17
+; CHECK-FP16-GI-NEXT: fcvt d17, h18
+; CHECK-FP16-GI-NEXT: fcvt d18, h19
+; CHECK-FP16-GI-NEXT: fcvt d19, h20
+; CHECK-FP16-GI-NEXT: fcvt d20, h21
+; CHECK-FP16-GI-NEXT: fcvt d21, h22
+; CHECK-FP16-GI-NEXT: fcvt d22, h23
+; CHECK-FP16-GI-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT: mov v4.d[1], v5.d[0]
+; CHECK-FP16-GI-NEXT: mov v6.d[1], v7.d[0]
+; CHECK-FP16-GI-NEXT: mov v16.d[1], v0.d[0]
+; CHECK-FP16-GI-NEXT: mov v24.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT: mov v17.d[1], v18.d[0]
+; CHECK-FP16-GI-NEXT: mov v19.d[1], v20.d[0]
+; CHECK-FP16-GI-NEXT: mov v21.d[1], v22.d[0]
+; CHECK-FP16-GI-NEXT: fcvtzu v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v1.2d, v4.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v2.2d, v6.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v3.2d, v16.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v4.2d, v24.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v5.2d, v17.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v6.2d, v19.2d
+; CHECK-FP16-GI-NEXT: fcvtzu v7.2d, v21.2d
; CHECK-FP16-GI-NEXT: ret
entry:
%c = fptoui <16 x half> %a to <16 x i64>
diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
index dbcfaff8aee05..b963acd8cb2a1 100644
--- a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
@@ -3088,14 +3088,30 @@ define <4 x i64> @test_signed_v4f16_v4i64(<4 x half> %f) {
; CHECK-SD-FP16-NEXT: mov v1.d[1], x11
; CHECK-SD-FP16-NEXT: ret
;
-; CHECK-GI-LABEL: test_signed_v4f16_v4i64:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-GI-NEXT: fcvtl v1.2d, v0.2s
-; CHECK-GI-NEXT: fcvtl2 v2.2d, v0.4s
-; CHECK-GI-NEXT: fcvtzs v0.2d, v1.2d
-; CHECK-GI-NEXT: fcvtzs v1.2d, v2.2d
-; CHECK-GI-NEXT: ret
+; CHECK-GI-CVT-LABEL: test_signed_v4f16_v4i64:
+; CHECK-GI-CVT: // %bb.0:
+; CHECK-GI-CVT-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-GI-CVT-NEXT: fcvtl v1.2d, v0.2s
+; CHECK-GI-CVT-NEXT: fcvtl2 v2.2d, v0.4s
+; CHECK-GI-CVT-NEXT: fcvtzs v0.2d, v1.2d
+; CHECK-GI-CVT-NEXT: fcvtzs v1.2d, v2.2d
+; CHECK-GI-CVT-NEXT: ret
+;
+; CHECK-GI-FP16-LABEL: test_signed_v4f16_v4i64:
+; CHECK-GI-FP16: // %bb.0:
+; CHECK-GI-FP16-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-FP16-NEXT: mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT: mov h2, v0.h[2]
+; CHECK-GI-FP16-NEXT: mov h3, v0.h[3]
+; CHECK-GI-FP16-NEXT: fcvt d0, h0
+; CHECK-GI-FP16-NEXT: fcvt d1, h1
+; CHECK-GI-FP16-NEXT: fcvt d2, h2
+; CHECK-GI-FP16-NEXT: fcvt d3, h3
+; CHECK-GI-FP16-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-GI-FP16-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-GI-FP16-NEXT: fcvtzs v0.2d, v0.2d
+; CHECK-GI-FP16-NEXT: fcvtzs v1.2d, v2.2d
+; CHECK-GI-FP16-NEXT: ret
%x = call <4 x i64> @llvm.fptosi.sat.v4f16.v4i64(<4 x half> %f)
ret <4 x i64> %x
}
@@ -3781,19 +3797,46 @@ define <8 x i64> @test_signed_v8f16_v8i64(<8 x half> %f) {
; CHECK-SD-FP16-NEXT: mov v3.d[1], x14
; CHECK-SD-FP16-NEXT: ret
;
-; CHECK-GI-LABEL: test_signed_v8f16_v8i64:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NEXT: fcvtl v2.2d, v1.2s
-; CHECK-GI-NEXT: fcvtl2 v1.2d, v1.4s
-; CHECK-GI-NEXT: fcvtl v3.2d, v0.2s
-; CHECK-GI-NEXT: fcvtl2 v4.2d, v0.4s
-; CHECK-GI-NEXT: fcvtzs v0.2d, v2.2d
-; CHECK-GI-NEXT: fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT: fcvtzs v2.2d, v3.2d
-; CHECK-GI-NEXT: fcvtzs v3.2d, v4.2d
-; CHECK-GI-NEXT: ret
+; CHECK-GI-CVT-LABEL: test_signed_v8f16_v8i64:
+; CHECK-GI-CVT: // %bb.0:
+; CHECK-GI-CVT-NEXT: fcvtl v1.4s, v0.4h
+; CHECK-GI-CVT-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-GI-CVT-NEXT: fcvtl v2.2d, v1.2s
+; CHECK-GI-CVT-NEXT: fcvtl2 v1.2d, v1.4s
+; CHECK-GI-CVT-NEXT: fcvtl v3.2d, v0.2s
+; CHECK-GI-CVT-NEXT: fcvtl2 v4.2d, v0.4s
+; CHECK-GI-CVT-NEXT: fcvtzs v0.2d, v2.2d
+; CHECK-GI-CVT-NEXT: fcvtzs v1.2d, v1.2d
+; CHECK-GI-CVT-NEXT: fcvtzs v2.2d, v3.2d
+; CHECK-GI-CVT-NEXT: fcvtzs v3.2d, v4.2d
+; CHECK-GI-CVT-NEXT: ret
+;
+; CHECK-GI-FP16-LABEL: test_signed_v8f16_v8i64:
+; CHECK-GI-FP16: // %bb.0:
+; CHECK-GI-FP16-NEXT: mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT: mov h2, v0.h[2]
+; CHECK-GI-FP16-NEXT: mov h3, v0.h[3]
+; CHECK-GI-FP16-NEXT: mov h4, v0.h[4]
+; CHECK-GI-FP16-NEXT: mov h5, v0.h[5]
+; CHECK-GI-FP16-NEXT: mov h6, v0.h[6]
+; CHECK-GI-FP16-NEXT: mov h7, v0.h[7]
+; CHECK-GI-FP16-NEXT: fcvt d0, h0
+; CHECK-GI-FP16-NEXT: fcvt d1, h1
+; CHECK-GI-FP16-NEXT: fcvt d2, h2
+; CHECK-GI-FP16-NEXT: fcvt d3, h3
+; CHECK-GI-FP16-NEXT: fcvt d4, h4
+; CHECK-GI-FP16-NEXT: fcvt d5, h5
+; CHECK-GI-FP16-NEXT: fcvt d6, h6
+; CHECK-GI-FP16-NEXT: fcvt d7, h7
+; CHECK-GI-FP16-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-GI-FP16-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-GI-FP16-NEXT: mov v4.d[1], v5.d[0]
+; CHECK-GI-FP16-NEXT: mov v6.d[1], v7.d[0]
+; CHECK-GI-FP16-NEXT: fcvtzs v0.2d, v0.2d
+; CHECK-GI-FP16-NEXT: fcvtzs v1.2d, v2.2d
+; CHECK-GI-FP16-NEXT: fcvtzs v2.2d, v4.2d
+; CHECK-GI-FP16-NEXT: fcvtzs v3.2d, v6.2d
+; CHECK-GI-FP16-NEXT: ret
%x = call <8 x i64> @llvm.fptosi.sat.v8f16.v8i64(<8 x half> %f)
ret <8 x i64> %x
}
diff --git a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
index 44e6e9415263b..5a66b68af8e96 100644
--- a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
@@ -2506,14 +2506,30 @@ define <4 x i64> @test_unsigned_v4f16_v4i64(<4 x half> %f) {
; CHECK-SD-FP16-NEXT: mov v1.d[1], x11
; CHECK-SD-FP16-NEXT: ret
;
-; CHECK-GI-LABEL: test_unsigned_v4f16_v4i64:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-GI-NEXT: fcvtl v1.2d, v0.2s
-; CHECK-GI-NEXT: fcvtl2 v2.2d, v0.4s
-; CHECK-GI-NEXT: fcvtzu v0.2d, v1.2d
-; CHECK-GI-NEXT: fcvtzu v1.2d, v2.2d
-; CHECK-GI-NEXT: ret
+; CHECK-GI-CVT-LABEL: test_unsigned_v4f16_v4i64:
+; CHECK-GI-CVT: // %bb.0:
+; CHECK-GI-CVT-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-GI-CVT-NEXT: fcvtl v1.2d, v0.2s
+; CHECK-GI-CVT-NEXT: fcvtl2 v2.2d, v0.4s
+; CHECK-GI-CVT-NEXT: fcvtzu v0.2d, v1.2d
+; CHECK-GI-CVT-NEXT: fcvtzu v1.2d, v2.2d
+; CHECK-GI-CVT-NEXT: ret
+;
+; CHECK-GI-FP16-LABEL: test_unsigned_v4f16_v4i64:
+; CHECK-GI-FP16: // %bb.0:
+; CHECK-GI-FP16-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-FP16-NEXT: mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT: mov h2, v0.h[2]
+; CHECK-GI-FP16-NEXT: mov h3, v0.h[3]
+; CHECK-GI-FP16-NEXT: fcvt d0, h0
+; CHECK-GI-FP16-NEXT: fcvt d1, h1
+; CHECK-GI-FP16-NEXT: fcvt d2, h2
+; CHECK-GI-FP16-NEXT: fcvt d3, h3
+; CHECK-GI-FP16-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-GI-FP16-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-GI-FP16-NEXT: fcvtzu v0.2d, v0.2d
+; CHECK-GI-FP16-NEXT: fcvtzu v1.2d, v2.2d
+; CHECK-GI-FP16-NEXT: ret
%x = call <4 x i64> @llvm.fptoui.sat.v4f16.v4i64(<4 x half> %f)
ret <4 x i64> %x
}
@@ -3098,19 +3114,46 @@ define <8 x i64> @test_unsigned_v8f16_v8i64(<8 x half> %f) {
; CHECK-SD-FP16-NEXT: mov v3.d[1], x14
; CHECK-SD-FP16-NEXT: ret
;
-; CHECK-GI-LABEL: test_unsigned_v8f16_v8i64:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NEXT: fcvtl v2.2d, v1.2s
-; CHECK-GI-NEXT: fcvtl2 v1.2d, v1.4s
-; CHECK-GI-NEXT: fcvtl v3.2d, v0.2s
-; CHECK-GI-NEXT: fcvtl2 v4.2d, v0.4s
-; CHECK-GI-NEXT: fcvtzu v0.2d, v2.2d
-; CHECK-GI-NEXT: fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT: fcvtzu v2.2d, v3.2d
-; CHECK-GI-NEXT: fcvtzu v3.2d, v4.2d
-; CHECK-GI-NEXT: ret
+; CHECK-GI-CVT-LABEL: test_unsigned_v8f16_v8i64:
+; CHECK-GI-CVT: // %bb.0:
+; CHECK-GI-CVT-NEXT: fcvtl v1.4s, v0.4h
+; CHECK-GI-CVT-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-GI-CVT-NEXT: fcvtl v2.2d, v1.2s
+; CHECK-GI-CVT-NEXT: fcvtl2 v1.2d, v1.4s
+; CHECK-GI-CVT-NEXT: fcvtl v3.2d, v0.2s
+; CHECK-GI-CVT-NEXT: fcvtl2 v4.2d, v0.4s
+; CHECK-GI-CVT-NEXT: fcvtzu v0.2d, v2.2d
+; CHECK-GI-CVT-NEXT: fcvtzu v1.2d, v1.2d
+; CHECK-GI-CVT-NEXT: fcvtzu v2.2d, v3.2d
+; CHECK-GI-CVT-NEXT: fcvtzu v3.2d, v4.2d
+; CHECK-GI-CVT-NEXT: ret
+;
+; CHECK-GI-FP16-LABEL: test_unsigned_v8f16_v8i64:
+; CHECK-GI-FP16: // %bb.0:
+; CHECK-GI-FP16-NEXT: mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT: mov h2, v0.h[2]
+; CHECK-GI-FP16-NEXT: mov h3, v0.h[3]
+; CHECK-GI-FP16-NEXT: mov h4, v0.h[4]
+; CHECK-GI-FP16-NEXT: mov h5, v0.h[5]
+; CHECK-GI-FP16-NEXT: mov h6, v0.h[6]
+; CHECK-GI-FP16-NEXT: mov h7, v0.h[7]
+; CHECK-GI-FP16-NEXT: fcvt d0, h0
+; CHECK-GI-FP16-NEXT: fcvt d1, h1
+; CHECK-GI-FP16-NEXT: fcvt d2, h2
+; CHECK-GI-FP16-NEXT: fcvt d3, h3
+; CHECK-GI-FP16-NEXT: fcvt d4, h4
+; CHECK-GI-FP16-NEXT: fcvt d5, h5
+; CHECK-GI-FP16-NEXT: fcvt d6, h6
+; CHECK-GI-FP16-NEXT: fcvt d7, h7
+; CHECK-GI-FP16-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-GI-FP16-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-GI-FP16-NEXT: mov v4.d[1], v5.d[0]
+; CHECK-GI-FP16-NEXT: mov v6.d[1], v7.d[0]
+; CHECK-GI-FP16-NEXT: fcvtzu v0.2d, v0.2d
+; CHECK-GI-FP16-NEXT: fcvtzu v1.2d, v2.2d
+; CHECK-GI-FP16-NEXT: fcvtzu v2.2d, v4.2d
+; CHECK-GI-FP16-NEXT: fcvtzu v3.2d, v6.2d
+; CHECK-GI-FP16-NEXT: ret
%x = call <8 x i64> @llvm.fptoui.sat.v8f16.v8i64(<8 x half> %f)
ret <8 x i64> %x
}
>From 26ad4d56ff0e669ef17b5e37e7e20e6b78c19205 Mon Sep 17 00:00:00 2001
From: Ryan Cowan <ryan.cowan at arm.com>
Date: Tue, 21 Oct 2025 08:13:04 +0000
Subject: [PATCH 04/12] Add new opcode for rounding to odd
---
.../llvm/CodeGen/GlobalISel/MachineIRBuilder.h | 14 ++++++++++++++
llvm/include/llvm/Support/TargetOpcodes.def | 3 +++
llvm/include/llvm/Target/GenericOpcodes.td | 6 ++++++
llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp | 1 +
llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp | 6 ++++++
.../Target/AArch64/GISel/AArch64LegalizerInfo.cpp | 7 ++++++-
6 files changed, 36 insertions(+), 1 deletion(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 40c7792f7e8a2..1e39b1f4452c6 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1332,6 +1332,20 @@ class LLVM_ABI MachineIRBuilder {
buildFPTrunc(const DstOp &Res, const SrcOp &Op,
std::optional<unsigned> Flags = std::nullopt);
+ /// Build and insert \p Res = G_FPTRUNC_ODD \p Op
+ ///
+ /// G_FPTRUNC_ODD converts a floating-point value into one with a smaller type using round to odd.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p Res must be a generic virtual register with scalar or vector type.
+ /// \pre \p Op must be a generic virtual register with scalar or vector type.
+ /// \pre \p Res must be smaller than \p Op
+ ///
+ /// \return The newly created instruction.
+ MachineInstrBuilder
+ buildFPTruncOdd(const DstOp &Res, const SrcOp &Op,
+ std::optional<unsigned> Flags = std::nullopt);
+
/// Build and insert \p Res = G_TRUNC \p Op
///
/// G_TRUNC extracts the low bits of a type. For a vector type each element is
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index e55314568d683..198499b7f4b30 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -692,6 +692,9 @@ HANDLE_TARGET_OPCODE(G_FPEXT)
/// Generic float to signed-int conversion
HANDLE_TARGET_OPCODE(G_FPTRUNC)
+/// Generic float to signed-int conversion using round to odd
+HANDLE_TARGET_OPCODE(G_FPTRUNC_ODD)
+
/// Generic float to signed-int conversion
HANDLE_TARGET_OPCODE(G_FPTOSI)
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index e3f995d53484f..f083bd9312983 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -782,6 +782,12 @@ def G_FPTRUNC : GenericInstruction {
let hasSideEffects = false;
}
+def G_FPTRUNC_ODD : GenericInstruction {
+ let OutOperandList = (outs type0:$dst);
+ let InOperandList = (ins type1:$src);
+ let hasSideEffects = false;
+}
+
def G_FPTOSI : GenericInstruction {
let OutOperandList = (outs type0:$dst);
let InOperandList = (ins type1:$src);
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 52c43a4ac4a04..450fe3220e1c2 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -5595,6 +5595,7 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
case G_ANYEXT:
case G_FPEXT:
case G_FPTRUNC:
+ case G_FPTRUNC_ODD:
case G_SITOFP:
case G_UITOFP:
case G_FPTOSI:
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 4b4df98024f4a..b96bb442f3e63 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -936,6 +936,12 @@ MachineIRBuilder::buildFPTrunc(const DstOp &Res, const SrcOp &Op,
return buildInstr(TargetOpcode::G_FPTRUNC, Res, Op, Flags);
}
+MachineInstrBuilder
+MachineIRBuilder::buildFPTruncOdd(const DstOp &Res, const SrcOp &Op,
+ std::optional<unsigned> Flags) {
+ return buildInstr(TargetOpcode::G_FPTRUNC_ODD, Res, Op, Flags);
+}
+
MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
const DstOp &Res,
const SrcOp &Op0,
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 4f3c8ab2e62e6..576a9b090a4ca 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -831,6 +831,11 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
.clampNumElements(1, v2s64, v2s64)
.scalarize(0);
+ getActionDefinitionsBuilder(G_FPTRUNC_ODD)
+ .legalFor({{s16, s32}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}})
+ .clampMaxNumElements(1, s32, 4)
+ .clampMaxNumElements(1, s64, 2);
+
getActionDefinitionsBuilder(G_FPEXT)
.legalFor(
{{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}})
@@ -2422,7 +2427,7 @@ bool AArch64LegalizerInfo::legalizeFptrunc(
default:
return false;
case TargetOpcode::G_FPTRUNC: {
- Mid = MIRBuilder.buildFPTrunc(MidTy, Src);
+ Mid = MIRBuilder.buildFPTruncOdd(MidTy, Src);
Fin = MIRBuilder.buildFPTrunc(DstTy, Mid.getReg(0));
break;
}
>From f2901629a8457e4f3b2df8cdb55ec1a0fccb1f10 Mon Sep 17 00:00:00 2001
From: Ryan Cowan <ryan.cowan at arm.com>
Date: Tue, 21 Oct 2025 08:49:29 +0000
Subject: [PATCH 05/12] Disable combiner
---
llvm/lib/Target/AArch64/AArch64Combine.td | 14 +++++++-------
.../AArch64/GISel/AArch64PostLegalizerLowering.cpp | 2 +-
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index fe3ba82ceb3cd..e8b2e0edc3815 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -332,12 +332,12 @@ def combine_mul_cmlt : GICombineRule<
(apply [{ applyCombineMulCMLT(*${root}, MRI, B, ${matchinfo}); }])
>;
-def lower_fptrunc_fptrunc: GICombineRule<
- (defs root:$root),
- (match (wip_match_opcode G_FPTRUNC):$root,
- [{ return matchFpTruncFpTrunc(*${root}, MRI); }]),
- (apply [{ applyFpTruncFpTrunc(*${root}, MRI, B); }])
->;
+// def lower_fptrunc_fptrunc: GICombineRule<
+// (defs root:$root),
+// (match (wip_match_opcode G_FPTRUNC):$root,
+// [{ return matchFpTruncFpTrunc(*${root}, MRI); }]),
+// (apply [{ applyFpTruncFpTrunc(*${root}, MRI, B); }])
+// >;
// Post-legalization combines which should happen at all optimization levels.
// (E.g. ones that facilitate matching for the selector) For example, matching
@@ -347,7 +347,7 @@ def AArch64PostLegalizerLowering
[shuffle_vector_lowering, vashr_vlshr_imm,
icmp_lowering, build_vector_lowering,
lower_vector_fcmp, form_truncstore, fconstant_to_constant,
- vector_sext_inreg_to_shift, lower_fptrunc_fptrunc,
+ vector_sext_inreg_to_shift,
unmerge_ext_to_unmerge, lower_mulv2s64,
vector_unmerge_lowering, insertelt_nonconst,
unmerge_duplanes]> {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
index 2fa1b86a8a9c6..8e84a8e472eca 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
@@ -905,7 +905,7 @@ unsigned getCmpOperandFoldingProfit(Register CmpOp, MachineRegisterInfo &MRI) {
// Checks that the given definition belongs to an FPTRUNC and that the source is
// not an integer, as no rounding is necessary due to the range of values
bool isFPTruncFromDouble(MachineRegisterInfo &MRI, MachineInstr *MaybeFpTrunc) {
- if (!MaybeFpTrunc || MaybeFpTrunc->getOpcode() != TargetOpcode::G_FPTRUNC)
+ if (!MaybeFpTrunc || MaybeFpTrunc->getOpcode() != TargetOpcode::G_FPTRUNC_ODD)
return false;
// Check the source is 64 bits as we only want to match a very specific
>From 7f8e4bdd3c627fa1eaa95c84a821e78684a2212b Mon Sep 17 00:00:00 2001
From: Ryan Cowan <ryan.cowan at arm.com>
Date: Tue, 21 Oct 2025 08:49:38 +0000
Subject: [PATCH 06/12] Use tablegen for matching
---
llvm/lib/Target/AArch64/AArch64InstrGISel.td | 2 +
.../CodeGen/AArch64/fp16-v8-instructions.ll | 48 ++++---------------
llvm/test/CodeGen/AArch64/fptrunc.ll | 18 +++++--
3 files changed, 24 insertions(+), 44 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrGISel.td b/llvm/lib/Target/AArch64/AArch64InstrGISel.td
index 30b7b03f7a69a..576db35b88c14 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrGISel.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrGISel.td
@@ -290,6 +290,8 @@ def : GINodeEquiv<G_EXTRACT_VECTOR_ELT, vector_extract>;
def : GINodeEquiv<G_AARCH64_PREFETCH, AArch64Prefetch>;
+def : GINodeEquiv<G_FPTRUNC_ODD, AArch64fcvtxn_n>;
+
// These are patterns that we only use for GlobalISel via the importer.
def : Pat<(f32 (fadd (vector_extract (v2f32 FPR64:$Rn), (i64 0)),
(vector_extract (v2f32 FPR64:$Rn), (i64 1)))),
diff --git a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
index 78db49613f1e6..0d138e59089c0 100644
--- a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
@@ -176,45 +176,15 @@ define <8 x half> @s_to_h(<8 x float> %a) {
}
define <8 x half> @d_to_h(<8 x double> %a) {
-; CHECK-CVT-SD-LABEL: d_to_h:
-; CHECK-CVT-SD: // %bb.0:
-; CHECK-CVT-SD-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-CVT-SD-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-CVT-SD-NEXT: fcvtxn2 v0.4s, v1.2d
-; CHECK-CVT-SD-NEXT: fcvtxn2 v2.4s, v3.2d
-; CHECK-CVT-SD-NEXT: fcvtn v0.4h, v0.4s
-; CHECK-CVT-SD-NEXT: fcvtn2 v0.8h, v2.4s
-; CHECK-CVT-SD-NEXT: ret
-;
-; CHECK-FP16-SD-LABEL: d_to_h:
-; CHECK-FP16-SD: // %bb.0:
-; CHECK-FP16-SD-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-FP16-SD-NEXT: fcvtxn v2.2s, v2.2d
-; CHECK-FP16-SD-NEXT: fcvtxn2 v0.4s, v1.2d
-; CHECK-FP16-SD-NEXT: fcvtxn2 v2.4s, v3.2d
-; CHECK-FP16-SD-NEXT: fcvtn v0.4h, v0.4s
-; CHECK-FP16-SD-NEXT: fcvtn2 v0.8h, v2.4s
-; CHECK-FP16-SD-NEXT: ret
-;
-; CHECK-CVT-GI-LABEL: d_to_h:
-; CHECK-CVT-GI: // %bb.0:
-; CHECK-CVT-GI-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-CVT-GI-NEXT: fcvtxn2 v0.4s, v1.2d
-; CHECK-CVT-GI-NEXT: fcvtxn v1.2s, v2.2d
-; CHECK-CVT-GI-NEXT: fcvtn v0.4h, v0.4s
-; CHECK-CVT-GI-NEXT: fcvtxn2 v1.4s, v3.2d
-; CHECK-CVT-GI-NEXT: fcvtn2 v0.8h, v1.4s
-; CHECK-CVT-GI-NEXT: ret
-;
-; CHECK-FP16-GI-LABEL: d_to_h:
-; CHECK-FP16-GI: // %bb.0:
-; CHECK-FP16-GI-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-FP16-GI-NEXT: fcvtxn2 v0.4s, v1.2d
-; CHECK-FP16-GI-NEXT: fcvtxn v1.2s, v2.2d
-; CHECK-FP16-GI-NEXT: fcvtn v0.4h, v0.4s
-; CHECK-FP16-GI-NEXT: fcvtxn2 v1.4s, v3.2d
-; CHECK-FP16-GI-NEXT: fcvtn2 v0.8h, v1.4s
-; CHECK-FP16-GI-NEXT: ret
+; CHECK-LABEL: d_to_h:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-NEXT: fcvtxn v2.2s, v2.2d
+; CHECK-NEXT: fcvtxn2 v0.4s, v1.2d
+; CHECK-NEXT: fcvtxn2 v2.4s, v3.2d
+; CHECK-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-NEXT: fcvtn2 v0.8h, v2.4s
+; CHECK-NEXT: ret
%1 = fptrunc <8 x double> %a to <8 x half>
ret <8 x half> %1
}
diff --git a/llvm/test/CodeGen/AArch64/fptrunc.ll b/llvm/test/CodeGen/AArch64/fptrunc.ll
index de780bf475138..aaaa7e2c599fa 100644
--- a/llvm/test/CodeGen/AArch64/fptrunc.ll
+++ b/llvm/test/CodeGen/AArch64/fptrunc.ll
@@ -277,11 +277,19 @@ entry:
}
define <2 x half> @fptrunc_v2f64_v2f16(<2 x double> %a) {
-; CHECK-LABEL: fptrunc_v2f64_v2f16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-NEXT: fcvtn v0.4h, v0.4s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: fptrunc_v2f64_v2f16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-SD-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: fptrunc_v2f64_v2f16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-GI-NEXT: mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT: mov v1.s[1], v0.s[1]
+; CHECK-GI-NEXT: fcvtn v0.4h, v1.4s
+; CHECK-GI-NEXT: ret
entry:
%c = fptrunc <2 x double> %a to <2 x half>
ret <2 x half> %c
>From 318313837a63229ca85b60b52029ad1792c6924f Mon Sep 17 00:00:00 2001
From: Ryan Cowan <ryan.cowan at arm.com>
Date: Tue, 21 Oct 2025 09:02:31 +0000
Subject: [PATCH 07/12] Remove unused code
---
llvm/lib/Target/AArch64/AArch64Combine.td | 7 -
.../GISel/AArch64PostLegalizerLowering.cpp | 192 ------------------
2 files changed, 199 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index e8b2e0edc3815..b3ec65cab51fa 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -332,13 +332,6 @@ def combine_mul_cmlt : GICombineRule<
(apply [{ applyCombineMulCMLT(*${root}, MRI, B, ${matchinfo}); }])
>;
-// def lower_fptrunc_fptrunc: GICombineRule<
-// (defs root:$root),
-// (match (wip_match_opcode G_FPTRUNC):$root,
-// [{ return matchFpTruncFpTrunc(*${root}, MRI); }]),
-// (apply [{ applyFpTruncFpTrunc(*${root}, MRI, B); }])
-// >;
-
// Post-legalization combines which should happen at all optimization levels.
// (E.g. ones that facilitate matching for the selector) For example, matching
// pseudos.
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
index 8e84a8e472eca..23dcaea2ac1a4 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
@@ -901,198 +901,6 @@ unsigned getCmpOperandFoldingProfit(Register CmpOp, MachineRegisterInfo &MRI) {
return 0;
}
-// Helper function for matchFpTruncFpTrunc.
-// Checks that the given definition belongs to an FPTRUNC and that the source is
-// not an integer, as no rounding is necessary due to the range of values
-bool isFPTruncFromDouble(MachineRegisterInfo &MRI, MachineInstr *MaybeFpTrunc) {
- if (!MaybeFpTrunc || MaybeFpTrunc->getOpcode() != TargetOpcode::G_FPTRUNC_ODD)
- return false;
-
- // Check the source is 64 bits as we only want to match a very specific
- // pattern
- Register FpTruncSrc = MaybeFpTrunc->getOperand(1).getReg();
- LLT SrcTy = MRI.getType(FpTruncSrc);
- if (SrcTy.getScalarSizeInBits() != 64)
- return false;
-
- // Need to check the float didn't come from an int as no rounding is
- // neccessary
- MachineInstr *FpTruncSrcDef = getDefIgnoringCopies(FpTruncSrc, MRI);
- if (FpTruncSrcDef->getOpcode() == TargetOpcode::G_SITOFP ||
- FpTruncSrcDef->getOpcode() == TargetOpcode::G_UITOFP)
- return false;
-
- return true;
-}
-
-// To avoid double rounding issues we need to lower FPTRUNC(FPTRUNC) to an odd
-// rounding truncate and a normal truncate. When
-// truncating an FP that came from an integer this is not a problem as the range
-// of values is lower in the int
-bool matchFpTruncFpTrunc(MachineInstr &MI, MachineRegisterInfo &MRI) {
- assert(MI.getOpcode() == TargetOpcode::G_FPTRUNC && "Expected G_FPTRUNC");
-
- // Check the destination is 16 bits as we only want to match a very specific
- // pattern
- Register Dst = MI.getOperand(0).getReg();
- LLT DstTy = MRI.getType(Dst);
- if (DstTy.getScalarSizeInBits() != 16)
- return false;
-
- Register Src = MI.getOperand(1).getReg();
-
- MachineInstr *ParentDef = getDefIgnoringCopies(Src, MRI);
- if (!ParentDef)
- return false;
-
- MachineInstr *FpTruncDef;
- switch (ParentDef->getOpcode()) {
- default:
- return false;
- case TargetOpcode::G_CONCAT_VECTORS: {
- // Expecting exactly two FPTRUNCs
- if (ParentDef->getNumOperands() != 3)
- return false;
-
- // All operands need to be FPTRUNC
- for (unsigned OpIdx = 1, NumOperands = ParentDef->getNumOperands();
- OpIdx != NumOperands; ++OpIdx) {
- Register FpTruncDst = ParentDef->getOperand(OpIdx).getReg();
- FpTruncDef = getDefIgnoringCopies(FpTruncDst, MRI);
-
- if (!isFPTruncFromDouble(MRI, FpTruncDef))
- return false;
- }
-
- return true;
- }
- // This is to match cases in which vectors are widened to a larger size
- case TargetOpcode::G_INSERT_VECTOR_ELT: {
- Register VecExtractDst = ParentDef->getOperand(2).getReg();
- MachineInstr *VecExtractDef = getDefIgnoringCopies(VecExtractDst, MRI);
-
- if (!VecExtractDef ||
- VecExtractDef->getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
- return false;
-
- Register FpTruncDst = VecExtractDef->getOperand(1).getReg();
- FpTruncDef = getDefIgnoringCopies(FpTruncDst, MRI);
- break;
- }
- case TargetOpcode::G_FPTRUNC: {
- Register FpTruncDst = ParentDef->getOperand(1).getReg();
- FpTruncDef = getDefIgnoringCopies(FpTruncDst, MRI);
- break;
- }
- }
-
- if (!isFPTruncFromDouble(MRI, FpTruncDef))
- return false;
-
- return true;
-}
-
-void applyFpTruncFpTrunc(MachineInstr &MI, MachineRegisterInfo &MRI,
- MachineIRBuilder &B) {
- assert(MI.getOpcode() == TargetOpcode::G_FPTRUNC && "Expected G_FPTRUNC");
- Register Dst = MI.getOperand(0).getReg();
- Register Src = MI.getOperand(1).getReg();
-
- MachineInstr *ParentDef = getDefIgnoringCopies(Src, MRI);
- if (!ParentDef)
- return;
-
- LLT V2F32 = LLT::fixed_vector(2, LLT::scalar(32));
- LLT V4F32 = LLT::fixed_vector(4, LLT::scalar(32));
- LLT V4F16 = LLT::fixed_vector(4, LLT::scalar(16));
-
- B.setInstrAndDebugLoc(MI);
-
- switch (ParentDef->getOpcode()) {
- default:
- return;
- case TargetOpcode::G_INSERT_VECTOR_ELT: {
- Register VecExtractDst = ParentDef->getOperand(2).getReg();
- MachineInstr *VecExtractDef = getDefIgnoringCopies(VecExtractDst, MRI);
-
- Register FpTruncDst = VecExtractDef->getOperand(1).getReg();
- MachineInstr *FpTruncDef = getDefIgnoringCopies(FpTruncDst, MRI);
-
- Register FpTruncSrc = FpTruncDef->getOperand(1).getReg();
- MRI.setRegClass(FpTruncSrc, &AArch64::FPR128RegClass);
-
- Register Fp32 = MRI.createGenericVirtualRegister(V2F32);
- MRI.setRegClass(Fp32, &AArch64::FPR64RegClass);
-
- B.buildInstr(AArch64::FCVTXNv2f32, {Fp32}, {FpTruncSrc});
-
- // Only 4f32 -> 4f16 is legal so we need to mimic that situation
- Register Fp32Padding = B.buildUndef(V2F32).getReg(0);
- MRI.setRegClass(Fp32Padding, &AArch64::FPR64RegClass);
-
- Register Fp32Full = MRI.createGenericVirtualRegister(V4F32);
- MRI.setRegClass(Fp32Full, &AArch64::FPR128RegClass);
- B.buildConcatVectors(Fp32Full, {Fp32, Fp32Padding});
-
- Register Fp16 = MRI.createGenericVirtualRegister(V4F16);
- MRI.setRegClass(Fp16, &AArch64::FPR64RegClass);
- B.buildFPTrunc(Fp16, Fp32Full);
-
- MRI.replaceRegWith(Dst, Fp16);
- MI.eraseFromParent();
- break;
- }
- case TargetOpcode::G_CONCAT_VECTORS: {
- // Get the two FP Truncs that are being concatenated
- Register FpTrunc1Dst = ParentDef->getOperand(1).getReg();
- Register FpTrunc2Dst = ParentDef->getOperand(2).getReg();
-
- MachineInstr *FpTrunc1Def = getDefIgnoringCopies(FpTrunc1Dst, MRI);
- MachineInstr *FpTrunc2Def = getDefIgnoringCopies(FpTrunc2Dst, MRI);
-
- // Make the registers 128bit to store the 2 doubles
- Register LoFp64 = FpTrunc1Def->getOperand(1).getReg();
- MRI.setRegClass(LoFp64, &AArch64::FPR128RegClass);
- Register HiFp64 = FpTrunc2Def->getOperand(1).getReg();
- MRI.setRegClass(HiFp64, &AArch64::FPR128RegClass);
-
- // Convert the lower half
- Register LoFp32 = MRI.createGenericVirtualRegister(V2F32);
- MRI.setRegClass(LoFp32, &AArch64::FPR64RegClass);
- B.buildInstr(AArch64::FCVTXNv2f32, {LoFp32}, {LoFp64});
-
- // Create a register for the high half to use
- Register AccUndef = MRI.createGenericVirtualRegister(V4F32);
- MRI.setRegClass(AccUndef, &AArch64::FPR128RegClass);
- B.buildUndef(AccUndef);
-
- Register Acc = MRI.createGenericVirtualRegister(V4F32);
- MRI.setRegClass(Acc, &AArch64::FPR128RegClass);
- B.buildInstr(TargetOpcode::INSERT_SUBREG)
- .addDef(Acc)
- .addUse(AccUndef)
- .addUse(LoFp32)
- .addImm(AArch64::dsub);
-
- // Convert the high half
- Register AccOut = MRI.createGenericVirtualRegister(V4F32);
- MRI.setRegClass(AccOut, &AArch64::FPR128RegClass);
- B.buildInstr(AArch64::FCVTXNv4f32)
- .addDef(AccOut)
- .addUse(Acc)
- .addUse(HiFp64);
-
- Register Fp16 = MRI.createGenericVirtualRegister(V4F16);
- MRI.setRegClass(Fp16, &AArch64::FPR64RegClass);
- B.buildFPTrunc(Fp16, AccOut);
-
- MRI.replaceRegWith(Dst, Fp16);
- MI.eraseFromParent();
- break;
- }
- }
-}
-
/// \returns true if it would be profitable to swap the LHS and RHS of a G_ICMP
/// instruction \p MI.
bool trySwapICmpOperands(MachineInstr &MI, MachineRegisterInfo &MRI) {
>From e6c5082f916e241bd67f9f1f5d75d669036de780 Mon Sep 17 00:00:00 2001
From: Ryan Cowan <ryan.cowan at arm.com>
Date: Mon, 27 Oct 2025 13:56:16 +0000
Subject: [PATCH 08/12] Working
---
llvm/lib/Target/AArch64/AArch64Combine.td | 9 ++-
.../GISel/AArch64PostLegalizerCombiner.cpp | 76 +++++++++++++++++++
llvm/test/CodeGen/AArch64/arm64-neon-copy.ll | 32 +++++---
llvm/test/CodeGen/AArch64/fptrunc.ll | 36 +++------
llvm/test/CodeGen/AArch64/itofp.ll | 48 +++---------
5 files changed, 128 insertions(+), 73 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index b3ec65cab51fa..db2e9007610e7 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -332,6 +332,13 @@ def combine_mul_cmlt : GICombineRule<
(apply [{ applyCombineMulCMLT(*${root}, MRI, B, ${matchinfo}); }])
>;
+def combine_build_unmerge : GICombineRule<
+ (defs root:$root, register_vector_matchinfo:$unmergedValues, register_vector_matchinfo:$undefinedValues),
+ (match (wip_match_opcode G_BUILD_VECTOR):$root,
+ [{ return matchCombineBuildUnmerge(*${root}, MRI, ${unmergedValues}, ${undefinedValues}); }]),
+ (apply [{ applyCombineBuildUnmerge(*${root}, MRI, B, ${unmergedValues}, ${undefinedValues}); }])
+>;
+
// Post-legalization combines which should happen at all optimization levels.
// (E.g. ones that facilitate matching for the selector) For example, matching
// pseudos.
@@ -366,6 +373,6 @@ def AArch64PostLegalizerCombiner
select_to_minmax, or_to_bsp, combine_concat_vector,
commute_constant_to_rhs, extract_vec_elt_combines,
push_freeze_to_prevent_poison_from_propagating,
- combine_mul_cmlt, combine_use_vector_truncate,
+ combine_mul_cmlt, combine_use_vector_truncate, combine_build_unmerge,
extmultomull, truncsat_combines, lshr_of_trunc_of_lshr]> {
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
index fa7bb6ecc35ee..4cef523951af5 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -20,6 +20,7 @@
//===----------------------------------------------------------------------===//
#include "AArch64TargetMachine.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
@@ -36,9 +37,13 @@
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/Register.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/Support/Debug.h"
+#include <cassert>
+#include <iostream>
#define GET_GICOMBINER_DEPS
#include "AArch64GenPostLegalizeGICombiner.inc"
@@ -133,6 +138,77 @@ bool isZeroExtended(Register R, MachineRegisterInfo &MRI) {
return MRI.getVRegDef(R)->getOpcode() == TargetOpcode::G_ZEXT;
}
+bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
+ SmallVectorImpl<Register> &unmergedValues,
+ SmallVectorImpl<Register> &undefinedValues) {
+ assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
+
+ undefinedValues.clear();
+ unmergedValues.clear();
+
+ for (auto Use : MI.all_uses()) {
+ auto *Def = getDefIgnoringCopies(Use.getReg(), MRI);
+
+ if (!Def) {
+ undefinedValues.clear();
+ unmergedValues.clear();
+ return false;
+ }
+
+ auto Opcode = Def->getOpcode();
+
+ switch (Opcode) {
+ default:
+ return false;
+ case TargetOpcode::G_IMPLICIT_DEF:
+ undefinedValues.push_back(Use.getReg());
+ break;
+ case TargetOpcode::G_UNMERGE_VALUES:
+ if (Def->getNumDefs() > 2 ||
+ MRI.getType(Use.getReg()) == LLT::scalar(16)) {
+ undefinedValues.clear();
+ unmergedValues.clear();
+ return false;
+ }
+
+ unmergedValues.push_back(Def->getOperand(2).getReg());
+
+ break;
+ }
+ }
+
+ if (!(undefinedValues.size() == 2 && unmergedValues.size() == 2) &&
+ !(undefinedValues.size() == 0 && unmergedValues.size() == 4)) {
+ undefinedValues.clear();
+ unmergedValues.clear();
+ return false;
+ }
+
+ return true;
+}
+
+void applyCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
+ MachineIRBuilder &B,
+ SmallVectorImpl<Register> &unmergedValues,
+ SmallVectorImpl<Register> &undefinedValues) {
+ B.setInstrAndDebugLoc(MI);
+
+ if (unmergedValues.size() == 2) {
+ auto llt = LLT::fixed_vector(
+ undefinedValues.size(),
+ LLT::scalar(MRI.getType(unmergedValues[0]).getScalarSizeInBits()));
+ Register DefVec = MRI.createGenericVirtualRegister(llt);
+
+ B.buildBuildVector(DefVec, undefinedValues);
+ B.buildConcatVectors(MI.getOperand(0), {unmergedValues[0], DefVec});
+ } else {
+ B.buildConcatVectors(MI.getOperand(0),
+ {unmergedValues[0], unmergedValues[3]});
+ }
+
+ MI.eraseFromParent();
+}
+
bool matchAArch64MulConstCombine(
MachineInstr &MI, MachineRegisterInfo &MRI,
std::function<void(MachineIRBuilder &B, Register DstReg)> &ApplyFn) {
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
index d8f370884c84a..8e4f022776a57 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
@@ -1268,11 +1268,17 @@ define <8 x i16> @testDUP.v1i16(<1 x i16> %a) {
}
define <4 x i32> @testDUP.v1i32(<1 x i32> %a) {
-; CHECK-LABEL: testDUP.v1i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: dup v0.4s, v0.s[0]
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: testDUP.v1i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT: dup v0.4s, v0.s[0]
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: testDUP.v1i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: mov v0.d[1], v0.d[0]
+; CHECK-GI-NEXT: ret
%b = extractelement <1 x i32> %a, i32 0
%c = insertelement <4 x i32> undef, i32 %b, i32 0
%d = insertelement <4 x i32> %c, i32 %b, i32 1
@@ -2243,11 +2249,17 @@ define <4 x i16> @concat_vector_v4i16(<1 x i16> %a) {
}
define <4 x i32> @concat_vector_v4i32(<1 x i32> %a) {
-; CHECK-LABEL: concat_vector_v4i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: dup v0.4s, v0.s[0]
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: concat_vector_v4i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT: dup v0.4s, v0.s[0]
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: concat_vector_v4i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: mov v0.d[1], v0.d[0]
+; CHECK-GI-NEXT: ret
%r = shufflevector <1 x i32> %a, <1 x i32> undef, <4 x i32> zeroinitializer
ret <4 x i32> %r
}
diff --git a/llvm/test/CodeGen/AArch64/fptrunc.ll b/llvm/test/CodeGen/AArch64/fptrunc.ll
index aaaa7e2c599fa..697b5e67d2e2a 100644
--- a/llvm/test/CodeGen/AArch64/fptrunc.ll
+++ b/llvm/test/CodeGen/AArch64/fptrunc.ll
@@ -277,19 +277,11 @@ entry:
}
define <2 x half> @fptrunc_v2f64_v2f16(<2 x double> %a) {
-; CHECK-SD-LABEL: fptrunc_v2f64_v2f16:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-SD-NEXT: fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: fptrunc_v2f64_v2f16:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: fcvtxn v0.2s, v0.2d
-; CHECK-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-GI-NEXT: fcvtn v0.4h, v1.4s
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: fptrunc_v2f64_v2f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fcvtxn v0.2s, v0.2d
+; CHECK-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-NEXT: ret
entry:
%c = fptrunc <2 x double> %a to <2 x half>
ret <2 x half> %c
@@ -334,19 +326,11 @@ entry:
}
define <2 x half> @fptrunc_v2f32_v2f16(<2 x float> %a) {
-; CHECK-SD-LABEL: fptrunc_v2f32_v2f16:
-; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT: fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: fptrunc_v2f32_v2f16:
-; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-GI-NEXT: fcvtn v0.4h, v1.4s
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: fptrunc_v2f32_v2f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-NEXT: ret
entry:
%c = fptrunc <2 x float> %a to <2 x half>
ret <2 x half> %c
diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll
index caf87a13f283b..6d168edf180a4 100644
--- a/llvm/test/CodeGen/AArch64/itofp.ll
+++ b/llvm/test/CodeGen/AArch64/itofp.ll
@@ -5763,18 +5763,14 @@ define <2 x half> @stofp_v2i64_v2f16(<2 x i64> %a) {
; CHECK-NOFP16-GI: // %bb.0: // %entry
; CHECK-NOFP16-GI-NEXT: scvtf v0.2d, v0.2d
; CHECK-NOFP16-GI-NEXT: fcvtn v0.2s, v0.2d
-; CHECK-NOFP16-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-NOFP16-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NOFP16-GI-NEXT: ret
;
; CHECK-FP16-GI-LABEL: stofp_v2i64_v2f16:
; CHECK-FP16-GI: // %bb.0: // %entry
; CHECK-FP16-GI-NEXT: scvtf v0.2d, v0.2d
; CHECK-FP16-GI-NEXT: fcvtn v0.2s, v0.2d
-; CHECK-FP16-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-FP16-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-FP16-GI-NEXT: fcvtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <2 x i64> %a to <2 x half>
@@ -5808,18 +5804,14 @@ define <2 x half> @utofp_v2i64_v2f16(<2 x i64> %a) {
; CHECK-NOFP16-GI: // %bb.0: // %entry
; CHECK-NOFP16-GI-NEXT: ucvtf v0.2d, v0.2d
; CHECK-NOFP16-GI-NEXT: fcvtn v0.2s, v0.2d
-; CHECK-NOFP16-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-NOFP16-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NOFP16-GI-NEXT: ret
;
; CHECK-FP16-GI-LABEL: utofp_v2i64_v2f16:
; CHECK-FP16-GI: // %bb.0: // %entry
; CHECK-FP16-GI-NEXT: ucvtf v0.2d, v0.2d
; CHECK-FP16-GI-NEXT: fcvtn v0.2s, v0.2d
-; CHECK-FP16-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-FP16-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-FP16-GI-NEXT: fcvtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <2 x i64> %a to <2 x half>
@@ -6232,17 +6224,13 @@ define <2 x half> @stofp_v2i32_v2f16(<2 x i32> %a) {
; CHECK-NOFP16-GI-LABEL: stofp_v2i32_v2f16:
; CHECK-NOFP16-GI: // %bb.0: // %entry
; CHECK-NOFP16-GI-NEXT: scvtf v0.2s, v0.2s
-; CHECK-NOFP16-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-NOFP16-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NOFP16-GI-NEXT: ret
;
; CHECK-FP16-GI-LABEL: stofp_v2i32_v2f16:
; CHECK-FP16-GI: // %bb.0: // %entry
; CHECK-FP16-GI-NEXT: scvtf v0.2s, v0.2s
-; CHECK-FP16-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-FP16-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-FP16-GI-NEXT: fcvtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%c = sitofp <2 x i32> %a to <2 x half>
@@ -6267,17 +6255,13 @@ define <2 x half> @utofp_v2i32_v2f16(<2 x i32> %a) {
; CHECK-NOFP16-GI-LABEL: utofp_v2i32_v2f16:
; CHECK-NOFP16-GI: // %bb.0: // %entry
; CHECK-NOFP16-GI-NEXT: ucvtf v0.2s, v0.2s
-; CHECK-NOFP16-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-NOFP16-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NOFP16-GI-NEXT: ret
;
; CHECK-FP16-GI-LABEL: utofp_v2i32_v2f16:
; CHECK-FP16-GI: // %bb.0: // %entry
; CHECK-FP16-GI-NEXT: ucvtf v0.2s, v0.2s
-; CHECK-FP16-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-FP16-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-FP16-GI-NEXT: fcvtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-FP16-GI-NEXT: ret
entry:
%c = uitofp <2 x i32> %a to <2 x half>
@@ -6480,9 +6464,7 @@ define <2 x half> @stofp_v2i16_v2f16(<2 x i16> %a) {
; CHECK-NOFP16-GI-NEXT: shl v0.2s, v0.2s, #16
; CHECK-NOFP16-GI-NEXT: sshr v0.2s, v0.2s, #16
; CHECK-NOFP16-GI-NEXT: scvtf v0.2s, v0.2s
-; CHECK-NOFP16-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-NOFP16-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = sitofp <2 x i16> %a to <2 x half>
@@ -6509,9 +6491,7 @@ define <2 x half> @utofp_v2i16_v2f16(<2 x i16> %a) {
; CHECK-NOFP16-GI-NEXT: movi d1, #0x00ffff0000ffff
; CHECK-NOFP16-GI-NEXT: and v0.8b, v0.8b, v1.8b
; CHECK-NOFP16-GI-NEXT: ucvtf v0.2s, v0.2s
-; CHECK-NOFP16-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-NOFP16-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NOFP16-GI-NEXT: ret
entry:
%c = uitofp <2 x i16> %a to <2 x half>
@@ -6766,9 +6746,7 @@ define <2 x half> @stofp_v2i8_v2f16(<2 x i8> %a) {
; CHECK-NOFP16-GI-NEXT: shl v0.2s, v0.2s, #24
; CHECK-NOFP16-GI-NEXT: sshr v0.2s, v0.2s, #24
; CHECK-NOFP16-GI-NEXT: scvtf v0.2s, v0.2s
-; CHECK-NOFP16-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-NOFP16-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NOFP16-GI-NEXT: ret
;
; CHECK-FP16-GI-LABEL: stofp_v2i8_v2f16:
@@ -6817,9 +6795,7 @@ define <2 x half> @utofp_v2i8_v2f16(<2 x i8> %a) {
; CHECK-NOFP16-GI-NEXT: movi d1, #0x0000ff000000ff
; CHECK-NOFP16-GI-NEXT: and v0.8b, v0.8b, v1.8b
; CHECK-NOFP16-GI-NEXT: ucvtf v0.2s, v0.2s
-; CHECK-NOFP16-GI-NEXT: mov v1.s[0], v0.s[0]
-; CHECK-NOFP16-GI-NEXT: mov v1.s[1], v0.s[1]
-; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NOFP16-GI-NEXT: ret
;
; CHECK-FP16-GI-LABEL: utofp_v2i8_v2f16:
>From 21007f63a418853d9625a547b30ca2e2989bacda Mon Sep 17 00:00:00 2001
From: Ryan Cowan <ryan.cowan at arm.com>
Date: Mon, 27 Oct 2025 15:06:55 +0000
Subject: [PATCH 09/12] Use set of sources
---
.../GISel/AArch64PostLegalizerCombiner.cpp | 21 ++++++++----
llvm/test/CodeGen/AArch64/arm64-neon-copy.ll | 32 ++++++-------------
2 files changed, 24 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
index 4cef523951af5..8c5db9059d5ae 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -43,7 +43,7 @@
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/Support/Debug.h"
#include <cassert>
-#include <iostream>
+#include <set>
#define GET_GICOMBINER_DEPS
#include "AArch64GenPostLegalizeGICombiner.inc"
@@ -146,6 +146,8 @@ bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
undefinedValues.clear();
unmergedValues.clear();
+ std::set<int> knownRegs;
+
for (auto Use : MI.all_uses()) {
auto *Def = getDefIgnoringCopies(Use.getReg(), MRI);
@@ -170,15 +172,19 @@ bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
unmergedValues.clear();
return false;
}
-
- unmergedValues.push_back(Def->getOperand(2).getReg());
+ if (knownRegs.find(Def->getOperand(2).getReg().id()) == knownRegs.end()) {
+ knownRegs.insert(Def->getOperand(2).getReg().id());
+ unmergedValues.push_back(Def->getOperand(2).getReg());
+ }
+ else
+ continue;
break;
}
}
- if (!(undefinedValues.size() == 2 && unmergedValues.size() == 2) &&
- !(undefinedValues.size() == 0 && unmergedValues.size() == 4)) {
+ if (!(undefinedValues.size() == 2 && unmergedValues.size() == 1) &&
+ !(undefinedValues.size() == 0 && unmergedValues.size() == 2)) {
undefinedValues.clear();
unmergedValues.clear();
return false;
@@ -193,7 +199,7 @@ void applyCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
SmallVectorImpl<Register> &undefinedValues) {
B.setInstrAndDebugLoc(MI);
- if (unmergedValues.size() == 2) {
+ if (unmergedValues.size() == 1) {
auto llt = LLT::fixed_vector(
undefinedValues.size(),
LLT::scalar(MRI.getType(unmergedValues[0]).getScalarSizeInBits()));
@@ -202,8 +208,9 @@ void applyCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
B.buildBuildVector(DefVec, undefinedValues);
B.buildConcatVectors(MI.getOperand(0), {unmergedValues[0], DefVec});
} else {
+ return;
B.buildConcatVectors(MI.getOperand(0),
- {unmergedValues[0], unmergedValues[3]});
+ {unmergedValues[0], unmergedValues[1]});
}
MI.eraseFromParent();
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
index 8e4f022776a57..d8f370884c84a 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
@@ -1268,17 +1268,11 @@ define <8 x i16> @testDUP.v1i16(<1 x i16> %a) {
}
define <4 x i32> @testDUP.v1i32(<1 x i32> %a) {
-; CHECK-SD-LABEL: testDUP.v1i32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT: dup v0.4s, v0.s[0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: testDUP.v1i32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: mov v0.d[1], v0.d[0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: testDUP.v1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: dup v0.4s, v0.s[0]
+; CHECK-NEXT: ret
%b = extractelement <1 x i32> %a, i32 0
%c = insertelement <4 x i32> undef, i32 %b, i32 0
%d = insertelement <4 x i32> %c, i32 %b, i32 1
@@ -2249,17 +2243,11 @@ define <4 x i16> @concat_vector_v4i16(<1 x i16> %a) {
}
define <4 x i32> @concat_vector_v4i32(<1 x i32> %a) {
-; CHECK-SD-LABEL: concat_vector_v4i32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT: dup v0.4s, v0.s[0]
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: concat_vector_v4i32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: mov v0.d[1], v0.d[0]
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: concat_vector_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: dup v0.4s, v0.s[0]
+; CHECK-NEXT: ret
%r = shufflevector <1 x i32> %a, <1 x i32> undef, <4 x i32> zeroinitializer
ret <4 x i32> %r
}
>From b8529694e5b24a0beedcc03c365b8e4372e892cc Mon Sep 17 00:00:00 2001
From: Ryan Cowan <ryan.cowan at arm.com>
Date: Mon, 27 Oct 2025 16:00:30 +0000
Subject: [PATCH 10/12] Simplify & rename variables
---
llvm/lib/Target/AArch64/AArch64Combine.td | 6 +--
.../GISel/AArch64PostLegalizerCombiner.cpp | 46 +++++++++----------
2 files changed, 25 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index db2e9007610e7..2850f995117a5 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -333,10 +333,10 @@ def combine_mul_cmlt : GICombineRule<
>;
def combine_build_unmerge : GICombineRule<
- (defs root:$root, register_vector_matchinfo:$unmergedValues, register_vector_matchinfo:$undefinedValues),
+ (defs root:$root, register_vector_matchinfo:$unmergeSrc, register_vector_matchinfo:$undefinedValues),
(match (wip_match_opcode G_BUILD_VECTOR):$root,
- [{ return matchCombineBuildUnmerge(*${root}, MRI, ${unmergedValues}, ${undefinedValues}); }]),
- (apply [{ applyCombineBuildUnmerge(*${root}, MRI, B, ${unmergedValues}, ${undefinedValues}); }])
+ [{ return matchCombineBuildUnmerge(*${root}, MRI, ${unmergeSrc}, ${undefinedValues}); }]),
+ (apply [{ applyCombineBuildUnmerge(*${root}, MRI, B, ${unmergeSrc}, ${undefinedValues}); }])
>;
// Post-legalization combines which should happen at all optimization levels.
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
index 8c5db9059d5ae..a7add5bd93eac 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -42,7 +42,6 @@
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/Support/Debug.h"
-#include <cassert>
#include <set>
#define GET_GICOMBINER_DEPS
@@ -139,12 +138,12 @@ bool isZeroExtended(Register R, MachineRegisterInfo &MRI) {
}
bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
- SmallVectorImpl<Register> &unmergedValues,
+ SmallVectorImpl<Register> &unmergeSrc,
SmallVectorImpl<Register> &undefinedValues) {
assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
undefinedValues.clear();
- unmergedValues.clear();
+ unmergeSrc.clear();
std::set<int> knownRegs;
@@ -153,7 +152,7 @@ bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
if (!Def) {
undefinedValues.clear();
- unmergedValues.clear();
+ unmergeSrc.clear();
return false;
}
@@ -166,27 +165,30 @@ bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
undefinedValues.push_back(Use.getReg());
break;
case TargetOpcode::G_UNMERGE_VALUES:
- if (Def->getNumDefs() > 2 ||
+ // We only want to match G_UNMERGE_VALUES <2 x Ty>
+ // s16 is troublesome as <2 x s16> is generally not legal
+ if (Def->getNumDefs() != 2 ||
MRI.getType(Use.getReg()) == LLT::scalar(16)) {
undefinedValues.clear();
- unmergedValues.clear();
+ unmergeSrc.clear();
return false;
}
- if (knownRegs.find(Def->getOperand(2).getReg().id()) == knownRegs.end()) {
- knownRegs.insert(Def->getOperand(2).getReg().id());
- unmergedValues.push_back(Def->getOperand(2).getReg());
- }
- else
- continue;
+ // Only track unique sources for the G_UNMERGE_VALUES
+ if (knownRegs.find(Def->getOperand(2).getReg().id()) != knownRegs.end())
+ continue;
+
+ knownRegs.insert(Def->getOperand(2).getReg().id());
+ unmergeSrc.push_back(Def->getOperand(2).getReg());
+
break;
}
}
- if (!(undefinedValues.size() == 2 && unmergedValues.size() == 1) &&
- !(undefinedValues.size() == 0 && unmergedValues.size() == 2)) {
+ // Only want to match patterns that pad two values with two undefined values
+ if (!(undefinedValues.size() == 2 && unmergeSrc.size() == 1)) {
undefinedValues.clear();
- unmergedValues.clear();
+ unmergeSrc.clear();
return false;
}
@@ -195,23 +197,19 @@ bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
void applyCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B,
- SmallVectorImpl<Register> &unmergedValues,
+ SmallVectorImpl<Register> &unmergeSrc,
SmallVectorImpl<Register> &undefinedValues) {
+ assert(unmergeSrc.size() == 1 && "Expected there to be one G_UNMERGE_VALUES");
B.setInstrAndDebugLoc(MI);
- if (unmergedValues.size() == 1) {
auto llt = LLT::fixed_vector(
undefinedValues.size(),
- LLT::scalar(MRI.getType(unmergedValues[0]).getScalarSizeInBits()));
+ LLT::scalar(MRI.getType(unmergeSrc[0]).getScalarSizeInBits()));
+
Register DefVec = MRI.createGenericVirtualRegister(llt);
B.buildBuildVector(DefVec, undefinedValues);
- B.buildConcatVectors(MI.getOperand(0), {unmergedValues[0], DefVec});
- } else {
- return;
- B.buildConcatVectors(MI.getOperand(0),
- {unmergedValues[0], unmergedValues[1]});
- }
+ B.buildConcatVectors(MI.getOperand(0), {unmergeSrc[0], DefVec});
MI.eraseFromParent();
}
>From 7742f134d45cb8282f4d457eec9b926e8e6ce38a Mon Sep 17 00:00:00 2001
From: Ryan Cowan <ryan.cowan at arm.com>
Date: Mon, 27 Oct 2025 16:00:37 +0000
Subject: [PATCH 11/12] Fix tests
---
.../GlobalISel/legalizer-info-validation.mir | 7 +-
.../match-table-cxx.td | 132 +++++++++---------
.../GlobalISelEmitter/GlobalISelEmitter.td | 2 +-
llvm/test/TableGen/get-named-operand-idx.td | 3 +-
4 files changed, 74 insertions(+), 70 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 0561f91b6e015..5207921a79e47 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -555,11 +555,14 @@
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FPEXT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. the first uncovered type index: 2, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
# DEBUG-NEXT: G_FPTRUNC (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: G_FPTRUNC_ODD (opcode 204): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. the first uncovered type index: 2, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
# DEBUG-NEXT: G_FPTOSI (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
index 18960b43ab97d..bf19f479b6104 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
@@ -96,71 +96,71 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK: const uint8_t *GenMyCombiner::getMatchTable() const {
// CHECK-NEXT: constexpr static uint8_t MatchTable0[] = {
-// CHECK-NEXT: /* 0 */ GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(99), GIMT_Encode2(211), /*)*//*default:*//*Label 5*/ GIMT_Encode4(524),
-// CHECK-NEXT: /* 10 */ /*TargetOpcode::G_STORE*//*Label 0*/ GIMT_Encode4(458), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT: /* 182 */ /*TargetOpcode::G_SEXT*//*Label 1*/ GIMT_Encode4(476), GIMT_Encode4(0),
-// CHECK-NEXT: /* 190 */ /*TargetOpcode::G_ZEXT*//*Label 2*/ GIMT_Encode4(488), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT: /* 418 */ /*TargetOpcode::G_FNEG*//*Label 3*/ GIMT_Encode4(500), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT: /* 454 */ /*TargetOpcode::G_FABS*//*Label 4*/ GIMT_Encode4(512),
-// CHECK-NEXT: /* 458 */ // Label 0: @458
-// CHECK-NEXT: /* 458 */ GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(475), // Rule ID 2 //
-// CHECK-NEXT: /* 463 */ GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
-// CHECK-NEXT: /* 466 */ // MIs[0] x
-// CHECK-NEXT: /* 466 */ // No operand predicates
-// CHECK-NEXT: /* 466 */ // MIs[0] y
-// CHECK-NEXT: /* 466 */ // No operand predicates
-// CHECK-NEXT: /* 466 */ GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner0),
-// CHECK-NEXT: /* 470 */ GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner1),
-// CHECK-NEXT: /* 474 */ // Combiner Rule #2: TwoMatchNoApply
-// CHECK-NEXT: /* 474 */ GIR_EraseRootFromParent_Done,
-// CHECK-NEXT: /* 475 */ // Label 6: @475
-// CHECK-NEXT: /* 475 */ GIM_Reject,
-// CHECK-NEXT: /* 476 */ // Label 1: @476
-// CHECK-NEXT: /* 476 */ GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(487), // Rule ID 3 //
-// CHECK-NEXT: /* 481 */ GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled),
-// CHECK-NEXT: /* 484 */ // MIs[0] a
-// CHECK-NEXT: /* 484 */ // No operand predicates
-// CHECK-NEXT: /* 484 */ // MIs[0] y
-// CHECK-NEXT: /* 484 */ // No operand predicates
-// CHECK-NEXT: /* 484 */ // Combiner Rule #3: NoMatchTwoApply
-// CHECK-NEXT: /* 484 */ GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner2),
-// CHECK-NEXT: /* 487 */ // Label 7: @487
-// CHECK-NEXT: /* 487 */ GIM_Reject,
-// CHECK-NEXT: /* 488 */ // Label 2: @488
-// CHECK-NEXT: /* 488 */ GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(499), // Rule ID 4 //
-// CHECK-NEXT: /* 493 */ GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule4Enabled),
-// CHECK-NEXT: /* 496 */ // MIs[0] a
-// CHECK-NEXT: /* 496 */ // No operand predicates
-// CHECK-NEXT: /* 496 */ // MIs[0] y
-// CHECK-NEXT: /* 496 */ // No operand predicates
-// CHECK-NEXT: /* 496 */ // Combiner Rule #4: CombineCXXOrder
-// CHECK-NEXT: /* 496 */ GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner3),
-// CHECK-NEXT: /* 499 */ // Label 8: @499
-// CHECK-NEXT: /* 499 */ GIM_Reject,
-// CHECK-NEXT: /* 500 */ // Label 3: @500
-// CHECK-NEXT: /* 500 */ GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(511), // Rule ID 1 //
-// CHECK-NEXT: /* 505 */ GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
-// CHECK-NEXT: /* 508 */ // MIs[0] a
-// CHECK-NEXT: /* 508 */ // No operand predicates
-// CHECK-NEXT: /* 508 */ // MIs[0] b
-// CHECK-NEXT: /* 508 */ // No operand predicates
-// CHECK-NEXT: /* 508 */ // Combiner Rule #1: TwoMatchTwoApply
-// CHECK-NEXT: /* 508 */ GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner1),
-// CHECK-NEXT: /* 511 */ // Label 9: @511
-// CHECK-NEXT: /* 511 */ GIM_Reject,
-// CHECK-NEXT: /* 512 */ // Label 4: @512
-// CHECK-NEXT: /* 512 */ GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(523), // Rule ID 0 //
-// CHECK-NEXT: /* 517 */ GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
-// CHECK-NEXT: /* 520 */ // MIs[0] a
-// CHECK-NEXT: /* 520 */ // No operand predicates
-// CHECK-NEXT: /* 520 */ // MIs[0] b
-// CHECK-NEXT: /* 520 */ // No operand predicates
-// CHECK-NEXT: /* 520 */ // Combiner Rule #0: OneMatchOneApply
-// CHECK-NEXT: /* 520 */ GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT: /* 523 */ // Label 10: @523
-// CHECK-NEXT: /* 523 */ GIM_Reject,
-// CHECK-NEXT: /* 524 */ // Label 5: @524
-// CHECK-NEXT: /* 524 */ GIM_Reject,
-// CHECK-NEXT: /* 525 */ }; // Size: 525 bytes
+// CHECK-NEXT: /* 0 */ GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(99), GIMT_Encode2(212), /*)*//*default:*//*Label 5*/ GIMT_Encode4(528),
+// CHECK-NEXT: /* 10 */ /*TargetOpcode::G_STORE*//*Label 0*/ GIMT_Encode4(462), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT: /* 182 */ /*TargetOpcode::G_SEXT*//*Label 1*/ GIMT_Encode4(480), GIMT_Encode4(0),
+// CHECK-NEXT: /* 190 */ /*TargetOpcode::G_ZEXT*//*Label 2*/ GIMT_Encode4(492), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT: /* 418 */ /*TargetOpcode::G_FNEG*//*Label 3*/ GIMT_Encode4(504), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT: /* 458 */ /*TargetOpcode::G_FABS*//*Label 4*/ GIMT_Encode4(516),
+// CHECK-NEXT: /* 462 */ // Label 0: @462
+// CHECK-NEXT: /* 462 */ GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(479), // Rule ID 2 //
+// CHECK-NEXT: /* 467 */ GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
+// CHECK-NEXT: /* 470 */ // MIs[0] x
+// CHECK-NEXT: /* 470 */ // No operand predicates
+// CHECK-NEXT: /* 470 */ // MIs[0] y
+// CHECK-NEXT: /* 470 */ // No operand predicates
+// CHECK-NEXT: /* 470 */ GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner0),
+// CHECK-NEXT: /* 474 */ GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner1),
+// CHECK-NEXT: /* 478 */ // Combiner Rule #2: TwoMatchNoApply
+// CHECK-NEXT: /* 478 */ GIR_EraseRootFromParent_Done,
+// CHECK-NEXT: /* 479 */ // Label 6: @479
+// CHECK-NEXT: /* 479 */ GIM_Reject,
+// CHECK-NEXT: /* 480 */ // Label 1: @480
+// CHECK-NEXT: /* 480 */ GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(491), // Rule ID 3 //
+// CHECK-NEXT: /* 485 */ GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled),
+// CHECK-NEXT: /* 488 */ // MIs[0] a
+// CHECK-NEXT: /* 488 */ // No operand predicates
+// CHECK-NEXT: /* 488 */ // MIs[0] y
+// CHECK-NEXT: /* 488 */ // No operand predicates
+// CHECK-NEXT: /* 488 */ // Combiner Rule #3: NoMatchTwoApply
+// CHECK-NEXT: /* 488 */ GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner2),
+// CHECK-NEXT: /* 491 */ // Label 7: @491
+// CHECK-NEXT: /* 491 */ GIM_Reject,
+// CHECK-NEXT: /* 492 */ // Label 2: @492
+// CHECK-NEXT: /* 492 */ GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(503), // Rule ID 4 //
+// CHECK-NEXT: /* 497 */ GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule4Enabled),
+// CHECK-NEXT: /* 500 */ // MIs[0] a
+// CHECK-NEXT: /* 500 */ // No operand predicates
+// CHECK-NEXT: /* 500 */ // MIs[0] y
+// CHECK-NEXT: /* 500 */ // No operand predicates
+// CHECK-NEXT: /* 500 */ // Combiner Rule #4: CombineCXXOrder
+// CHECK-NEXT: /* 500 */ GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner3),
+// CHECK-NEXT: /* 503 */ // Label 8: @503
+// CHECK-NEXT: /* 503 */ GIM_Reject,
+// CHECK-NEXT: /* 504 */ // Label 3: @504
+// CHECK-NEXT: /* 504 */ GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(515), // Rule ID 1 //
+// CHECK-NEXT: /* 509 */ GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
+// CHECK-NEXT: /* 512 */ // MIs[0] a
+// CHECK-NEXT: /* 512 */ // No operand predicates
+// CHECK-NEXT: /* 512 */ // MIs[0] b
+// CHECK-NEXT: /* 512 */ // No operand predicates
+// CHECK-NEXT: /* 512 */ // Combiner Rule #1: TwoMatchTwoApply
+// CHECK-NEXT: /* 512 */ GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner1),
+// CHECK-NEXT: /* 515 */ // Label 9: @515
+// CHECK-NEXT: /* 515 */ GIM_Reject,
+// CHECK-NEXT: /* 516 */ // Label 4: @516
+// CHECK-NEXT: /* 516 */ GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(527), // Rule ID 0 //
+// CHECK-NEXT: /* 521 */ GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
+// CHECK-NEXT: /* 524 */ // MIs[0] a
+// CHECK-NEXT: /* 524 */ // No operand predicates
+// CHECK-NEXT: /* 524 */ // MIs[0] b
+// CHECK-NEXT: /* 524 */ // No operand predicates
+// CHECK-NEXT: /* 524 */ // Combiner Rule #0: OneMatchOneApply
+// CHECK-NEXT: /* 524 */ GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT: /* 527 */ // Label 10: @527
+// CHECK-NEXT: /* 527 */ GIM_Reject,
+// CHECK-NEXT: /* 528 */ // Label 5: @528
+// CHECK-NEXT: /* 528 */ GIM_Reject,
+// CHECK-NEXT: /* 529 */ }; // Size: 529 bytes
// CHECK-NEXT: return MatchTable0;
// CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
index fdabc53a3ff3b..905e16f504048 100644
--- a/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
@@ -535,7 +535,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3),
// R00O-NEXT: GIM_Reject,
// R00O: // Label [[DEFAULT_NUM]]: @[[DEFAULT]]
// R00O-NEXT: GIM_Reject,
-// R00O-NEXT: }; // Size: 1902 bytes
+// R00O-NEXT: }; // Size: 1906 bytes
def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4),
[(set GPR32:$dst,
diff --git a/llvm/test/TableGen/get-named-operand-idx.td b/llvm/test/TableGen/get-named-operand-idx.td
index e6f6331cd9c48..a10cdd9696a4e 100644
--- a/llvm/test/TableGen/get-named-operand-idx.td
+++ b/llvm/test/TableGen/get-named-operand-idx.td
@@ -89,7 +89,8 @@ def InstD : InstBase {
// CHECK-NEXT: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
// CHECK-NEXT: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
// CHECK-NEXT: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0,
+// CHECK-NEXT: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2,
+// CHECK-NEXT: 0,
// CHECK-NEXT: };
// CHECK-NEXT: return InstructionIndex[Opcode];
// CHECK-NEXT: }
>From 0efa36673fd9531dfca79759a15602fdb606f5b8 Mon Sep 17 00:00:00 2001
From: Ryan Cowan <ryan.cowan at arm.com>
Date: Mon, 27 Oct 2025 16:04:29 +0000
Subject: [PATCH 12/12] Add comment
---
.../GISel/AArch64PostLegalizerCombiner.cpp | 70 +++++++++++--------
.../GlobalISel/legalizer-info-validation.mir | 3 +
2 files changed, 45 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
index a7add5bd93eac..c64bee9d00d08 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -137,22 +137,37 @@ bool isZeroExtended(Register R, MachineRegisterInfo &MRI) {
return MRI.getVRegDef(R)->getOpcode() == TargetOpcode::G_ZEXT;
}
+// This patten aims to match the following shape to avoid extra mov instructions
+// G_BUILD_VECTOR(
+// G_UNMERGE_VALUES(src, 0)
+// G_UNMERGE_VALUES(src, 1)
+// G_IMPLICIT_DEF
+// G_IMPLICIT_DEF
+// )
+// ->
+// G_CONCAT_VECTORS(
+// G_BUILD_VECTOR(
+// G_IMPLICIT_DEF
+// G_IMPLICIT_DEF
+// )
+// src
+// )
bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
- SmallVectorImpl<Register> &unmergeSrc,
- SmallVectorImpl<Register> &undefinedValues) {
+ SmallVectorImpl<Register> &UnmergeSrc,
+ SmallVectorImpl<Register> &UndefinedValues) {
assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
- undefinedValues.clear();
- unmergeSrc.clear();
+ UndefinedValues.clear();
+ UnmergeSrc.clear();
- std::set<int> knownRegs;
+ std::set<int> KnownRegs;
for (auto Use : MI.all_uses()) {
auto *Def = getDefIgnoringCopies(Use.getReg(), MRI);
if (!Def) {
- undefinedValues.clear();
- unmergeSrc.clear();
+ UndefinedValues.clear();
+ UnmergeSrc.clear();
return false;
}
@@ -162,33 +177,33 @@ bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
default:
return false;
case TargetOpcode::G_IMPLICIT_DEF:
- undefinedValues.push_back(Use.getReg());
+ UndefinedValues.push_back(Use.getReg());
break;
case TargetOpcode::G_UNMERGE_VALUES:
// We only want to match G_UNMERGE_VALUES <2 x Ty>
// s16 is troublesome as <2 x s16> is generally not legal
if (Def->getNumDefs() != 2 ||
MRI.getType(Use.getReg()) == LLT::scalar(16)) {
- undefinedValues.clear();
- unmergeSrc.clear();
+ UndefinedValues.clear();
+ UnmergeSrc.clear();
return false;
}
// Only track unique sources for the G_UNMERGE_VALUES
- if (knownRegs.find(Def->getOperand(2).getReg().id()) != knownRegs.end())
- continue;
+ if (KnownRegs.find(Def->getOperand(2).getReg().id()) != KnownRegs.end())
+ continue;
+
+ KnownRegs.insert(Def->getOperand(2).getReg().id());
+ UnmergeSrc.push_back(Def->getOperand(2).getReg());
- knownRegs.insert(Def->getOperand(2).getReg().id());
- unmergeSrc.push_back(Def->getOperand(2).getReg());
-
break;
}
}
// Only want to match patterns that pad two values with two undefined values
- if (!(undefinedValues.size() == 2 && unmergeSrc.size() == 1)) {
- undefinedValues.clear();
- unmergeSrc.clear();
+ if (!(UndefinedValues.size() == 2 && UnmergeSrc.size() == 1)) {
+ UndefinedValues.clear();
+ UnmergeSrc.clear();
return false;
}
@@ -197,19 +212,18 @@ bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
void applyCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B,
- SmallVectorImpl<Register> &unmergeSrc,
- SmallVectorImpl<Register> &undefinedValues) {
- assert(unmergeSrc.size() == 1 && "Expected there to be one G_UNMERGE_VALUES");
+ SmallVectorImpl<Register> &UnmergeSrc,
+ SmallVectorImpl<Register> &UndefinedValues) {
+ assert(UnmergeSrc.size() == 1 && "Expected there to be one G_UNMERGE_VALUES");
B.setInstrAndDebugLoc(MI);
- auto llt = LLT::fixed_vector(
- undefinedValues.size(),
- LLT::scalar(MRI.getType(unmergeSrc[0]).getScalarSizeInBits()));
-
- Register DefVec = MRI.createGenericVirtualRegister(llt);
+ auto UndefVecLLT = LLT::fixed_vector(
+ UndefinedValues.size(),
+ LLT::scalar(MRI.getType(UnmergeSrc[0]).getScalarSizeInBits()));
+ Register UndefVec = MRI.createGenericVirtualRegister(UndefVecLLT);
- B.buildBuildVector(DefVec, undefinedValues);
- B.buildConcatVectors(MI.getOperand(0), {unmergeSrc[0], DefVec});
+ B.buildBuildVector(UndefVec, UndefinedValues);
+ B.buildConcatVectors(MI.getOperand(0), {UnmergeSrc[0], UndefVec});
MI.eraseFromParent();
}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
index da7546e12e58b..d3694c02bab78 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
@@ -556,6 +556,9 @@
# DEBUG-NEXT: G_FPTRUNC (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. the first uncovered type index: 2, OK
# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: G_FPTRUNC_ODD (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_FPTOSI (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. the first uncovered type index: 2, OK
# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
More information about the llvm-commits
mailing list