[llvm] [X86] Lower scalar integer div/rem to FP division (PR #216589)
Ankit Kumar Tiwari via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 10:14:23 PDT 2026
https://github.com/ankit-cybertron updated https://github.com/llvm/llvm-project/pull/216589
>From c357a0f0d52fc616e2eab2e9b1dbda78eac7bae4 Mon Sep 17 00:00:00 2001
From: Ankit Kumar Tiwari <ankit.cybertron at gmail.com>
Date: Sun, 16 Aug 2026 22:43:07 +0530
Subject: [PATCH 1/5] Baseline Test
---
llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll | 119 +++++++++++++++++++
1 file changed, 119 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll
diff --git a/llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll b/llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll
new file mode 100644
index 0000000000000..10100c65cab3f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll
@@ -0,0 +1,119 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s
+
+define i32 @sdiv_i32(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: sdiv_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: cltd
+; CHECK-NEXT: idivl %esi
+; CHECK-NEXT: retq
+ %r = sdiv i32 %a, %b
+ ret i32 %r
+}
+
+define i16 @sdiv_i16(i16 %a, i16 %b) nounwind {
+; CHECK-LABEL: sdiv_i16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: cwtd
+; CHECK-NEXT: idivw %si
+; CHECK-NEXT: retq
+ %r = sdiv i16 %a, %b
+ ret i16 %r
+}
+
+define i8 @sdiv_i8(i8 %a, i8 %b) nounwind {
+; CHECK-LABEL: sdiv_i8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movsbl %dil, %eax
+; CHECK-NEXT: idivb %sil
+; CHECK-NEXT: retq
+ %r = sdiv i8 %a, %b
+ ret i8 %r
+}
+
+define i32 @udiv_i32(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: udiv_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %esi
+; CHECK-NEXT: retq
+ %r = udiv i32 %a, %b
+ ret i32 %r
+}
+
+define i16 @udiv_i16(i16 %a, i16 %b) nounwind {
+; CHECK-LABEL: udiv_i16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divw %si
+; CHECK-NEXT: retq
+ %r = udiv i16 %a, %b
+ ret i16 %r
+}
+
+define i8 @udiv_i8(i8 %a, i8 %b) nounwind {
+; CHECK-LABEL: udiv_i8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movzbl %dil, %eax
+; CHECK-NEXT: divb %sil
+; CHECK-NEXT: retq
+ %r = udiv i8 %a, %b
+ ret i8 %r
+}
+
+define i32 @urem_i32(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: urem_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: divl %esi
+; CHECK-NEXT: movl %edx, %eax
+; CHECK-NEXT: retq
+ %r = urem i32 %a, %b
+ ret i32 %r
+}
+
+define i64 @sdiv_i64(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: sdiv_i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: cqto
+; CHECK-NEXT: idivq %rsi
+; CHECK-NEXT: retq
+ %r = sdiv i64 %a, %b
+ ret i64 %r
+}
+
+define i32 @sdiv_i32_const(i32 %a) nounwind {
+; CHECK-LABEL: sdiv_i32_const:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movslq %edi, %rax
+; CHECK-NEXT: imulq $-1840700269, %rax, %rcx # imm = 0x92492493
+; CHECK-NEXT: shrq $32, %rcx
+; CHECK-NEXT: addl %ecx, %eax
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: shrl $31, %ecx
+; CHECK-NEXT: sarl $2, %eax
+; CHECK-NEXT: addl %ecx, %eax
+; CHECK-NEXT: # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT: retq
+ %r = sdiv i32 %a, 7
+ ret i32 %r
+}
+
+define i32 @sdiv_i32_noimplicitfloat(i32 %a, i32 %b) nounwind noimplicitfloat {
+; CHECK-LABEL: sdiv_i32_noimplicitfloat:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: cltd
+; CHECK-NEXT: idivl %esi
+; CHECK-NEXT: retq
+ %r = sdiv i32 %a, %b
+ ret i32 %r
+}
>From 16b6894b87f43e531c8768e3f27eb0d85d8a5cd8 Mon Sep 17 00:00:00 2001
From: Ankit Kumar Tiwari <ankit.cybertron at gmail.com>
Date: Sun, 16 Aug 2026 23:12:51 +0530
Subject: [PATCH 2/5] [X86] Lower scalar i8/i16/i32 integer div/rem to FP
division
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 34 ++++++++----
llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll | 58 ++++++++++++++------
2 files changed, 66 insertions(+), 26 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 0dc3340a13c73..1f10178fabd72 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50720,8 +50720,8 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
EVT VT = N->getValueType(0);
SDLoc DL(N);
- // Vector division never survives op legalization to reach later rounds.
- if (!VT.isVector() || !Subtarget.hasSSE2())
+
+ if (!Subtarget.hasSSE2())
return SDValue();
SDValue Dividend = N->getOperand(0);
@@ -50735,9 +50735,17 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
bool IsRem = Opc == ISD::UREM || Opc == ISD::SREM;
bool IsSigned = Opc == ISD::SDIV || Opc == ISD::SREM;
+ // Scalar functions that do not use XMM registers must not get this combine.
+ if (!VT.isVector() &&
+ (Subtarget.useSoftFloat() ||
+ DAG.getMachineFunction().getFunction().hasFnAttribute(
+ Attribute::NoImplicitFloat)))
+ return SDValue();
+
// If the result is only read back as scalar extracts, scalarization computes
// just the demanded lanes.
- if (all_of(N->users(), [](const SDNode *U) {
+ if (VT.isVector() &&
+ all_of(N->users(), [](const SDNode *U) {
return U->getOpcode() == ISD::EXTRACT_VECTOR_ELT;
}))
return SDValue();
@@ -50764,16 +50772,21 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
MVT FPSclVT = MVT::f64;
if (EltBits <= 16 || BothFitFP(APFloat::IEEEsingle()))
FPSclVT = MVT::f32;
- EVT FPVT = VT.changeVectorElementType(*DAG.getContext(), FPSclVT);
+ EVT FPVT = VT.isVector()
+ ? VT.changeVectorElementType(*DAG.getContext(), FPSclVT)
+ : EVT(FPSclVT);
bool IsStrict = DAG.getMachineFunction().getFunction().hasFnAttribute(
Attribute::StrictFP);
if (IsStrict) {
+ // Scalar strictfp support is not implemented yet.
+ if (!VT.isVector())
+ return SDValue();
// The SAE forms are 512-bit only. Inputs widen into a zmm below, which
// requires 512-bit types to be legal and a power of 2 lane count.
if (!Subtarget.useAVX512Regs() || !isPowerOf2_32(VT.getVectorNumElements()))
return SDValue();
- } else if (!IsSigned && VT.getScalarSizeInBits() == 32 &&
+ } else if (VT.isVector() && !IsSigned && VT.getScalarSizeInBits() == 32 &&
!Subtarget.hasAVX2()) {
// Unsigned i32 needs FP_TO_UINT(f64->u32) which is emulated and a loss
// for latency and code size before AVX2.
@@ -50781,11 +50794,12 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
}
// Nothing will split an illegal FP type after type legalization and the
- // strict SAE divide is 512-bit only.
- bool FPVTUsable = IsStrict
- ? FPVT.getSizeInBits() <= 512
- : DCI.isBeforeLegalize() ||
- DAG.getTargetLoweringInfo().isTypeLegal(FPVT);
+ // strict SAE divide is 512-bit only. Scalar f32/f64 are always legal.
+ bool FPVTUsable = !VT.isVector() ||
+ (IsStrict
+ ? FPVT.getSizeInBits() <= 512
+ : DCI.isBeforeLegalize() ||
+ DAG.getTargetLoweringInfo().isTypeLegal(FPVT));
// Halve the divide while the integer halves stay legal.
if (!FPVTUsable) {
diff --git a/llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll b/llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll
index 10100c65cab3f..c752a26703275 100644
--- a/llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll
+++ b/llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll
@@ -4,9 +4,10 @@
define i32 @sdiv_i32(i32 %a, i32 %b) nounwind {
; CHECK-LABEL: sdiv_i32:
; CHECK: # %bb.0:
-; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: cltd
-; CHECK-NEXT: idivl %esi
+; CHECK-NEXT: vcvtsi2sd %esi, %xmm15, %xmm0
+; CHECK-NEXT: vcvtsi2sd %edi, %xmm15, %xmm1
+; CHECK-NEXT: vdivsd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vcvttsd2si %xmm0, %eax
; CHECK-NEXT: retq
%r = sdiv i32 %a, %b
ret i32 %r
@@ -15,10 +16,13 @@ define i32 @sdiv_i32(i32 %a, i32 %b) nounwind {
define i16 @sdiv_i16(i16 %a, i16 %b) nounwind {
; CHECK-LABEL: sdiv_i16:
; CHECK: # %bb.0:
-; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: movswl %si, %eax
+; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm0
+; CHECK-NEXT: movswl %di, %eax
+; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm1
+; CHECK-NEXT: vdivss %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vcvttss2si %xmm0, %eax
; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: cwtd
-; CHECK-NEXT: idivw %si
; CHECK-NEXT: retq
%r = sdiv i16 %a, %b
ret i16 %r
@@ -27,8 +31,13 @@ define i16 @sdiv_i16(i16 %a, i16 %b) nounwind {
define i8 @sdiv_i8(i8 %a, i8 %b) nounwind {
; CHECK-LABEL: sdiv_i8:
; CHECK: # %bb.0:
+; CHECK-NEXT: movsbl %sil, %eax
+; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm0
; CHECK-NEXT: movsbl %dil, %eax
-; CHECK-NEXT: idivb %sil
+; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm1
+; CHECK-NEXT: vdivss %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vcvttss2si %xmm0, %eax
+; CHECK-NEXT: # kill: def $al killed $al killed $eax
; CHECK-NEXT: retq
%r = sdiv i8 %a, %b
ret i8 %r
@@ -37,9 +46,13 @@ define i8 @sdiv_i8(i8 %a, i8 %b) nounwind {
define i32 @udiv_i32(i32 %a, i32 %b) nounwind {
; CHECK-LABEL: udiv_i32:
; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: vcvtsi2sd %rax, %xmm15, %xmm0
; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %esi
+; CHECK-NEXT: vcvtsi2sd %rax, %xmm15, %xmm1
+; CHECK-NEXT: vdivsd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vcvttsd2si %xmm0, %rax
+; CHECK-NEXT: # kill: def $eax killed $eax killed $rax
; CHECK-NEXT: retq
%r = udiv i32 %a, %b
ret i32 %r
@@ -48,10 +61,13 @@ define i32 @udiv_i32(i32 %a, i32 %b) nounwind {
define i16 @udiv_i16(i16 %a, i16 %b) nounwind {
; CHECK-LABEL: udiv_i16:
; CHECK: # %bb.0:
-; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: movzwl %si, %eax
+; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm0
+; CHECK-NEXT: movzwl %di, %eax
+; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm1
+; CHECK-NEXT: vdivss %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vcvttss2si %xmm0, %eax
; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divw %si
; CHECK-NEXT: retq
%r = udiv i16 %a, %b
ret i16 %r
@@ -60,8 +76,13 @@ define i16 @udiv_i16(i16 %a, i16 %b) nounwind {
define i8 @udiv_i8(i8 %a, i8 %b) nounwind {
; CHECK-LABEL: udiv_i8:
; CHECK: # %bb.0:
+; CHECK-NEXT: movzbl %sil, %eax
+; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm0
; CHECK-NEXT: movzbl %dil, %eax
-; CHECK-NEXT: divb %sil
+; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm1
+; CHECK-NEXT: vdivss %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vcvttss2si %xmm0, %eax
+; CHECK-NEXT: # kill: def $al killed $al killed $eax
; CHECK-NEXT: retq
%r = udiv i8 %a, %b
ret i8 %r
@@ -71,9 +92,14 @@ define i32 @urem_i32(i32 %a, i32 %b) nounwind {
; CHECK-LABEL: urem_i32:
; CHECK: # %bb.0:
; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: divl %esi
-; CHECK-NEXT: movl %edx, %eax
+; CHECK-NEXT: movl %esi, %ecx
+; CHECK-NEXT: vcvtsi2sd %rcx, %xmm15, %xmm0
+; CHECK-NEXT: movl %edi, %ecx
+; CHECK-NEXT: vcvtsi2sd %rcx, %xmm15, %xmm1
+; CHECK-NEXT: vdivsd %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vcvttsd2si %xmm0, %rcx
+; CHECK-NEXT: imull %esi, %ecx
+; CHECK-NEXT: subl %ecx, %eax
; CHECK-NEXT: retq
%r = urem i32 %a, %b
ret i32 %r
>From cc7692dc738def9c8dbc3e4a0b48b77624ad7090 Mon Sep 17 00:00:00 2001
From: Ankit Kumar Tiwari <ankit.cybertron at gmail.com>
Date: Tue, 18 Aug 2026 01:19:49 +0530
Subject: [PATCH 3/5] Fix scalar test parameter extensions
---
llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll | 112 +++++++++----------
1 file changed, 52 insertions(+), 60 deletions(-)
diff --git a/llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll b/llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll
index c752a26703275..d1b9d041944fa 100644
--- a/llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll
+++ b/llvm/test/CodeGen/X86/scalar-intdiv-to-fp.ll
@@ -13,36 +13,6 @@ define i32 @sdiv_i32(i32 %a, i32 %b) nounwind {
ret i32 %r
}
-define i16 @sdiv_i16(i16 %a, i16 %b) nounwind {
-; CHECK-LABEL: sdiv_i16:
-; CHECK: # %bb.0:
-; CHECK-NEXT: movswl %si, %eax
-; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm0
-; CHECK-NEXT: movswl %di, %eax
-; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm1
-; CHECK-NEXT: vdivss %xmm0, %xmm1, %xmm0
-; CHECK-NEXT: vcvttss2si %xmm0, %eax
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: retq
- %r = sdiv i16 %a, %b
- ret i16 %r
-}
-
-define i8 @sdiv_i8(i8 %a, i8 %b) nounwind {
-; CHECK-LABEL: sdiv_i8:
-; CHECK: # %bb.0:
-; CHECK-NEXT: movsbl %sil, %eax
-; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm0
-; CHECK-NEXT: movsbl %dil, %eax
-; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm1
-; CHECK-NEXT: vdivss %xmm0, %xmm1, %xmm0
-; CHECK-NEXT: vcvttss2si %xmm0, %eax
-; CHECK-NEXT: # kill: def $al killed $al killed $eax
-; CHECK-NEXT: retq
- %r = sdiv i8 %a, %b
- ret i8 %r
-}
-
define i32 @udiv_i32(i32 %a, i32 %b) nounwind {
; CHECK-LABEL: udiv_i32:
; CHECK: # %bb.0:
@@ -58,36 +28,6 @@ define i32 @udiv_i32(i32 %a, i32 %b) nounwind {
ret i32 %r
}
-define i16 @udiv_i16(i16 %a, i16 %b) nounwind {
-; CHECK-LABEL: udiv_i16:
-; CHECK: # %bb.0:
-; CHECK-NEXT: movzwl %si, %eax
-; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm0
-; CHECK-NEXT: movzwl %di, %eax
-; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm1
-; CHECK-NEXT: vdivss %xmm0, %xmm1, %xmm0
-; CHECK-NEXT: vcvttss2si %xmm0, %eax
-; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT: retq
- %r = udiv i16 %a, %b
- ret i16 %r
-}
-
-define i8 @udiv_i8(i8 %a, i8 %b) nounwind {
-; CHECK-LABEL: udiv_i8:
-; CHECK: # %bb.0:
-; CHECK-NEXT: movzbl %sil, %eax
-; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm0
-; CHECK-NEXT: movzbl %dil, %eax
-; CHECK-NEXT: vcvtsi2ss %eax, %xmm15, %xmm1
-; CHECK-NEXT: vdivss %xmm0, %xmm1, %xmm0
-; CHECK-NEXT: vcvttss2si %xmm0, %eax
-; CHECK-NEXT: # kill: def $al killed $al killed $eax
-; CHECK-NEXT: retq
- %r = udiv i8 %a, %b
- ret i8 %r
-}
-
define i32 @urem_i32(i32 %a, i32 %b) nounwind {
; CHECK-LABEL: urem_i32:
; CHECK: # %bb.0:
@@ -143,3 +83,55 @@ define i32 @sdiv_i32_noimplicitfloat(i32 %a, i32 %b) nounwind noimplicitfloat {
%r = sdiv i32 %a, %b
ret i32 %r
}
+
+define signext i8 @sdiv_i8(i8 signext %a, i8 signext %b) nounwind {
+; CHECK-LABEL: sdiv_i8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vcvtsi2ss %esi, %xmm15, %xmm0
+; CHECK-NEXT: vcvtsi2ss %edi, %xmm15, %xmm1
+; CHECK-NEXT: vdivss %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vcvttss2si %xmm0, %eax
+; CHECK-NEXT: # kill: def $al killed $al killed $eax
+; CHECK-NEXT: retq
+ %r = sdiv i8 %a, %b
+ ret i8 %r
+}
+
+define zeroext i8 @udiv_i8(i8 zeroext %a, i8 zeroext %b) nounwind {
+; CHECK-LABEL: udiv_i8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vcvtsi2ss %esi, %xmm15, %xmm0
+; CHECK-NEXT: vcvtsi2ss %edi, %xmm15, %xmm1
+; CHECK-NEXT: vdivss %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vcvttss2si %xmm0, %eax
+; CHECK-NEXT: # kill: def $al killed $al killed $eax
+; CHECK-NEXT: retq
+ %r = udiv i8 %a, %b
+ ret i8 %r
+}
+
+define signext i16 @sdiv_i16(i16 signext %a, i16 signext %b) nounwind {
+; CHECK-LABEL: sdiv_i16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vcvtsi2ss %esi, %xmm15, %xmm0
+; CHECK-NEXT: vcvtsi2ss %edi, %xmm15, %xmm1
+; CHECK-NEXT: vdivss %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vcvttss2si %xmm0, %eax
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: retq
+ %r = sdiv i16 %a, %b
+ ret i16 %r
+}
+
+define zeroext i16 @udiv_i16(i16 zeroext %a, i16 zeroext %b) nounwind {
+; CHECK-LABEL: udiv_i16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vcvtsi2ss %esi, %xmm15, %xmm0
+; CHECK-NEXT: vcvtsi2ss %edi, %xmm15, %xmm1
+; CHECK-NEXT: vdivss %xmm0, %xmm1, %xmm0
+; CHECK-NEXT: vcvttss2si %xmm0, %eax
+; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: retq
+ %r = udiv i16 %a, %b
+ ret i16 %r
+}
>From 372715baff083a282282ef7cd167376bfa74190b Mon Sep 17 00:00:00 2001
From: Ankit Kumar Tiwari <ankit.cybertron at gmail.com>
Date: Tue, 18 Aug 2026 01:26:07 +0530
Subject: [PATCH 4/5] Fix formatting
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 4038 ++++++++++++-----------
1 file changed, 2041 insertions(+), 1997 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 1f10178fabd72..b81e09bf5226d 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -189,10 +189,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// We don't accept any truncstore of integer registers.
setTruncStoreAction(MVT::i64, MVT::i32, Expand);
setTruncStoreAction(MVT::i64, MVT::i16, Expand);
- setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
+ setTruncStoreAction(MVT::i64, MVT::i8, Expand);
setTruncStoreAction(MVT::i32, MVT::i16, Expand);
- setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
- setTruncStoreAction(MVT::i16, MVT::i8, Expand);
+ setTruncStoreAction(MVT::i32, MVT::i8, Expand);
+ setTruncStoreAction(MVT::i16, MVT::i8, Expand);
setTruncStoreAction(MVT::f64, MVT::f32, Expand);
@@ -204,106 +204,106 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// Integer absolute.
if (Subtarget.canUseCMOV()) {
- setOperationAction(ISD::ABS , MVT::i16 , Custom);
- setOperationAction(ISD::ABS , MVT::i32 , Custom);
+ setOperationAction(ISD::ABS, MVT::i16, Custom);
+ setOperationAction(ISD::ABS, MVT::i32, Custom);
if (Subtarget.is64Bit())
- setOperationAction(ISD::ABS , MVT::i64 , Custom);
+ setOperationAction(ISD::ABS, MVT::i64, Custom);
}
// Absolute difference.
for (auto Op : {ISD::ABDS, ISD::ABDU}) {
- setOperationAction(Op , MVT::i8 , Custom);
- setOperationAction(Op , MVT::i16 , Custom);
- setOperationAction(Op , MVT::i32 , Custom);
+ setOperationAction(Op, MVT::i8, Custom);
+ setOperationAction(Op, MVT::i16, Custom);
+ setOperationAction(Op, MVT::i32, Custom);
if (Subtarget.is64Bit())
- setOperationAction(Op , MVT::i64 , Custom);
+ setOperationAction(Op, MVT::i64, Custom);
}
// Signed saturation subtraction.
- setOperationAction(ISD::SSUBSAT , MVT::i8 , Custom);
- setOperationAction(ISD::SSUBSAT , MVT::i16 , Custom);
- setOperationAction(ISD::SSUBSAT , MVT::i32 , Custom);
+ setOperationAction(ISD::SSUBSAT, MVT::i8, Custom);
+ setOperationAction(ISD::SSUBSAT, MVT::i16, Custom);
+ setOperationAction(ISD::SSUBSAT, MVT::i32, Custom);
if (Subtarget.is64Bit())
- setOperationAction(ISD::SSUBSAT , MVT::i64 , Custom);
+ setOperationAction(ISD::SSUBSAT, MVT::i64, Custom);
// Funnel shifts.
for (auto ShiftOp : {ISD::FSHL, ISD::FSHR}) {
// For slow shld targets we only lower for code size.
LegalizeAction ShiftDoubleAction = Subtarget.isSHLDSlow() ? Custom : Legal;
- setOperationAction(ShiftOp , MVT::i8 , Custom);
- setOperationAction(ShiftOp , MVT::i16 , Custom);
- setOperationAction(ShiftOp , MVT::i32 , ShiftDoubleAction);
+ setOperationAction(ShiftOp, MVT::i8, Custom);
+ setOperationAction(ShiftOp, MVT::i16, Custom);
+ setOperationAction(ShiftOp, MVT::i32, ShiftDoubleAction);
if (Subtarget.is64Bit())
- setOperationAction(ShiftOp , MVT::i64 , ShiftDoubleAction);
+ setOperationAction(ShiftOp, MVT::i64, ShiftDoubleAction);
}
if (!Subtarget.useSoftFloat()) {
// Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
// operation.
- setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);
+ setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i8, Promote);
- setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
+ setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i16, Promote);
// We have an algorithm for SSE2, and we turn this into a 64-bit
// FILD or VCVTUSI2SS/SD for other targets.
- setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
// We have an algorithm for SSE2->double, and we turn this into a
// 64-bit FILD followed by conditional FADD for other targets.
- setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
// Promote i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
// this operation.
- setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);
+ setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i8, Promote);
// SSE has no i16 to fp conversion, only i32. We promote in the handler
// to allow f80 to use i16 and f64 to use i16 with sse1 only
- setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i16, Custom);
// f32 and f64 cases are Legal with SSE1/SSE2, f80 case is not
- setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
// In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
// are Legal, f80 is custom lowered.
- setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
// Promote i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
// this operation.
- setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote);
+ setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote);
// FIXME: This doesn't generate invalid exception when it should. PR44019.
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i8, Promote);
- setOperationAction(ISD::FP_TO_SINT, MVT::i16, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i8, Promote);
+ setOperationAction(ISD::FP_TO_SINT, MVT::i16, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i16, Custom);
- setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
// In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
// are Legal, f80 is custom lowered.
- setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
// Handle FP_TO_UINT by promoting the destination to a larger signed
// conversion.
- setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote);
+ setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote);
// FIXME: This doesn't generate invalid exception when it should. PR44019.
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i8, Promote);
- setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i8, Promote);
+ setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
// FIXME: This doesn't generate invalid exception when it should. PR44019.
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i16, Promote);
- setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
- setOperationAction(ISD::LRINT, MVT::f32, Custom);
- setOperationAction(ISD::LRINT, MVT::f64, Custom);
- setOperationAction(ISD::LLRINT, MVT::f32, Custom);
- setOperationAction(ISD::LLRINT, MVT::f64, Custom);
+ setOperationAction(ISD::LRINT, MVT::f32, Custom);
+ setOperationAction(ISD::LRINT, MVT::f64, Custom);
+ setOperationAction(ISD::LLRINT, MVT::f32, Custom);
+ setOperationAction(ISD::LLRINT, MVT::f64, Custom);
if (!Subtarget.is64Bit() && Subtarget.hasX87()) {
- setOperationAction(ISD::LRINT, MVT::i64, Custom);
+ setOperationAction(ISD::LRINT, MVT::i64, Custom);
setOperationAction(ISD::LLRINT, MVT::i64, Custom);
}
}
@@ -311,7 +311,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
if (Subtarget.hasSSE2()) {
// Custom lowering for saturating float to int conversions.
// We handle promotion to larger result types manually.
- for (MVT VT : { MVT::i8, MVT::i16, MVT::i32 }) {
+ for (MVT VT : {MVT::i8, MVT::i16, MVT::i32}) {
setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom);
setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
}
@@ -346,15 +346,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// TODO: when we have SSE, these could be more efficient, by using movd/movq.
if (!Subtarget.hasSSE2()) {
- setOperationAction(ISD::BITCAST , MVT::f32 , Expand);
- setOperationAction(ISD::BITCAST , MVT::i32 , Expand);
+ setOperationAction(ISD::BITCAST, MVT::f32, Expand);
+ setOperationAction(ISD::BITCAST, MVT::i32, Expand);
if (Subtarget.is64Bit()) {
- setOperationAction(ISD::BITCAST , MVT::f64 , Expand);
+ setOperationAction(ISD::BITCAST, MVT::f64, Expand);
// Without SSE, i64->f64 goes through memory.
- setOperationAction(ISD::BITCAST , MVT::i64 , Expand);
+ setOperationAction(ISD::BITCAST, MVT::i64, Expand);
}
} else if (!Subtarget.is64Bit())
- setOperationAction(ISD::BITCAST , MVT::i64 , Custom);
+ setOperationAction(ISD::BITCAST, MVT::i64, Custom);
// Scalar integer divide and remainder are lowered to use operations that
// produce two results, to match the available instructions. This exposes
@@ -366,7 +366,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// (low) operations are left as Legal, as there are single-result
// instructions for this in x86. Using the two-result multiply instructions
// when both high and low results are needed must be arranged by dagcombine.
- for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
+ for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
setOperationAction(ISD::MULHS, VT, Expand);
setOperationAction(ISD::MULHU, VT, Expand);
setOperationAction(ISD::SDIV, VT, Expand);
@@ -375,18 +375,18 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::UREM, VT, Expand);
}
- setOperationAction(ISD::BR_JT , MVT::Other, Expand);
- setOperationAction(ISD::BRCOND , MVT::Other, Custom);
- for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128,
- MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
- setOperationAction(ISD::BR_CC, VT, Expand);
+ setOperationAction(ISD::BR_JT, MVT::Other, Expand);
+ setOperationAction(ISD::BRCOND, MVT::Other, Custom);
+ for (auto VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128, MVT::i8, MVT::i16,
+ MVT::i32, MVT::i64}) {
+ setOperationAction(ISD::BR_CC, VT, Expand);
setOperationAction(ISD::SELECT_CC, VT, Expand);
}
if (Subtarget.is64Bit())
setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
- setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal);
- setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
- setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand);
+ setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Legal);
+ setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Legal);
+ setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
setOperationAction(ISD::FREM, MVT::f32, LibCall);
setOperationAction(ISD::FREM, MVT::f64, LibCall);
@@ -394,11 +394,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FREM, MVT::f128, LibCall);
if (!Subtarget.useSoftFloat() && Subtarget.hasX87()) {
- setOperationAction(ISD::GET_ROUNDING , MVT::i32 , Custom);
- setOperationAction(ISD::SET_ROUNDING , MVT::Other, Custom);
- setOperationAction(ISD::GET_FPENV_MEM , MVT::Other, Custom);
- setOperationAction(ISD::SET_FPENV_MEM , MVT::Other, Custom);
- setOperationAction(ISD::RESET_FPENV , MVT::Other, Custom);
+ setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom);
+ setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
+ setOperationAction(ISD::GET_FPENV_MEM, MVT::Other, Custom);
+ setOperationAction(ISD::SET_FPENV_MEM, MVT::Other, Custom);
+ setOperationAction(ISD::RESET_FPENV, MVT::Other, Custom);
}
// Promote the i8 variants and force them on up to i32 which has a shorter
@@ -474,10 +474,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// on the dest that popcntl hasn't had since Cannon Lake.
setOperationPromotedToType(ISD::CTPOP, MVT::i16, MVT::i32);
} else {
- setOperationAction(ISD::CTPOP , MVT::i8 , Custom);
- setOperationAction(ISD::CTPOP , MVT::i16 , Custom);
- setOperationAction(ISD::CTPOP , MVT::i32 , Custom);
- setOperationAction(ISD::CTPOP , MVT::i64 , Custom);
+ setOperationAction(ISD::CTPOP, MVT::i8, Custom);
+ setOperationAction(ISD::CTPOP, MVT::i16, Custom);
+ setOperationAction(ISD::CTPOP, MVT::i32, Custom);
+ setOperationAction(ISD::CTPOP, MVT::i64, Custom);
}
if (Subtarget.hasBMI2()) {
@@ -488,23 +488,23 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction({ISD::PEXT, ISD::PDEP}, MVT::i64, Legal);
}
- setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
+ setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom);
if (!Subtarget.hasMOVBE())
- setOperationAction(ISD::BSWAP , MVT::i16 , Expand);
+ setOperationAction(ISD::BSWAP, MVT::i16, Expand);
// X86 wants to expand cmov itself.
- for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
+ for (auto VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) {
setOperationAction(ISD::SELECT, VT, Custom);
setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
}
- for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
+ for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
if (VT == MVT::i64 && !Subtarget.is64Bit())
continue;
setOperationAction(ISD::SELECT, VT, Custom);
- setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::SETCC, VT, Custom);
}
setOperationAction(ISD::COND_LOOP, MVT::Other, Custom);
@@ -513,7 +513,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SELECT, MVT::x86mmx, Custom);
setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand);
- setOperationAction(ISD::EH_RETURN , MVT::Other, Custom);
+ setOperationAction(ISD::EH_RETURN, MVT::Other, Custom);
// NOTE: EH_SJLJ_SETJMP/_LONGJMP are not recommended, since
// LLVM/Clang supports zero-cost DWARF and SEH exception handling.
setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
@@ -521,19 +521,19 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
// Darwin ABI issue.
- for (auto VT : { MVT::i32, MVT::i64 }) {
+ for (auto VT : {MVT::i32, MVT::i64}) {
if (VT == MVT::i64 && !Subtarget.is64Bit())
continue;
- setOperationAction(ISD::ConstantPool , VT, Custom);
- setOperationAction(ISD::JumpTable , VT, Custom);
- setOperationAction(ISD::GlobalAddress , VT, Custom);
+ setOperationAction(ISD::ConstantPool, VT, Custom);
+ setOperationAction(ISD::JumpTable, VT, Custom);
+ setOperationAction(ISD::GlobalAddress, VT, Custom);
setOperationAction(ISD::GlobalTLSAddress, VT, Custom);
- setOperationAction(ISD::ExternalSymbol , VT, Custom);
- setOperationAction(ISD::BlockAddress , VT, Custom);
+ setOperationAction(ISD::ExternalSymbol, VT, Custom);
+ setOperationAction(ISD::BlockAddress, VT, Custom);
}
// 64-bit shl, sra, srl (iff 32-bit x86)
- for (auto VT : { MVT::i32, MVT::i64 }) {
+ for (auto VT : {MVT::i32, MVT::i64}) {
if (VT == MVT::i64 && !Subtarget.is64Bit())
continue;
setOperationAction(ISD::SHL_PARTS, VT, Custom);
@@ -542,12 +542,12 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
if (Subtarget.hasSSEPrefetch())
- setOperationAction(ISD::PREFETCH , MVT::Other, Custom);
+ setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
- setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom);
+ setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
// Expand certain atomics
- for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
+ for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom);
setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom);
@@ -591,14 +591,14 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
// VASTART needs to be custom lowered to use the VarArgsFrameIndex
- setOperationAction(ISD::VASTART , MVT::Other, Custom);
- setOperationAction(ISD::VAEND , MVT::Other, Expand);
+ setOperationAction(ISD::VASTART, MVT::Other, Custom);
+ setOperationAction(ISD::VAEND, MVT::Other, Expand);
bool Is64Bit = Subtarget.is64Bit();
- setOperationAction(ISD::VAARG, MVT::Other, Is64Bit ? Custom : Expand);
+ setOperationAction(ISD::VAARG, MVT::Other, Is64Bit ? Custom : Expand);
setOperationAction(ISD::VACOPY, MVT::Other, Is64Bit ? Custom : Expand);
- setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
- setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
+ setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
+ setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
@@ -608,7 +608,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
- auto setF16Action = [&] (MVT VT, LegalizeAction Action) {
+ auto setF16Action = [&](MVT VT, LegalizeAction Action) {
setOperationAction(ISD::FABS, VT, Action);
setOperationAction(ISD::FNEG, VT, Action);
setOperationAction(ISD::FCOPYSIGN, VT, Expand);
@@ -665,7 +665,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// non-optsize case.
setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
- for (auto VT : { MVT::f32, MVT::f64 }) {
+ for (auto VT : {MVT::f32, MVT::f64}) {
// Use ANDPD to simulate FABS.
setOperationAction(ISD::FABS, VT, Custom);
@@ -680,8 +680,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FSUB, VT, Custom);
// We don't support sin/cos/fmod
- setOperationAction(ISD::FSIN , VT, Expand);
- setOperationAction(ISD::FCOS , VT, Expand);
+ setOperationAction(ISD::FSIN, VT, Expand);
+ setOperationAction(ISD::FCOS, VT, Expand);
setOperationAction(ISD::FSINCOS, VT, Expand);
}
@@ -751,10 +751,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addRegisterClass(MVT::f64, &X86::RFP64RegClass);
// Use ANDPS to simulate FABS.
- setOperationAction(ISD::FABS , MVT::f32, Custom);
+ setOperationAction(ISD::FABS, MVT::f32, Custom);
// Use XORP to simulate FNEG.
- setOperationAction(ISD::FNEG , MVT::f32, Custom);
+ setOperationAction(ISD::FNEG, MVT::f32, Custom);
if (UseX87)
setOperationAction({ISD::UNDEF, ISD::POISON}, MVT::f64, Expand);
@@ -765,8 +765,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
// We don't support sin/cos/fmod
- setOperationAction(ISD::FSIN , MVT::f32, Expand);
- setOperationAction(ISD::FCOS , MVT::f32, Expand);
+ setOperationAction(ISD::FSIN, MVT::f32, Expand);
+ setOperationAction(ISD::FCOS, MVT::f32, Expand);
setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
if (UseX87) {
@@ -781,13 +781,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addRegisterClass(MVT::f64, &X86::RFP64RegClass);
addRegisterClass(MVT::f32, &X86::RFP32RegClass);
- for (auto VT : { MVT::f32, MVT::f64 }) {
+ for (auto VT : {MVT::f32, MVT::f64}) {
setOperationAction({ISD::UNDEF, ISD::POISON}, VT, Expand);
setOperationAction(ISD::FCOPYSIGN, VT, Expand);
// Always expand sin/cos functions even though x87 has an instruction.
- setOperationAction(ISD::FSIN , VT, Expand);
- setOperationAction(ISD::FCOS , VT, Expand);
+ setOperationAction(ISD::FSIN, VT, Expand);
+ setOperationAction(ISD::FCOS, VT, Expand);
setOperationAction(ISD::FSINCOS, VT, Expand);
}
}
@@ -799,7 +799,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addLegalFPImmediate(APFloat(+1.0f)); // FLD1
addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
- } else // SSE immediates.
+ } else // SSE immediates.
addLegalFPImmediate(APFloat(+0.0f)); // xorps
}
// Expand FP64 immediates into loads from the stack, save special cases.
@@ -809,7 +809,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addLegalFPImmediate(APFloat(+1.0)); // FLD1
addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
- } else // SSE immediates.
+ } else // SSE immediates.
addLegalFPImmediate(APFloat(+0.0)); // xorpd
}
// Support fp16 0 immediate.
@@ -817,18 +817,18 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addLegalFPImmediate(APFloat::getZero(APFloat::IEEEhalf()));
// Handle constrained floating-point operations of scalar.
- setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
// We don't support FMA.
setOperationAction(ISD::FMA, MVT::f64, Expand);
@@ -841,17 +841,17 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
{
APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended());
- addLegalFPImmediate(TmpFlt); // FLD0
+ addLegalFPImmediate(TmpFlt); // FLD0
TmpFlt.changeSign();
- addLegalFPImmediate(TmpFlt); // FLD0/FCHS
+ addLegalFPImmediate(TmpFlt); // FLD0/FCHS
bool ignored;
APFloat TmpFlt2(+1.0);
- TmpFlt2.convert(APFloat::x87DoubleExtended(), APFloat::rmNearestTiesToEven,
- &ignored);
- addLegalFPImmediate(TmpFlt2); // FLD1
+ TmpFlt2.convert(APFloat::x87DoubleExtended(),
+ APFloat::rmNearestTiesToEven, &ignored);
+ addLegalFPImmediate(TmpFlt2); // FLD1
TmpFlt2.changeSign();
- addLegalFPImmediate(TmpFlt2); // FLD1/FCHS
+ addLegalFPImmediate(TmpFlt2); // FLD1/FCHS
}
// Always expand sin/cos functions even though x87 has an instruction.
@@ -870,9 +870,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// clang-format on
setOperationAction(ISD::FFLOOR, MVT::f80, Expand);
- setOperationAction(ISD::FCEIL, MVT::f80, Expand);
+ setOperationAction(ISD::FCEIL, MVT::f80, Expand);
setOperationAction(ISD::FTRUNC, MVT::f80, Expand);
- setOperationAction(ISD::FRINT, MVT::f80, Expand);
+ setOperationAction(ISD::FRINT, MVT::f80, Expand);
setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand);
setOperationAction(ISD::FROUNDEVEN, MVT::f80, Expand);
setOperationAction(ISD::FMA, MVT::f80, Expand);
@@ -882,11 +882,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::LLRINT, MVT::f80, Custom);
// Handle constrained floating-point operations of scalar.
- setOperationAction(ISD::STRICT_FADD , MVT::f80, Legal);
- setOperationAction(ISD::STRICT_FSUB , MVT::f80, Legal);
- setOperationAction(ISD::STRICT_FMUL , MVT::f80, Legal);
- setOperationAction(ISD::STRICT_FDIV , MVT::f80, Legal);
- setOperationAction(ISD::STRICT_FSQRT , MVT::f80, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::f80, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::f80, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::f80, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::f80, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::f80, Legal);
setOperationAction(ISD::FCANONICALIZE, MVT::f80, Expand);
if (isTypeLegal(MVT::f16)) {
setOperationAction(ISD::FP_EXTEND, MVT::f80, Custom);
@@ -906,16 +906,16 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addLegalFPImmediate(APFloat::getZero(APFloat::IEEEquad())); // xorps
- setOperationAction(ISD::FADD, MVT::f128, LibCall);
+ setOperationAction(ISD::FADD, MVT::f128, LibCall);
setOperationAction(ISD::STRICT_FADD, MVT::f128, LibCall);
- setOperationAction(ISD::FSUB, MVT::f128, LibCall);
+ setOperationAction(ISD::FSUB, MVT::f128, LibCall);
setOperationAction(ISD::STRICT_FSUB, MVT::f128, LibCall);
- setOperationAction(ISD::FDIV, MVT::f128, LibCall);
+ setOperationAction(ISD::FDIV, MVT::f128, LibCall);
setOperationAction(ISD::STRICT_FDIV, MVT::f128, LibCall);
- setOperationAction(ISD::FMUL, MVT::f128, LibCall);
+ setOperationAction(ISD::FMUL, MVT::f128, LibCall);
setOperationAction(ISD::STRICT_FMUL, MVT::f128, LibCall);
- setOperationAction(ISD::FMA, MVT::f128, LibCall);
- setOperationAction(ISD::STRICT_FMA, MVT::f128, LibCall);
+ setOperationAction(ISD::FMA, MVT::f128, LibCall);
+ setOperationAction(ISD::STRICT_FMA, MVT::f128, LibCall);
setOperationAction(ISD::FABS, MVT::f128, Custom);
setOperationAction(ISD::FNEG, MVT::f128, Custom);
@@ -931,10 +931,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::STRICT_FTAN, MVT::f128, LibCall);
// clang-format on
// No STRICT_FSINCOS
- setOperationAction(ISD::FSQRT, MVT::f128, LibCall);
+ setOperationAction(ISD::FSQRT, MVT::f128, LibCall);
setOperationAction(ISD::STRICT_FSQRT, MVT::f128, LibCall);
- setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
+ setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Custom);
// We need to custom handle any FP_ROUND with an f128 input, but
// LegalizeDAG uses the result type to know when to run a custom handler.
@@ -964,10 +964,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
// Always use a library call for pow.
- setOperationAction(ISD::FPOW , MVT::f32 , Expand);
- setOperationAction(ISD::FPOW , MVT::f64 , Expand);
- setOperationAction(ISD::FPOW , MVT::f80 , Expand);
- setOperationAction(ISD::FPOW , MVT::f128 , Expand);
+ setOperationAction(ISD::FPOW, MVT::f32, Expand);
+ setOperationAction(ISD::FPOW, MVT::f64, Expand);
+ setOperationAction(ISD::FPOW, MVT::f80, Expand);
+ setOperationAction(ISD::FPOW, MVT::f128, Expand);
setOperationAction(ISD::FLOG, MVT::f80, Expand);
setOperationAction(ISD::FLOG2, MVT::f80, Expand);
@@ -979,9 +979,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);
// Some FP actions are always expanded for vector types.
- for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16,
- MVT::v4f32, MVT::v8f32, MVT::v16f32,
- MVT::v2f64, MVT::v4f64, MVT::v8f64 }) {
+ for (auto VT : {MVT::v8f16, MVT::v16f16, MVT::v32f16, MVT::v4f32, MVT::v8f32,
+ MVT::v16f32, MVT::v2f64, MVT::v4f64, MVT::v8f64}) {
// clang-format off
setOperationAction(ISD::FSIN, VT, Expand);
setOperationAction(ISD::FSINCOS, VT, Expand);
@@ -1007,11 +1006,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::UDIV, VT, Expand);
setOperationAction(ISD::SREM, VT, Expand);
setOperationAction(ISD::UREM, VT, Expand);
- setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);
+ setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Expand);
setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
- setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);
- setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);
- setOperationAction(ISD::FMA, VT, Expand);
+ setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Expand);
+ setOperationAction(ISD::INSERT_SUBVECTOR, VT, Expand);
+ setOperationAction(ISD::FMA, VT, Expand);
setOperationAction(ISD::FFLOOR, VT, Expand);
setOperationAction(ISD::FCEIL, VT, Expand);
setOperationAction(ISD::FTRUNC, VT, Expand);
@@ -1035,7 +1034,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FP_TO_SINT, VT, Expand);
setOperationAction(ISD::UINT_TO_FP, VT, Expand);
setOperationAction(ISD::SINT_TO_FP, VT, Expand);
- setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);
+ setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
setOperationAction(ISD::TRUNCATE, VT, Expand);
setOperationAction(ISD::SIGN_EXTEND, VT, Expand);
setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
@@ -1086,25 +1085,25 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
SetFPMinMaxAction(MVT::f32);
- setOperationAction(ISD::FNEG, MVT::v4f32, Custom);
- setOperationAction(ISD::FABS, MVT::v4f32, Custom);
- setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Custom);
- setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
- setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
+ setOperationAction(ISD::FNEG, MVT::v4f32, Custom);
+ setOperationAction(ISD::FABS, MVT::v4f32, Custom);
+ setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Custom);
+ setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
+ setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
- setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
+ setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
setOperationAction(ISD::FCANONICALIZE, MVT::v4f32, Expand);
- setOperationAction(ISD::LOAD, MVT::v2f32, Custom);
- setOperationAction(ISD::STORE, MVT::v2f32, Custom);
+ setOperationAction(ISD::LOAD, MVT::v2f32, Custom);
+ setOperationAction(ISD::STORE, MVT::v2f32, Custom);
setOperationAction(ISD::FCANONICALIZE, MVT::v2f32, Expand);
- setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal);
}
if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
@@ -1124,34 +1123,34 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass
: &X86::VR128RegClass);
- for (auto VT : { MVT::f64, MVT::v4f32, MVT::v2f64 })
+ for (auto VT : {MVT::f64, MVT::v4f32, MVT::v2f64})
SetFPMinMaxAction(VT);
- setOperationAction(ISD::MUL, MVT::v2i8, Custom);
- setOperationAction(ISD::MUL, MVT::v4i8, Custom);
- setOperationAction(ISD::MUL, MVT::v8i8, Custom);
-
- setOperationAction(ISD::MUL, MVT::v16i8, Custom);
- setOperationAction(ISD::MUL, MVT::v4i32, Custom);
- setOperationAction(ISD::MUL, MVT::v2i64, Custom);
- setOperationAction(ISD::MULHU, MVT::v4i32, Custom);
- setOperationAction(ISD::MULHS, MVT::v4i32, Custom);
- setOperationAction(ISD::MULHU, MVT::v16i8, Custom);
- setOperationAction(ISD::MULHS, MVT::v16i8, Custom);
- setOperationAction(ISD::MULHU, MVT::v8i16, Legal);
- setOperationAction(ISD::MULHS, MVT::v8i16, Legal);
- setOperationAction(ISD::MUL, MVT::v8i16, Legal);
- setOperationAction(ISD::AVGCEILU, MVT::v16i8, Legal);
- setOperationAction(ISD::AVGCEILU, MVT::v8i16, Legal);
-
- setOperationAction(ISD::SMULO, MVT::v16i8, Custom);
- setOperationAction(ISD::UMULO, MVT::v16i8, Custom);
- setOperationAction(ISD::UMULO, MVT::v2i32, Custom);
-
- setOperationAction(ISD::FNEG, MVT::v2f64, Custom);
+ setOperationAction(ISD::MUL, MVT::v2i8, Custom);
+ setOperationAction(ISD::MUL, MVT::v4i8, Custom);
+ setOperationAction(ISD::MUL, MVT::v8i8, Custom);
+
+ setOperationAction(ISD::MUL, MVT::v16i8, Custom);
+ setOperationAction(ISD::MUL, MVT::v4i32, Custom);
+ setOperationAction(ISD::MUL, MVT::v2i64, Custom);
+ setOperationAction(ISD::MULHU, MVT::v4i32, Custom);
+ setOperationAction(ISD::MULHS, MVT::v4i32, Custom);
+ setOperationAction(ISD::MULHU, MVT::v16i8, Custom);
+ setOperationAction(ISD::MULHS, MVT::v16i8, Custom);
+ setOperationAction(ISD::MULHU, MVT::v8i16, Legal);
+ setOperationAction(ISD::MULHS, MVT::v8i16, Legal);
+ setOperationAction(ISD::MUL, MVT::v8i16, Legal);
+ setOperationAction(ISD::AVGCEILU, MVT::v16i8, Legal);
+ setOperationAction(ISD::AVGCEILU, MVT::v8i16, Legal);
+
+ setOperationAction(ISD::SMULO, MVT::v16i8, Custom);
+ setOperationAction(ISD::UMULO, MVT::v16i8, Custom);
+ setOperationAction(ISD::UMULO, MVT::v2i32, Custom);
+
+ setOperationAction(ISD::FNEG, MVT::v2f64, Custom);
setOperationAction(ISD::FCANONICALIZE, MVT::v2f64, Expand);
- setOperationAction(ISD::FABS, MVT::v2f64, Custom);
- setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Custom);
+ setOperationAction(ISD::FABS, MVT::v2f64, Custom);
+ setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Custom);
setOperationAction(ISD::LRINT, MVT::v4f32, Custom);
setOperationAction(ISD::LRINT, MVT::v2i32, Custom);
@@ -1171,7 +1170,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::CLMUL, MVT::i8, Custom);
}
- for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
+ for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom);
setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom);
setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom);
@@ -1185,30 +1184,30 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
- setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
}
- setOperationAction(ISD::UADDSAT, MVT::v16i8, Legal);
- setOperationAction(ISD::SADDSAT, MVT::v16i8, Legal);
- setOperationAction(ISD::USUBSAT, MVT::v16i8, Legal);
- setOperationAction(ISD::SSUBSAT, MVT::v16i8, Legal);
- setOperationAction(ISD::UADDSAT, MVT::v8i16, Legal);
- setOperationAction(ISD::SADDSAT, MVT::v8i16, Legal);
- setOperationAction(ISD::USUBSAT, MVT::v8i16, Legal);
- setOperationAction(ISD::SSUBSAT, MVT::v8i16, Legal);
- setOperationAction(ISD::USUBSAT, MVT::v4i32, Custom);
- setOperationAction(ISD::USUBSAT, MVT::v2i64, Custom);
+ setOperationAction(ISD::UADDSAT, MVT::v16i8, Legal);
+ setOperationAction(ISD::SADDSAT, MVT::v16i8, Legal);
+ setOperationAction(ISD::USUBSAT, MVT::v16i8, Legal);
+ setOperationAction(ISD::SSUBSAT, MVT::v16i8, Legal);
+ setOperationAction(ISD::UADDSAT, MVT::v8i16, Legal);
+ setOperationAction(ISD::SADDSAT, MVT::v8i16, Legal);
+ setOperationAction(ISD::USUBSAT, MVT::v8i16, Legal);
+ setOperationAction(ISD::SSUBSAT, MVT::v8i16, Legal);
+ setOperationAction(ISD::USUBSAT, MVT::v4i32, Custom);
+ setOperationAction(ISD::USUBSAT, MVT::v2i64, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
- for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
+ for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::CTPOP, VT, Custom);
setOperationAction(ISD::ABS, VT, Custom);
@@ -1221,30 +1220,30 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setCondCodeAction(ISD::SETLE, VT, Custom);
}
- setOperationAction(ISD::SETCC, MVT::v2f64, Custom);
- setOperationAction(ISD::SETCC, MVT::v4f32, Custom);
- setOperationAction(ISD::STRICT_FSETCC, MVT::v2f64, Custom);
- setOperationAction(ISD::STRICT_FSETCC, MVT::v4f32, Custom);
+ setOperationAction(ISD::SETCC, MVT::v2f64, Custom);
+ setOperationAction(ISD::SETCC, MVT::v4f32, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, MVT::v2f64, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, MVT::v4f32, Custom);
setOperationAction(ISD::STRICT_FSETCCS, MVT::v2f64, Custom);
setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f32, Custom);
- for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
- setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
- setOperationAction(ISD::VSELECT, VT, Custom);
+ for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32}) {
+ setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::VSELECT, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
}
- for (auto VT : { MVT::v8f16, MVT::v2f64, MVT::v2i64 }) {
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
- setOperationAction(ISD::VSELECT, VT, Custom);
+ for (auto VT : {MVT::v8f16, MVT::v2f64, MVT::v2i64}) {
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::VSELECT, VT, Custom);
if (VT == MVT::v2i64 && !Subtarget.is64Bit())
continue;
- setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
}
setF16Action(MVT::v8f16, Expand);
@@ -1257,67 +1256,67 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Custom);
// Custom lower v2i64 and v2f64 selects.
- setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
- setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
- setOperationAction(ISD::SELECT, MVT::v4i32, Custom);
- setOperationAction(ISD::SELECT, MVT::v8i16, Custom);
- setOperationAction(ISD::SELECT, MVT::v8f16, Custom);
- setOperationAction(ISD::SELECT, MVT::v16i8, Custom);
-
- setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Custom);
- setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v2i32, Custom);
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Custom);
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i32, Custom);
+ setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
+ setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
+ setOperationAction(ISD::SELECT, MVT::v4i32, Custom);
+ setOperationAction(ISD::SELECT, MVT::v8i16, Custom);
+ setOperationAction(ISD::SELECT, MVT::v8f16, Custom);
+ setOperationAction(ISD::SELECT, MVT::v16i8, Custom);
+
+ setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v2i32, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i32, Custom);
// Custom legalize these to avoid over promotion or custom promotion.
for (auto VT : {MVT::v2i8, MVT::v4i8, MVT::v8i8, MVT::v2i16, MVT::v4i16}) {
- setOperationAction(ISD::FP_TO_SINT, VT, Custom);
- setOperationAction(ISD::FP_TO_UINT, VT, Custom);
+ setOperationAction(ISD::FP_TO_SINT, VT, Custom);
+ setOperationAction(ISD::FP_TO_UINT, VT, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
}
- setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Custom);
- setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i32, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i32, Custom);
- setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
- setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i32, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
+ setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i32, Custom);
- setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
- setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
+ setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Custom);
// Fast v2f32 UINT_TO_FP( v2i32 ) custom conversion.
- setOperationAction(ISD::SINT_TO_FP, MVT::v2f32, Custom);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f32, Custom);
- setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom);
- setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f32, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v2f32, Custom);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f32, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom);
+ setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f32, Custom);
- setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f32, Custom);
- setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f32, Custom);
+ setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f32, Custom);
+ setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f32, Custom);
// We want to legalize this to an f64 load rather than an i64 load on
// 64-bit targets and two 32-bit loads on a 32-bit target. Similar for
// store.
- setOperationAction(ISD::LOAD, MVT::v2i32, Custom);
- setOperationAction(ISD::LOAD, MVT::v4i16, Custom);
- setOperationAction(ISD::LOAD, MVT::v8i8, Custom);
- setOperationAction(ISD::STORE, MVT::v2i32, Custom);
- setOperationAction(ISD::STORE, MVT::v4i16, Custom);
- setOperationAction(ISD::STORE, MVT::v8i8, Custom);
+ setOperationAction(ISD::LOAD, MVT::v2i32, Custom);
+ setOperationAction(ISD::LOAD, MVT::v4i16, Custom);
+ setOperationAction(ISD::LOAD, MVT::v8i8, Custom);
+ setOperationAction(ISD::STORE, MVT::v2i32, Custom);
+ setOperationAction(ISD::STORE, MVT::v4i16, Custom);
+ setOperationAction(ISD::STORE, MVT::v8i8, Custom);
// Add 32-bit vector stores to help vectorization opportunities.
- setOperationAction(ISD::STORE, MVT::v2i16, Custom);
- setOperationAction(ISD::STORE, MVT::v4i8, Custom);
+ setOperationAction(ISD::STORE, MVT::v2i16, Custom);
+ setOperationAction(ISD::STORE, MVT::v4i8, Custom);
- setOperationAction(ISD::BITCAST, MVT::v2i32, Custom);
- setOperationAction(ISD::BITCAST, MVT::v4i16, Custom);
- setOperationAction(ISD::BITCAST, MVT::v8i8, Custom);
+ setOperationAction(ISD::BITCAST, MVT::v2i32, Custom);
+ setOperationAction(ISD::BITCAST, MVT::v4i16, Custom);
+ setOperationAction(ISD::BITCAST, MVT::v8i8, Custom);
if (!Subtarget.hasAVX512())
setOperationAction(ISD::BITCAST, MVT::v16i1, Custom);
@@ -1327,41 +1326,42 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v2i32, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v2i64, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v4i64, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v8i64, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v16i32, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v16i64, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v2i32, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v2i64, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v4i64, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v8i64, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v16i32, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v16i64, Custom);
// In the customized shift lowering, the legal v4i32/v2i64 cases
// in AVX2 will be recognized.
- for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
- setOperationAction(ISD::SRL, VT, Custom);
- setOperationAction(ISD::SHL, VT, Custom);
- setOperationAction(ISD::SRA, VT, Custom);
- if (VT == MVT::v2i64) continue;
- setOperationAction(ISD::ROTL, VT, Custom);
- setOperationAction(ISD::ROTR, VT, Custom);
- setOperationAction(ISD::FSHL, VT, Custom);
- setOperationAction(ISD::FSHR, VT, Custom);
+ for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
+ setOperationAction(ISD::SRL, VT, Custom);
+ setOperationAction(ISD::SHL, VT, Custom);
+ setOperationAction(ISD::SRA, VT, Custom);
+ if (VT == MVT::v2i64)
+ continue;
+ setOperationAction(ISD::ROTL, VT, Custom);
+ setOperationAction(ISD::ROTR, VT, Custom);
+ setOperationAction(ISD::FSHL, VT, Custom);
+ setOperationAction(ISD::FSHR, VT, Custom);
}
- setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal);
- setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal);
}
if (!Subtarget.useSoftFloat() && Subtarget.hasGFNI()) {
@@ -1379,21 +1379,21 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
if (!Subtarget.useSoftFloat() && Subtarget.hasSSSE3()) {
- setOperationAction(ISD::ABS, MVT::v16i8, Legal);
- setOperationAction(ISD::ABS, MVT::v8i16, Legal);
- setOperationAction(ISD::ABS, MVT::v4i32, Legal);
+ setOperationAction(ISD::ABS, MVT::v16i8, Legal);
+ setOperationAction(ISD::ABS, MVT::v8i16, Legal);
+ setOperationAction(ISD::ABS, MVT::v4i32, Legal);
for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
- setOperationAction(ISD::BITREVERSE, VT, Custom);
- setOperationAction(ISD::CTLZ, VT, Custom);
+ setOperationAction(ISD::BITREVERSE, VT, Custom);
+ setOperationAction(ISD::CTLZ, VT, Custom);
}
- setOperationAction(ISD::BITREVERSE, MVT::i128, Custom);
+ setOperationAction(ISD::BITREVERSE, MVT::i128, Custom);
// These might be better off as horizontal vector ops.
- setOperationAction(ISD::ADD, MVT::i16, Custom);
- setOperationAction(ISD::ADD, MVT::i32, Custom);
- setOperationAction(ISD::SUB, MVT::i16, Custom);
- setOperationAction(ISD::SUB, MVT::i32, Custom);
+ setOperationAction(ISD::ADD, MVT::i16, Custom);
+ setOperationAction(ISD::ADD, MVT::i32, Custom);
+ setOperationAction(ISD::SUB, MVT::i16, Custom);
+ setOperationAction(ISD::SUB, MVT::i32, Custom);
}
if (Subtarget.hasNDD()) {
// Enable custom lowering for scalar USUBSAT to optimize usub.sat(X,1)
@@ -1405,55 +1405,55 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
- setOperationAction(ISD::FFLOOR, RoundedTy, Legal);
- setOperationAction(ISD::STRICT_FFLOOR, RoundedTy, Legal);
- setOperationAction(ISD::FCEIL, RoundedTy, Legal);
- setOperationAction(ISD::STRICT_FCEIL, RoundedTy, Legal);
- setOperationAction(ISD::FTRUNC, RoundedTy, Legal);
- setOperationAction(ISD::STRICT_FTRUNC, RoundedTy, Legal);
- setOperationAction(ISD::FRINT, RoundedTy, Legal);
- setOperationAction(ISD::STRICT_FRINT, RoundedTy, Legal);
- setOperationAction(ISD::FNEARBYINT, RoundedTy, Legal);
- setOperationAction(ISD::STRICT_FNEARBYINT, RoundedTy, Legal);
- setOperationAction(ISD::FROUNDEVEN, RoundedTy, Legal);
- setOperationAction(ISD::STRICT_FROUNDEVEN, RoundedTy, Legal);
-
- setOperationAction(ISD::FROUND, RoundedTy, Custom);
- }
-
- setOperationAction(ISD::SMAX, MVT::v16i8, Legal);
- setOperationAction(ISD::SMAX, MVT::v4i32, Legal);
- setOperationAction(ISD::UMAX, MVT::v8i16, Legal);
- setOperationAction(ISD::UMAX, MVT::v4i32, Legal);
- setOperationAction(ISD::SMIN, MVT::v16i8, Legal);
- setOperationAction(ISD::SMIN, MVT::v4i32, Legal);
- setOperationAction(ISD::UMIN, MVT::v8i16, Legal);
- setOperationAction(ISD::UMIN, MVT::v4i32, Legal);
-
- setOperationAction(ISD::UADDSAT, MVT::v4i32, Custom);
- setOperationAction(ISD::SADDSAT, MVT::v2i64, Custom);
- setOperationAction(ISD::SSUBSAT, MVT::v2i64, Custom);
+ setOperationAction(ISD::FFLOOR, RoundedTy, Legal);
+ setOperationAction(ISD::STRICT_FFLOOR, RoundedTy, Legal);
+ setOperationAction(ISD::FCEIL, RoundedTy, Legal);
+ setOperationAction(ISD::STRICT_FCEIL, RoundedTy, Legal);
+ setOperationAction(ISD::FTRUNC, RoundedTy, Legal);
+ setOperationAction(ISD::STRICT_FTRUNC, RoundedTy, Legal);
+ setOperationAction(ISD::FRINT, RoundedTy, Legal);
+ setOperationAction(ISD::STRICT_FRINT, RoundedTy, Legal);
+ setOperationAction(ISD::FNEARBYINT, RoundedTy, Legal);
+ setOperationAction(ISD::STRICT_FNEARBYINT, RoundedTy, Legal);
+ setOperationAction(ISD::FROUNDEVEN, RoundedTy, Legal);
+ setOperationAction(ISD::STRICT_FROUNDEVEN, RoundedTy, Legal);
+
+ setOperationAction(ISD::FROUND, RoundedTy, Custom);
+ }
+
+ setOperationAction(ISD::SMAX, MVT::v16i8, Legal);
+ setOperationAction(ISD::SMAX, MVT::v4i32, Legal);
+ setOperationAction(ISD::UMAX, MVT::v8i16, Legal);
+ setOperationAction(ISD::UMAX, MVT::v4i32, Legal);
+ setOperationAction(ISD::SMIN, MVT::v16i8, Legal);
+ setOperationAction(ISD::SMIN, MVT::v4i32, Legal);
+ setOperationAction(ISD::UMIN, MVT::v8i16, Legal);
+ setOperationAction(ISD::UMIN, MVT::v4i32, Legal);
+
+ setOperationAction(ISD::UADDSAT, MVT::v4i32, Custom);
+ setOperationAction(ISD::SADDSAT, MVT::v2i64, Custom);
+ setOperationAction(ISD::SSUBSAT, MVT::v2i64, Custom);
// FIXME: Do we need to handle scalar-to-vector here?
- setOperationAction(ISD::MUL, MVT::v4i32, Legal);
- setOperationAction(ISD::SMULO, MVT::v2i32, Custom);
+ setOperationAction(ISD::MUL, MVT::v4i32, Legal);
+ setOperationAction(ISD::SMULO, MVT::v2i32, Custom);
// We directly match byte blends in the backend as they match the VSELECT
// condition form.
- setOperationAction(ISD::VSELECT, MVT::v16i8, Legal);
+ setOperationAction(ISD::VSELECT, MVT::v16i8, Legal);
// SSE41 brings specific instructions for doing vector sign extend even in
// cases where we don't have SRA.
- for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
+ for (auto VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal);
setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal);
}
// SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
- for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
- setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8, Legal);
- setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8, Legal);
- setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8, Legal);
+ for (auto LoadExtOp : {ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
+ setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8, Legal);
setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal);
setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal);
setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i32, Legal);
@@ -1462,69 +1462,69 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
if (Subtarget.is64Bit() && !Subtarget.hasAVX512()) {
// We need to scalarize v4i64->v432 uint_to_fp using cvtsi2ss, but we can
// do the pre and post work in the vector domain.
- setOperationAction(ISD::UINT_TO_FP, MVT::v4i64, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v4i64, Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i64, Custom);
// We need to mark SINT_TO_FP as Custom even though we want to expand it
// so that DAG combine doesn't try to turn it into uint_to_fp.
- setOperationAction(ISD::SINT_TO_FP, MVT::v4i64, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v4i64, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i64, Custom);
}
}
if (!Subtarget.useSoftFloat() && Subtarget.hasSSE42()) {
- setOperationAction(ISD::UADDSAT, MVT::v2i64, Custom);
+ setOperationAction(ISD::UADDSAT, MVT::v2i64, Custom);
}
if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
- for (MVT VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
- MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
+ for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v32i8,
+ MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
setOperationAction(ISD::ROTL, VT, VT.is128BitVector() ? Legal : Custom);
setOperationAction(ISD::ROTR, VT, Custom);
}
// XOP can efficiently perform BITREVERSE with VPPERM.
- for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 })
+ for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64})
setOperationAction(ISD::BITREVERSE, VT, Custom);
}
if (!Subtarget.useSoftFloat() && Subtarget.hasAVX()) {
bool HasInt256 = Subtarget.hasInt256();
- addRegisterClass(MVT::v32i8, Subtarget.hasVLX() ? &X86::VR256XRegClass
- : &X86::VR256RegClass);
+ addRegisterClass(MVT::v32i8, Subtarget.hasVLX() ? &X86::VR256XRegClass
+ : &X86::VR256RegClass);
addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass
: &X86::VR256RegClass);
addRegisterClass(MVT::v16f16, Subtarget.hasVLX() ? &X86::VR256XRegClass
: &X86::VR256RegClass);
- addRegisterClass(MVT::v8i32, Subtarget.hasVLX() ? &X86::VR256XRegClass
- : &X86::VR256RegClass);
- addRegisterClass(MVT::v8f32, Subtarget.hasVLX() ? &X86::VR256XRegClass
- : &X86::VR256RegClass);
- addRegisterClass(MVT::v4i64, Subtarget.hasVLX() ? &X86::VR256XRegClass
- : &X86::VR256RegClass);
- addRegisterClass(MVT::v4f64, Subtarget.hasVLX() ? &X86::VR256XRegClass
- : &X86::VR256RegClass);
-
- for (auto VT : { MVT::v8f32, MVT::v4f64 }) {
- setOperationAction(ISD::FFLOOR, VT, Legal);
- setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
- setOperationAction(ISD::FCEIL, VT, Legal);
- setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
- setOperationAction(ISD::FTRUNC, VT, Legal);
- setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
- setOperationAction(ISD::FRINT, VT, Legal);
- setOperationAction(ISD::STRICT_FRINT, VT, Legal);
- setOperationAction(ISD::FNEARBYINT, VT, Legal);
+ addRegisterClass(MVT::v8i32, Subtarget.hasVLX() ? &X86::VR256XRegClass
+ : &X86::VR256RegClass);
+ addRegisterClass(MVT::v8f32, Subtarget.hasVLX() ? &X86::VR256XRegClass
+ : &X86::VR256RegClass);
+ addRegisterClass(MVT::v4i64, Subtarget.hasVLX() ? &X86::VR256XRegClass
+ : &X86::VR256RegClass);
+ addRegisterClass(MVT::v4f64, Subtarget.hasVLX() ? &X86::VR256XRegClass
+ : &X86::VR256RegClass);
+
+ for (auto VT : {MVT::v8f32, MVT::v4f64}) {
+ setOperationAction(ISD::FFLOOR, VT, Legal);
+ setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
+ setOperationAction(ISD::FCEIL, VT, Legal);
+ setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
+ setOperationAction(ISD::FTRUNC, VT, Legal);
+ setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
+ setOperationAction(ISD::FRINT, VT, Legal);
+ setOperationAction(ISD::STRICT_FRINT, VT, Legal);
+ setOperationAction(ISD::FNEARBYINT, VT, Legal);
setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
- setOperationAction(ISD::FROUNDEVEN, VT, Legal);
+ setOperationAction(ISD::FROUNDEVEN, VT, Legal);
setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
- setOperationAction(ISD::FROUND, VT, Custom);
+ setOperationAction(ISD::FROUND, VT, Custom);
- setOperationAction(ISD::FNEG, VT, Custom);
- setOperationAction(ISD::FABS, VT, Custom);
- setOperationAction(ISD::FCOPYSIGN, VT, Custom);
- setOperationAction(ISD::FCANONICALIZE, VT, Expand);
+ setOperationAction(ISD::FNEG, VT, Custom);
+ setOperationAction(ISD::FABS, VT, Custom);
+ setOperationAction(ISD::FCOPYSIGN, VT, Custom);
+ setOperationAction(ISD::FCANONICALIZE, VT, Expand);
SetFPMinMaxAction(VT);
}
@@ -1539,89 +1539,90 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted
// even though v8i16 is a legal type.
- setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i16, MVT::v8i32);
- setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i16, MVT::v8i32);
+ setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i16, MVT::v8i32);
+ setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i16, MVT::v8i32);
setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i16, MVT::v8i32);
setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i16, MVT::v8i32);
- setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Custom);
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i32, Custom);
-
- setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Custom);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i32, Custom);
- setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Expand);
- setOperationAction(ISD::FP_ROUND, MVT::v8f16, Expand);
- setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Custom);
-
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f32, Legal);
- setOperationAction(ISD::STRICT_FADD, MVT::v8f32, Legal);
- setOperationAction(ISD::STRICT_FADD, MVT::v4f64, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::v8f32, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::v4f64, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::v8f32, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::v4f64, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::v8f32, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::v4f64, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::v8f32, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::v4f64, Legal);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i32, Custom);
+
+ setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Custom);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i32, Custom);
+ setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Expand);
+ setOperationAction(ISD::FP_ROUND, MVT::v8f16, Expand);
+ setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Custom);
+
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f32, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::v8f32, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::v4f64, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::v8f32, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::v4f64, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::v8f32, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::v4f64, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::v8f32, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::v4f64, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::v8f32, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::v4f64, Legal);
if (!Subtarget.hasAVX512())
setOperationAction(ISD::BITCAST, MVT::v32i1, Custom);
// In the customized shift lowering, the legal v8i32/v4i64 cases
// in AVX2 will be recognized.
- for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
- setOperationAction(ISD::SRL, VT, Custom);
- setOperationAction(ISD::SHL, VT, Custom);
- setOperationAction(ISD::SRA, VT, Custom);
- setOperationAction(ISD::ABDS, VT, Custom);
- setOperationAction(ISD::ABDU, VT, Custom);
- setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
- setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
- setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
- setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
- setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
- setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
- setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
- setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
- if (VT == MVT::v4i64) continue;
- setOperationAction(ISD::ROTL, VT, Custom);
- setOperationAction(ISD::ROTR, VT, Custom);
- setOperationAction(ISD::FSHL, VT, Custom);
- setOperationAction(ISD::FSHR, VT, Custom);
+ for (auto VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
+ setOperationAction(ISD::SRL, VT, Custom);
+ setOperationAction(ISD::SHL, VT, Custom);
+ setOperationAction(ISD::SRA, VT, Custom);
+ setOperationAction(ISD::ABDS, VT, Custom);
+ setOperationAction(ISD::ABDU, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
+ if (VT == MVT::v4i64)
+ continue;
+ setOperationAction(ISD::ROTL, VT, Custom);
+ setOperationAction(ISD::ROTR, VT, Custom);
+ setOperationAction(ISD::FSHL, VT, Custom);
+ setOperationAction(ISD::FSHR, VT, Custom);
}
// These types need custom splitting if their input is a 128-bit vector.
- setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
- setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
- setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
- setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
-
- setOperationAction(ISD::SELECT, MVT::v4f64, Custom);
- setOperationAction(ISD::SELECT, MVT::v4i64, Custom);
- setOperationAction(ISD::SELECT, MVT::v8i32, Custom);
- setOperationAction(ISD::SELECT, MVT::v16i16, Custom);
- setOperationAction(ISD::SELECT, MVT::v16f16, Custom);
- setOperationAction(ISD::SELECT, MVT::v32i8, Custom);
- setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
-
- for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
- setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
- setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
- setOperationAction(ISD::ANY_EXTEND, VT, Custom);
- }
-
- setOperationAction(ISD::TRUNCATE, MVT::v32i8, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v32i16, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v32i32, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v32i64, Custom);
-
- for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
- setOperationAction(ISD::SETCC, VT, Custom);
- setOperationAction(ISD::CTPOP, VT, Custom);
- setOperationAction(ISD::CTLZ, VT, Custom);
- setOperationAction(ISD::BITREVERSE, VT, Custom);
+ setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
+ setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
+
+ setOperationAction(ISD::SELECT, MVT::v4f64, Custom);
+ setOperationAction(ISD::SELECT, MVT::v4i64, Custom);
+ setOperationAction(ISD::SELECT, MVT::v8i32, Custom);
+ setOperationAction(ISD::SELECT, MVT::v16i16, Custom);
+ setOperationAction(ISD::SELECT, MVT::v16f16, Custom);
+ setOperationAction(ISD::SELECT, MVT::v32i8, Custom);
+ setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
+
+ for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
+ setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
+ setOperationAction(ISD::ANY_EXTEND, VT, Custom);
+ }
+
+ setOperationAction(ISD::TRUNCATE, MVT::v32i8, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v32i16, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v32i32, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v32i64, Custom);
+
+ for (auto VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
+ setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::CTPOP, VT, Custom);
+ setOperationAction(ISD::CTLZ, VT, Custom);
+ setOperationAction(ISD::BITREVERSE, VT, Custom);
// The condition codes aren't legal in SSE/AVX and under AVX512 we use
// setcc all the way to isel and prefer SETGT in some isel patterns.
@@ -1629,64 +1630,64 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setCondCodeAction(ISD::SETLE, VT, Custom);
}
- setOperationAction(ISD::SETCC, MVT::v4f64, Custom);
- setOperationAction(ISD::SETCC, MVT::v8f32, Custom);
- setOperationAction(ISD::STRICT_FSETCC, MVT::v4f64, Custom);
- setOperationAction(ISD::STRICT_FSETCC, MVT::v8f32, Custom);
+ setOperationAction(ISD::SETCC, MVT::v4f64, Custom);
+ setOperationAction(ISD::SETCC, MVT::v8f32, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, MVT::v4f64, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, MVT::v8f32, Custom);
setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f64, Custom);
setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f32, Custom);
if (Subtarget.hasAnyFMA()) {
- for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32,
- MVT::v2f64, MVT::v4f64 }) {
+ for (auto VT : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32, MVT::v2f64,
+ MVT::v4f64}) {
setOperationAction(ISD::FMA, VT, Legal);
setOperationAction(ISD::STRICT_FMA, VT, Legal);
}
}
- for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
+ for (auto VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom);
setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom);
}
- setOperationAction(ISD::MUL, MVT::v4i64, Custom);
- setOperationAction(ISD::MUL, MVT::v8i32, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::MUL, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::MUL, MVT::v32i8, Custom);
-
- setOperationAction(ISD::MULHU, MVT::v8i32, Custom);
- setOperationAction(ISD::MULHS, MVT::v8i32, Custom);
- setOperationAction(ISD::MULHU, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::MULHS, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::MULHU, MVT::v32i8, Custom);
- setOperationAction(ISD::MULHS, MVT::v32i8, Custom);
- setOperationAction(ISD::AVGCEILU, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::AVGCEILU, MVT::v32i8, HasInt256 ? Legal : Custom);
-
- setOperationAction(ISD::SMULO, MVT::v32i8, Custom);
- setOperationAction(ISD::UMULO, MVT::v32i8, Custom);
-
- setOperationAction(ISD::ABS, MVT::v4i64, Custom);
- setOperationAction(ISD::SMAX, MVT::v4i64, Custom);
- setOperationAction(ISD::UMAX, MVT::v4i64, Custom);
- setOperationAction(ISD::SMIN, MVT::v4i64, Custom);
- setOperationAction(ISD::UMIN, MVT::v4i64, Custom);
-
- setOperationAction(ISD::UADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::SADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::USUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::SSUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::UADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::SADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::USUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::SSUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::UADDSAT, MVT::v8i32, Custom);
- setOperationAction(ISD::USUBSAT, MVT::v8i32, Custom);
- setOperationAction(ISD::UADDSAT, MVT::v4i64, Custom);
- setOperationAction(ISD::USUBSAT, MVT::v4i64, Custom);
-
- for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) {
- setOperationAction(ISD::ABS, VT, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::MUL, MVT::v4i64, Custom);
+ setOperationAction(ISD::MUL, MVT::v8i32, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::MUL, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::MUL, MVT::v32i8, Custom);
+
+ setOperationAction(ISD::MULHU, MVT::v8i32, Custom);
+ setOperationAction(ISD::MULHS, MVT::v8i32, Custom);
+ setOperationAction(ISD::MULHU, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::MULHS, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::MULHU, MVT::v32i8, Custom);
+ setOperationAction(ISD::MULHS, MVT::v32i8, Custom);
+ setOperationAction(ISD::AVGCEILU, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::AVGCEILU, MVT::v32i8, HasInt256 ? Legal : Custom);
+
+ setOperationAction(ISD::SMULO, MVT::v32i8, Custom);
+ setOperationAction(ISD::UMULO, MVT::v32i8, Custom);
+
+ setOperationAction(ISD::ABS, MVT::v4i64, Custom);
+ setOperationAction(ISD::SMAX, MVT::v4i64, Custom);
+ setOperationAction(ISD::UMAX, MVT::v4i64, Custom);
+ setOperationAction(ISD::SMIN, MVT::v4i64, Custom);
+ setOperationAction(ISD::UMIN, MVT::v4i64, Custom);
+
+ setOperationAction(ISD::UADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::SADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::USUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::SSUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::UADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::SADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::USUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::SSUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::UADDSAT, MVT::v8i32, Custom);
+ setOperationAction(ISD::USUBSAT, MVT::v8i32, Custom);
+ setOperationAction(ISD::UADDSAT, MVT::v4i64, Custom);
+ setOperationAction(ISD::USUBSAT, MVT::v4i64, Custom);
+
+ for (auto VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32}) {
+ setOperationAction(ISD::ABS, VT, HasInt256 ? Legal : Custom);
setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom);
setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom);
setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom);
@@ -1705,41 +1706,41 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i32, Custom);
// AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
- for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
+ for (auto LoadExtOp : {ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
setLoadExtAction(LoadExtOp, MVT::v16i16, MVT::v16i8, Legal);
- setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i8, Legal);
- setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i8, Legal);
- setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i16, Legal);
- setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i16, Legal);
- setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i32, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i16, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i16, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i32, Legal);
}
}
- for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
- MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) {
- setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
+ for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, MVT::v4f32,
+ MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
+ setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
setOperationAction(ISD::MSTORE, VT, Legal);
}
// Extract subvector is special because the value type
// (result) is 128-bit but the source is 256-bit wide.
- for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
- MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
+ for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v8f16,
+ MVT::v4f32, MVT::v2f64}) {
setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
}
// Custom lower several nodes for 256-bit types.
- for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
- MVT::v16f16, MVT::v8f32, MVT::v4f64 }) {
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
- setOperationAction(ISD::VSELECT, VT, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
+ for (MVT VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v16f16,
+ MVT::v8f32, MVT::v4f64}) {
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::VSELECT, VT, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
- setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
- setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
- setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
- setOperationAction(ISD::STORE, VT, Custom);
+ setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
+ setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
+ setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
+ setOperationAction(ISD::STORE, VT, Custom);
}
setF16Action(MVT::v16f16, Expand);
setOperationAction(ISD::FNEG, MVT::v16f16, Custom);
@@ -1770,9 +1771,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::MGATHER, MVT::v2f32, Custom);
setOperationAction(ISD::MGATHER, MVT::v2i32, Custom);
- for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
- MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
- setOperationAction(ISD::MGATHER, VT, Custom);
+ for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
+ MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64})
+ setOperationAction(ISD::MGATHER, VT, Custom);
}
if (Subtarget.hasGFNI()) {
@@ -1783,13 +1784,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
if (!Subtarget.useSoftFloat() && !Subtarget.hasFP16() &&
Subtarget.hasF16C()) {
- for (MVT VT : { MVT::f16, MVT::v2f16, MVT::v4f16, MVT::v8f16 }) {
- setOperationAction(ISD::FP_ROUND, VT, Custom);
- setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom);
+ for (MVT VT : {MVT::f16, MVT::v2f16, MVT::v4f16, MVT::v8f16}) {
+ setOperationAction(ISD::FP_ROUND, VT, Custom);
+ setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom);
}
- for (MVT VT : { MVT::f32, MVT::v2f32, MVT::v4f32, MVT::v8f32 }) {
- setOperationAction(ISD::FP_EXTEND, VT, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, VT, Custom);
+ for (MVT VT : {MVT::f32, MVT::v2f32, MVT::v4f32, MVT::v8f32}) {
+ setOperationAction(ISD::FP_EXTEND, VT, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, VT, Custom);
}
for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
setOperationPromotedToType(Opc, MVT::v8f16, MVT::v8f32);
@@ -1803,28 +1804,28 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// available with AVX512. 512-bit vectors are in a separate block controlled
// by useAVX512Regs.
if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
- addRegisterClass(MVT::v1i1, &X86::VK1RegClass);
- addRegisterClass(MVT::v2i1, &X86::VK2RegClass);
- addRegisterClass(MVT::v4i1, &X86::VK4RegClass);
- addRegisterClass(MVT::v8i1, &X86::VK8RegClass);
- addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
+ addRegisterClass(MVT::v1i1, &X86::VK1RegClass);
+ addRegisterClass(MVT::v2i1, &X86::VK2RegClass);
+ addRegisterClass(MVT::v4i1, &X86::VK4RegClass);
+ addRegisterClass(MVT::v8i1, &X86::VK8RegClass);
+ addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
- setOperationAction(ISD::SELECT, MVT::v1i1, Custom);
+ setOperationAction(ISD::SELECT, MVT::v1i1, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom);
- setOperationAction(ISD::BUILD_VECTOR, MVT::v1i1, Custom);
-
- setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i1, MVT::v8i32);
- setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i1, MVT::v8i32);
- setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v4i1, MVT::v4i32);
- setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v4i1, MVT::v4i32);
- setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i1, MVT::v8i32);
- setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i1, MVT::v8i32);
- setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v4i1, MVT::v4i32);
- setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v4i1, MVT::v4i32);
- setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Custom);
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i1, Custom);
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i1, Custom);
+ setOperationAction(ISD::BUILD_VECTOR, MVT::v1i1, Custom);
+
+ setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i1, MVT::v8i32);
+ setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i1, MVT::v8i32);
+ setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v4i1, MVT::v4i32);
+ setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v4i1, MVT::v4i32);
+ setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i1, MVT::v8i32);
+ setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i1, MVT::v8i32);
+ setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v4i1, MVT::v4i32);
+ setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v4i1, MVT::v4i32);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i1, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i1, Custom);
setOperationAction(ISD::FCANONICALIZE, MVT::v8f16, Expand);
setOperationAction(ISD::FCANONICALIZE, MVT::v16f16, Expand);
setOperationAction(ISD::FCANONICALIZE, MVT::v32f16, Expand);
@@ -1843,29 +1844,29 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
// Extends of v16i1/v8i1/v4i1/v2i1 to 128-bit vectors.
- for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
+ for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
- setOperationAction(ISD::ANY_EXTEND, VT, Custom);
+ setOperationAction(ISD::ANY_EXTEND, VT, Custom);
}
- for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 })
- setOperationAction(ISD::VSELECT, VT, Expand);
+ for (auto VT : {MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1})
+ setOperationAction(ISD::VSELECT, VT, Expand);
- for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {
- setOperationAction(ISD::SETCC, VT, Custom);
- setOperationAction(ISD::SELECT, VT, Custom);
- setOperationAction(ISD::TRUNCATE, VT, Custom);
+ for (auto VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1}) {
+ setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::TRUNCATE, VT, Custom);
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
- setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
}
- for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 })
+ for (auto VT : {MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1})
setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
}
if (Subtarget.hasDQI() && Subtarget.hasVLX()) {
@@ -1883,27 +1884,27 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
- addRegisterClass(MVT::v8i64, &X86::VR512RegClass);
- addRegisterClass(MVT::v8f64, &X86::VR512RegClass);
+ addRegisterClass(MVT::v8i64, &X86::VR512RegClass);
+ addRegisterClass(MVT::v8f64, &X86::VR512RegClass);
addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
addRegisterClass(MVT::v32f16, &X86::VR512RegClass);
- addRegisterClass(MVT::v64i8, &X86::VR512RegClass);
+ addRegisterClass(MVT::v64i8, &X86::VR512RegClass);
for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD}) {
- setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8, Legal);
+ setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8, Legal);
setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i16, Legal);
- setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i8, Legal);
- setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i16, Legal);
- setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i32, Legal);
+ setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i8, Legal);
+ setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i16, Legal);
+ setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i32, Legal);
if (HasBWI)
setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal);
}
- for (MVT VT : { MVT::v16f32, MVT::v8f64 }) {
+ for (MVT VT : {MVT::v16f32, MVT::v8f64}) {
SetFPMinMaxAction(VT);
- setOperationAction(ISD::FNEG, VT, Custom);
- setOperationAction(ISD::FABS, VT, Custom);
- setOperationAction(ISD::FMA, VT, Legal);
+ setOperationAction(ISD::FNEG, VT, Custom);
+ setOperationAction(ISD::FABS, VT, Custom);
+ setOperationAction(ISD::FMA, VT, Legal);
setOperationAction(ISD::STRICT_FMA, VT, Legal);
setOperationAction(ISD::FCOPYSIGN, VT, Custom);
setOperationAction(ISD::FCANONICALIZE, VT, Expand);
@@ -1931,93 +1932,93 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SELECT, MVT::i512, Custom);
setOperationAction(ISD::BITREVERSE, MVT::i512, Custom);
- for (MVT VT : { MVT::v16i1, MVT::v16i8 }) {
- setOperationPromotedToType(ISD::FP_TO_SINT , VT, MVT::v16i32);
- setOperationPromotedToType(ISD::FP_TO_UINT , VT, MVT::v16i32);
+ for (MVT VT : {MVT::v16i1, MVT::v16i8}) {
+ setOperationPromotedToType(ISD::FP_TO_SINT, VT, MVT::v16i32);
+ setOperationPromotedToType(ISD::FP_TO_UINT, VT, MVT::v16i32);
setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, VT, MVT::v16i32);
setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, VT, MVT::v16i32);
}
- for (MVT VT : { MVT::v16i16, MVT::v16i32 }) {
- setOperationAction(ISD::FP_TO_SINT, VT, Custom);
- setOperationAction(ISD::FP_TO_UINT, VT, Custom);
+ for (MVT VT : {MVT::v16i16, MVT::v16i32}) {
+ setOperationAction(ISD::FP_TO_SINT, VT, Custom);
+ setOperationAction(ISD::FP_TO_UINT, VT, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
}
- setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Custom);
- setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i32, Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i32, Custom);
- setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Custom);
-
- setOperationAction(ISD::STRICT_FADD, MVT::v16f32, Legal);
- setOperationAction(ISD::STRICT_FADD, MVT::v8f64, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::v16f32, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::v8f64, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::v16f32, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::v8f64, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::v16f32, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::v8f64, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::v16f32, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::v8f64, Legal);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f32, Legal);
-
- setTruncStoreAction(MVT::v8i64, MVT::v8i8, Legal);
- setTruncStoreAction(MVT::v8i64, MVT::v8i16, Legal);
- setTruncStoreAction(MVT::v8i64, MVT::v8i32, Legal);
- setTruncStoreAction(MVT::v16i32, MVT::v16i8, Legal);
- setTruncStoreAction(MVT::v16i32, MVT::v16i16, Legal);
+ setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Custom);
+
+ setOperationAction(ISD::STRICT_FADD, MVT::v16f32, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::v8f64, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::v16f32, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::v8f64, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::v16f32, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::v8f64, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::v16f32, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::v8f64, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::v16f32, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::v8f64, Legal);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f32, Legal);
+
+ setTruncStoreAction(MVT::v8i64, MVT::v8i8, Legal);
+ setTruncStoreAction(MVT::v8i64, MVT::v8i16, Legal);
+ setTruncStoreAction(MVT::v8i64, MVT::v8i32, Legal);
+ setTruncStoreAction(MVT::v16i32, MVT::v16i8, Legal);
+ setTruncStoreAction(MVT::v16i32, MVT::v16i16, Legal);
if (HasBWI)
- setTruncStoreAction(MVT::v32i16, MVT::v32i8, Legal);
+ setTruncStoreAction(MVT::v32i16, MVT::v32i8, Legal);
// With 512-bit vectors and no VLX, we prefer to widen MLOAD/MSTORE
// to 512-bit rather than use the AVX2 instructions so that we can use
// k-masks.
if (!Subtarget.hasVLX()) {
for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
- MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
- setOperationAction(ISD::MLOAD, VT, Custom);
+ MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
+ setOperationAction(ISD::MLOAD, VT, Custom);
setOperationAction(ISD::MSTORE, VT, Custom);
}
}
- setOperationAction(ISD::TRUNCATE, MVT::v8i32, Legal);
- setOperationAction(ISD::TRUNCATE, MVT::v16i16, Legal);
- setOperationAction(ISD::TRUNCATE, MVT::v32i8, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v8i32, Legal);
+ setOperationAction(ISD::TRUNCATE, MVT::v16i16, Legal);
+ setOperationAction(ISD::TRUNCATE, MVT::v32i8, HasBWI ? Legal : Custom);
setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom);
setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
- setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
- setOperationAction(ISD::ANY_EXTEND, MVT::v32i16, Custom);
- setOperationAction(ISD::ANY_EXTEND, MVT::v16i32, Custom);
- setOperationAction(ISD::ANY_EXTEND, MVT::v8i64, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
+ setOperationAction(ISD::ANY_EXTEND, MVT::v32i16, Custom);
+ setOperationAction(ISD::ANY_EXTEND, MVT::v16i32, Custom);
+ setOperationAction(ISD::ANY_EXTEND, MVT::v8i64, Custom);
setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom);
setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
- setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
+ setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
if (HasBWI) {
// Extends from v64i1 masks to 512-bit vectors.
- setOperationAction(ISD::SIGN_EXTEND, MVT::v64i8, Custom);
- setOperationAction(ISD::ZERO_EXTEND, MVT::v64i8, Custom);
- setOperationAction(ISD::ANY_EXTEND, MVT::v64i8, Custom);
- }
-
- for (auto VT : { MVT::v16f32, MVT::v8f64 }) {
- setOperationAction(ISD::FFLOOR, VT, Legal);
- setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
- setOperationAction(ISD::FCEIL, VT, Legal);
- setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
- setOperationAction(ISD::FTRUNC, VT, Legal);
- setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
- setOperationAction(ISD::FRINT, VT, Legal);
- setOperationAction(ISD::STRICT_FRINT, VT, Legal);
- setOperationAction(ISD::FNEARBYINT, VT, Legal);
+ setOperationAction(ISD::SIGN_EXTEND, MVT::v64i8, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, MVT::v64i8, Custom);
+ setOperationAction(ISD::ANY_EXTEND, MVT::v64i8, Custom);
+ }
+
+ for (auto VT : {MVT::v16f32, MVT::v8f64}) {
+ setOperationAction(ISD::FFLOOR, VT, Legal);
+ setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
+ setOperationAction(ISD::FCEIL, VT, Legal);
+ setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
+ setOperationAction(ISD::FTRUNC, VT, Legal);
+ setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
+ setOperationAction(ISD::FRINT, VT, Legal);
+ setOperationAction(ISD::STRICT_FRINT, VT, Legal);
+ setOperationAction(ISD::FNEARBYINT, VT, Legal);
setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
- setOperationAction(ISD::FROUNDEVEN, VT, Legal);
+ setOperationAction(ISD::FROUNDEVEN, VT, Legal);
setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
- setOperationAction(ISD::FROUND, VT, Custom);
+ setOperationAction(ISD::FROUND, VT, Custom);
}
for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
@@ -2027,13 +2028,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::ADD, MVT::v32i16, HasBWI ? Legal : Custom);
setOperationAction(ISD::SUB, MVT::v32i16, HasBWI ? Legal : Custom);
- setOperationAction(ISD::ADD, MVT::v64i8, HasBWI ? Legal : Custom);
- setOperationAction(ISD::SUB, MVT::v64i8, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::ADD, MVT::v64i8, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::SUB, MVT::v64i8, HasBWI ? Legal : Custom);
- setOperationAction(ISD::MUL, MVT::v8i64, Custom);
+ setOperationAction(ISD::MUL, MVT::v8i64, Custom);
setOperationAction(ISD::MUL, MVT::v16i32, Legal);
setOperationAction(ISD::MUL, MVT::v32i16, HasBWI ? Legal : Custom);
- setOperationAction(ISD::MUL, MVT::v64i8, Custom);
+ setOperationAction(ISD::MUL, MVT::v64i8, Custom);
setOperationAction(ISD::MULHU, MVT::v8i64, Custom);
setOperationAction(ISD::UMUL_LOHI, MVT::v8i64, Custom);
@@ -2042,32 +2043,32 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::MULHS, MVT::v16i32, Custom);
setOperationAction(ISD::MULHS, MVT::v32i16, HasBWI ? Legal : Custom);
setOperationAction(ISD::MULHU, MVT::v32i16, HasBWI ? Legal : Custom);
- setOperationAction(ISD::MULHS, MVT::v64i8, Custom);
- setOperationAction(ISD::MULHU, MVT::v64i8, Custom);
+ setOperationAction(ISD::MULHS, MVT::v64i8, Custom);
+ setOperationAction(ISD::MULHU, MVT::v64i8, Custom);
setOperationAction(ISD::AVGCEILU, MVT::v32i16, HasBWI ? Legal : Custom);
- setOperationAction(ISD::AVGCEILU, MVT::v64i8, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::AVGCEILU, MVT::v64i8, HasBWI ? Legal : Custom);
setOperationAction(ISD::SMULO, MVT::v64i8, Custom);
setOperationAction(ISD::UMULO, MVT::v64i8, Custom);
- for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64 }) {
- setOperationAction(ISD::SRL, VT, Custom);
- setOperationAction(ISD::SHL, VT, Custom);
- setOperationAction(ISD::SRA, VT, Custom);
- setOperationAction(ISD::ROTL, VT, Custom);
- setOperationAction(ISD::ROTR, VT, Custom);
- setOperationAction(ISD::SETCC, VT, Custom);
- setOperationAction(ISD::ABDS, VT, Custom);
- setOperationAction(ISD::ABDU, VT, Custom);
- setOperationAction(ISD::BITREVERSE, VT, Custom);
- setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
- setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
- setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
- setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
- setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
- setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
- setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
- setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
+ for (auto VT : {MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
+ setOperationAction(ISD::SRL, VT, Custom);
+ setOperationAction(ISD::SHL, VT, Custom);
+ setOperationAction(ISD::SRA, VT, Custom);
+ setOperationAction(ISD::ROTL, VT, Custom);
+ setOperationAction(ISD::ROTR, VT, Custom);
+ setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::ABDS, VT, Custom);
+ setOperationAction(ISD::ABDU, VT, Custom);
+ setOperationAction(ISD::BITREVERSE, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
// The condition codes aren't legal in SSE/AVX and under AVX512 we use
// setcc all the way to isel and prefer SETGT in some isel patterns.
@@ -2075,53 +2076,54 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setCondCodeAction(ISD::SETLE, VT, Custom);
}
- setOperationAction(ISD::SETCC, MVT::v8f64, Custom);
- setOperationAction(ISD::SETCC, MVT::v16f32, Custom);
- setOperationAction(ISD::STRICT_FSETCC, MVT::v8f64, Custom);
- setOperationAction(ISD::STRICT_FSETCC, MVT::v16f32, Custom);
+ setOperationAction(ISD::SETCC, MVT::v8f64, Custom);
+ setOperationAction(ISD::SETCC, MVT::v16f32, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, MVT::v8f64, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, MVT::v16f32, Custom);
setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f64, Custom);
setOperationAction(ISD::STRICT_FSETCCS, MVT::v16f32, Custom);
- for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
- setOperationAction(ISD::SMAX, VT, Legal);
- setOperationAction(ISD::UMAX, VT, Legal);
- setOperationAction(ISD::SMIN, VT, Legal);
- setOperationAction(ISD::UMIN, VT, Legal);
- setOperationAction(ISD::ROTL, VT, Legal);
- setOperationAction(ISD::ROTR, VT, Legal);
- setOperationAction(ISD::ABS, VT, Legal);
- setOperationAction(ISD::CTPOP, VT, Custom);
- }
-
- for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
- setOperationAction(ISD::ABS, VT, HasBWI ? Legal : Custom);
- setOperationAction(ISD::CTPOP, VT, Subtarget.hasBITALG() ? Legal : Custom);
- setOperationAction(ISD::CTLZ, VT, Custom);
- setOperationAction(ISD::SMAX, VT, HasBWI ? Legal : Custom);
- setOperationAction(ISD::UMAX, VT, HasBWI ? Legal : Custom);
- setOperationAction(ISD::SMIN, VT, HasBWI ? Legal : Custom);
- setOperationAction(ISD::UMIN, VT, HasBWI ? Legal : Custom);
+ for (auto VT : {MVT::v16i32, MVT::v8i64}) {
+ setOperationAction(ISD::SMAX, VT, Legal);
+ setOperationAction(ISD::UMAX, VT, Legal);
+ setOperationAction(ISD::SMIN, VT, Legal);
+ setOperationAction(ISD::UMIN, VT, Legal);
+ setOperationAction(ISD::ROTL, VT, Legal);
+ setOperationAction(ISD::ROTR, VT, Legal);
+ setOperationAction(ISD::ABS, VT, Legal);
+ setOperationAction(ISD::CTPOP, VT, Custom);
+ }
+
+ for (auto VT : {MVT::v64i8, MVT::v32i16}) {
+ setOperationAction(ISD::ABS, VT, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::CTPOP, VT,
+ Subtarget.hasBITALG() ? Legal : Custom);
+ setOperationAction(ISD::CTLZ, VT, Custom);
+ setOperationAction(ISD::SMAX, VT, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::UMAX, VT, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::SMIN, VT, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::UMIN, VT, HasBWI ? Legal : Custom);
setOperationAction(ISD::UADDSAT, VT, HasBWI ? Legal : Custom);
setOperationAction(ISD::SADDSAT, VT, HasBWI ? Legal : Custom);
setOperationAction(ISD::USUBSAT, VT, HasBWI ? Legal : Custom);
setOperationAction(ISD::SSUBSAT, VT, HasBWI ? Legal : Custom);
}
- setOperationAction(ISD::FSHL, MVT::v64i8, Custom);
- setOperationAction(ISD::FSHR, MVT::v64i8, Custom);
- setOperationAction(ISD::FSHL, MVT::v32i16, Custom);
- setOperationAction(ISD::FSHR, MVT::v32i16, Custom);
- setOperationAction(ISD::FSHL, MVT::v16i32, Custom);
- setOperationAction(ISD::FSHR, MVT::v16i32, Custom);
+ setOperationAction(ISD::FSHL, MVT::v64i8, Custom);
+ setOperationAction(ISD::FSHR, MVT::v64i8, Custom);
+ setOperationAction(ISD::FSHL, MVT::v32i16, Custom);
+ setOperationAction(ISD::FSHR, MVT::v32i16, Custom);
+ setOperationAction(ISD::FSHL, MVT::v16i32, Custom);
+ setOperationAction(ISD::FSHR, MVT::v16i32, Custom);
if (Subtarget.hasDQI() || Subtarget.hasFP16())
for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT})
- setOperationAction(Opc, MVT::v8i64, Custom);
+ setOperationAction(Opc, MVT::v8i64, Custom);
if (Subtarget.hasDQI()) {
- setOperationAction(ISD::MUL, MVT::v8i64, Legal);
+ setOperationAction(ISD::MUL, MVT::v8i64, Legal);
// MULHS needs vpmullq (AVX512DQ) for its low multiply to be a win.
setOperationAction(ISD::MULHS, MVT::v8i64, Custom);
@@ -2129,13 +2131,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
if (Subtarget.hasCDI()) {
// NonVLX sub-targets extend 128/256 vectors to use the 512 version.
- for (auto VT : { MVT::v16i32, MVT::v8i64} ) {
- setOperationAction(ISD::CTLZ, VT, Legal);
+ for (auto VT : {MVT::v16i32, MVT::v8i64}) {
+ setOperationAction(ISD::CTLZ, VT, Legal);
}
} // Subtarget.hasCDI()
if (Subtarget.hasVPOPCNTDQ()) {
- for (auto VT : { MVT::v16i32, MVT::v8i64 })
+ for (auto VT : {MVT::v16i32, MVT::v8i64})
setOperationAction(ISD::CTPOP, VT, Legal);
setOperationAction(ISD::CTPOP, MVT::i512, Custom);
}
@@ -2143,21 +2145,21 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// Extract subvector is special because the value type
// (result) is 256-bit but the source is 512-bit wide.
// 128-bit was made Legal under AVX1.
- for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
- MVT::v16f16, MVT::v8f32, MVT::v4f64 })
+ for (auto VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
+ MVT::v16f16, MVT::v8f32, MVT::v4f64})
setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
- for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64,
- MVT::v32f16, MVT::v16f32, MVT::v8f64 }) {
- setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
- setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
- setOperationAction(ISD::SELECT, VT, Custom);
- setOperationAction(ISD::VSELECT, VT, Custom);
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ for (auto VT : {MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64,
+ MVT::v32f16, MVT::v16f32, MVT::v8f64}) {
+ setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
+ setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
+ setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::VSELECT, VT, Custom);
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
- setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
}
setF16Action(MVT::v32f16, Expand);
setOperationAction(ISD::FP_ROUND, MVT::v16f16, Custom);
@@ -2168,20 +2170,20 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationPromotedToType(Opc, MVT::v32f16, MVT::v32f32);
setOperationAction(ISD::SETCC, MVT::v32f16, Custom);
- for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) {
- setOperationAction(ISD::MLOAD, VT, Legal);
- setOperationAction(ISD::MSTORE, VT, Legal);
- setOperationAction(ISD::MGATHER, VT, Custom);
- setOperationAction(ISD::MSCATTER, VT, Custom);
+ for (auto VT : {MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64}) {
+ setOperationAction(ISD::MLOAD, VT, Legal);
+ setOperationAction(ISD::MSTORE, VT, Legal);
+ setOperationAction(ISD::MGATHER, VT, Custom);
+ setOperationAction(ISD::MSCATTER, VT, Custom);
}
if (HasBWI) {
- for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
- setOperationAction(ISD::MLOAD, VT, Legal);
- setOperationAction(ISD::MSTORE, VT, Legal);
+ for (auto VT : {MVT::v64i8, MVT::v32i16}) {
+ setOperationAction(ISD::MLOAD, VT, Legal);
+ setOperationAction(ISD::MSTORE, VT, Legal);
}
} else {
setOperationAction(ISD::STORE, MVT::v32i16, Custom);
- setOperationAction(ISD::STORE, MVT::v64i8, Custom);
+ setOperationAction(ISD::STORE, MVT::v64i8, Custom);
}
if (Subtarget.hasVBMI2()) {
@@ -2211,7 +2213,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::CTLZ, MVT::v64i8, Custom);
setOperationAction(ISD::CTTZ, MVT::v64i8, Custom);
}
- }// useAVX512Regs
+ } // useAVX512Regs
if (!Subtarget.useSoftFloat() && Subtarget.hasVBMI2()) {
for (auto VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v16i16, MVT::v8i32,
@@ -2236,9 +2238,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// These operations are handled on non-VLX by artificially widening in
// isel patterns.
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i32, Custom);
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Custom);
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i32, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i32, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i32, Custom);
if (Subtarget.hasDQI()) {
// Fast v2f32 SINT_TO_FP( v2i64 ) custom conversion.
@@ -2247,31 +2249,31 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
isOperationCustom(ISD::STRICT_UINT_TO_FP, MVT::v2f32) &&
"Unexpected operation action!");
// v2i64 FP_TO_S/UINT(v2f32) custom conversion.
- setOperationAction(ISD::FP_TO_SINT, MVT::v2f32, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v2f32, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v2f32, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v2f32, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f32, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f32, Custom);
}
- for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
+ for (auto VT : {MVT::v2i64, MVT::v4i64}) {
setOperationAction(ISD::SMAX, VT, Legal);
setOperationAction(ISD::UMAX, VT, Legal);
setOperationAction(ISD::SMIN, VT, Legal);
setOperationAction(ISD::UMIN, VT, Legal);
- setOperationAction(ISD::ABS, VT, Legal);
+ setOperationAction(ISD::ABS, VT, Legal);
}
- for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
- setOperationAction(ISD::ROTL, VT, Legal);
- setOperationAction(ISD::ROTR, VT, Legal);
+ for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64}) {
+ setOperationAction(ISD::ROTL, VT, Legal);
+ setOperationAction(ISD::ROTR, VT, Legal);
}
// Custom legalize 2x32 to get a little better code.
setOperationAction(ISD::MSCATTER, MVT::v2f32, Custom);
setOperationAction(ISD::MSCATTER, MVT::v2i32, Custom);
- for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
- MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
+ for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, MVT::v4f32,
+ MVT::v8f32, MVT::v2f64, MVT::v4f64})
setOperationAction(ISD::MSCATTER, VT, Custom);
if (Subtarget.hasDQI()) {
@@ -2298,8 +2300,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::CTLZ_ZERO_POISON, VT, Custom);
setOperationAction(ISD::CTTZ_ZERO_POISON, VT, Custom);
}
- for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
- setOperationAction(ISD::CTLZ, VT, Legal);
+ for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64}) {
+ setOperationAction(ISD::CTLZ, VT, Legal);
}
} // Subtarget.hasCDI()
@@ -2342,33 +2344,33 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// This block control legalization of v32i1/v64i1 which are available with
// AVX512BW..
if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
- addRegisterClass(MVT::v32i1, &X86::VK32RegClass);
- addRegisterClass(MVT::v64i1, &X86::VK64RegClass);
+ addRegisterClass(MVT::v32i1, &X86::VK32RegClass);
+ addRegisterClass(MVT::v64i1, &X86::VK64RegClass);
- for (auto VT : { MVT::v32i1, MVT::v64i1 }) {
- setOperationAction(ISD::VSELECT, VT, Expand);
- setOperationAction(ISD::TRUNCATE, VT, Custom);
- setOperationAction(ISD::SETCC, VT, Custom);
+ for (auto VT : {MVT::v32i1, MVT::v64i1}) {
+ setOperationAction(ISD::VSELECT, VT, Expand);
+ setOperationAction(ISD::TRUNCATE, VT, Custom);
+ setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
- setOperationAction(ISD::SELECT, VT, Custom);
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
- setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
- setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
+ setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
+ setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
}
- for (auto VT : { MVT::v16i1, MVT::v32i1 })
+ for (auto VT : {MVT::v16i1, MVT::v32i1})
setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
// Extends from v32i1 masks to 256-bit vectors.
- setOperationAction(ISD::SIGN_EXTEND, MVT::v32i8, Custom);
- setOperationAction(ISD::ZERO_EXTEND, MVT::v32i8, Custom);
- setOperationAction(ISD::ANY_EXTEND, MVT::v32i8, Custom);
+ setOperationAction(ISD::SIGN_EXTEND, MVT::v32i8, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, MVT::v32i8, Custom);
+ setOperationAction(ISD::ANY_EXTEND, MVT::v32i8, Custom);
for (auto VT : {MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16,
MVT::v16f16, MVT::v8f16}) {
- setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
+ setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom);
}
@@ -2377,7 +2379,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// TODO: Custom widen in lowering on non-VLX and drop the isel patterns?
if (Subtarget.hasBITALG()) {
- for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 })
+ for (auto VT : {MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16})
setOperationAction(ISD::CTPOP, VT, Legal);
}
@@ -2393,110 +2395,110 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
if (!Subtarget.useSoftFloat() && Subtarget.hasFP16()) {
- auto setGroup = [&] (MVT VT) {
- setOperationAction(ISD::FADD, VT, Legal);
- setOperationAction(ISD::STRICT_FADD, VT, Legal);
- setOperationAction(ISD::FSUB, VT, Legal);
- setOperationAction(ISD::STRICT_FSUB, VT, Legal);
- setOperationAction(ISD::FMUL, VT, Legal);
- setOperationAction(ISD::STRICT_FMUL, VT, Legal);
- setOperationAction(ISD::FDIV, VT, Legal);
- setOperationAction(ISD::STRICT_FDIV, VT, Legal);
- setOperationAction(ISD::FSQRT, VT, Legal);
- setOperationAction(ISD::STRICT_FSQRT, VT, Legal);
-
- setOperationAction(ISD::FFLOOR, VT, Legal);
- setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
- setOperationAction(ISD::FCEIL, VT, Legal);
- setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
- setOperationAction(ISD::FTRUNC, VT, Legal);
- setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
- setOperationAction(ISD::FRINT, VT, Legal);
- setOperationAction(ISD::STRICT_FRINT, VT, Legal);
- setOperationAction(ISD::FNEARBYINT, VT, Legal);
- setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
+ auto setGroup = [&](MVT VT) {
+ setOperationAction(ISD::FADD, VT, Legal);
+ setOperationAction(ISD::STRICT_FADD, VT, Legal);
+ setOperationAction(ISD::FSUB, VT, Legal);
+ setOperationAction(ISD::STRICT_FSUB, VT, Legal);
+ setOperationAction(ISD::FMUL, VT, Legal);
+ setOperationAction(ISD::STRICT_FMUL, VT, Legal);
+ setOperationAction(ISD::FDIV, VT, Legal);
+ setOperationAction(ISD::STRICT_FDIV, VT, Legal);
+ setOperationAction(ISD::FSQRT, VT, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, VT, Legal);
+
+ setOperationAction(ISD::FFLOOR, VT, Legal);
+ setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
+ setOperationAction(ISD::FCEIL, VT, Legal);
+ setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
+ setOperationAction(ISD::FTRUNC, VT, Legal);
+ setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
+ setOperationAction(ISD::FRINT, VT, Legal);
+ setOperationAction(ISD::STRICT_FRINT, VT, Legal);
+ setOperationAction(ISD::FNEARBYINT, VT, Legal);
+ setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
setOperationAction(ISD::FROUNDEVEN, VT, Legal);
setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
- setOperationAction(ISD::FROUND, VT, Custom);
+ setOperationAction(ISD::FROUND, VT, Custom);
- setOperationAction(ISD::LOAD, VT, Legal);
- setOperationAction(ISD::STORE, VT, Legal);
+ setOperationAction(ISD::LOAD, VT, Legal);
+ setOperationAction(ISD::STORE, VT, Legal);
- setOperationAction(ISD::FMA, VT, Legal);
- setOperationAction(ISD::STRICT_FMA, VT, Legal);
- setOperationAction(ISD::VSELECT, VT, Legal);
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
- setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::FMA, VT, Legal);
+ setOperationAction(ISD::STRICT_FMA, VT, Legal);
+ setOperationAction(ISD::VSELECT, VT, Legal);
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ setOperationAction(ISD::SELECT, VT, Custom);
- setOperationAction(ISD::FNEG, VT, Custom);
- setOperationAction(ISD::FABS, VT, Custom);
- setOperationAction(ISD::FCOPYSIGN, VT, Custom);
+ setOperationAction(ISD::FNEG, VT, Custom);
+ setOperationAction(ISD::FABS, VT, Custom);
+ setOperationAction(ISD::FCOPYSIGN, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
- setOperationAction(ISD::SETCC, VT, Custom);
- setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
- setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
+ setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
+ setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
};
// AVX512_FP16 scalar operations
setGroup(MVT::f16);
SetFPMinMaxAction(MVT::f16);
- setOperationAction(ISD::FREM, MVT::f16, Promote);
- setOperationAction(ISD::STRICT_FREM, MVT::f16, Promote);
- setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
- setOperationAction(ISD::BR_CC, MVT::f16, Expand);
- setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);
- setOperationAction(ISD::FROUNDEVEN, MVT::f16, Legal);
- setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Legal);
- setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
- setOperationAction(ISD::FP_EXTEND, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
- setOperationAction(ISD::LRINT, MVT::f16, Legal);
- setOperationAction(ISD::LLRINT, MVT::f16, Legal);
+ setOperationAction(ISD::FREM, MVT::f16, Promote);
+ setOperationAction(ISD::STRICT_FREM, MVT::f16, Promote);
+ setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
+ setOperationAction(ISD::BR_CC, MVT::f16, Expand);
+ setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);
+ setOperationAction(ISD::FROUNDEVEN, MVT::f16, Legal);
+ setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Legal);
+ setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
+ setOperationAction(ISD::FP_EXTEND, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
+ setOperationAction(ISD::LRINT, MVT::f16, Legal);
+ setOperationAction(ISD::LLRINT, MVT::f16, Legal);
setCondCodeAction(ISD::SETOEQ, MVT::f16, Expand);
setCondCodeAction(ISD::SETUNE, MVT::f16, Expand);
if (Subtarget.useAVX512Regs()) {
setGroup(MVT::v32f16);
- setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v32f16, Custom);
- setOperationAction(ISD::SINT_TO_FP, MVT::v32i16, Legal);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v32i16, Legal);
- setOperationAction(ISD::UINT_TO_FP, MVT::v32i16, Legal);
- setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v32i16, Legal);
- setOperationAction(ISD::FP_ROUND, MVT::v16f16, Legal);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Legal);
- setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Legal);
- setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Legal);
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32f16, Custom);
-
- setOperationAction(ISD::FP_TO_SINT, MVT::v32i16, Custom);
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v32i16, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v32i16, Custom);
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v32i16, Custom);
- setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i8, MVT::v32i16);
+ setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v32f16, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v32i16, Legal);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v32i16, Legal);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v32i16, Legal);
+ setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v32i16, Legal);
+ setOperationAction(ISD::FP_ROUND, MVT::v16f16, Legal);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Legal);
+ setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Legal);
+ setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Legal);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32f16, Custom);
+
+ setOperationAction(ISD::FP_TO_SINT, MVT::v32i16, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v32i16, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v32i16, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v32i16, Custom);
+ setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i8, MVT::v32i16);
setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i8,
MVT::v32i16);
- setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i8, MVT::v32i16);
+ setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i8, MVT::v32i16);
setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i8,
MVT::v32i16);
- setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i1, MVT::v32i16);
+ setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i1, MVT::v32i16);
setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i1,
MVT::v32i16);
- setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i1, MVT::v32i16);
+ setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i1, MVT::v32i16);
setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i1,
MVT::v32i16);
- setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v16f16, Legal);
- setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32f16, Legal);
- setOperationAction(ISD::CONCAT_VECTORS, MVT::v32f16, Custom);
+ setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v16f16, Legal);
+ setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32f16, Legal);
+ setOperationAction(ISD::CONCAT_VECTORS, MVT::v32f16, Custom);
- setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Legal);
+ setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Legal);
setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Legal);
SetFPMinMaxAction(MVT::v32f16);
@@ -2504,40 +2506,40 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::LLRINT, MVT::v8f16, Legal);
}
- setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom);
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i16, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Custom);
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i16, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i16, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i16, Custom);
if (Subtarget.hasVLX()) {
setGroup(MVT::v8f16);
setGroup(MVT::v16f16);
- setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8f16, Legal);
- setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16f16, Custom);
- setOperationAction(ISD::SINT_TO_FP, MVT::v16i16, Legal);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i16, Legal);
- setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Legal);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i16, Legal);
- setOperationAction(ISD::UINT_TO_FP, MVT::v16i16, Legal);
- setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i16, Legal);
- setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Legal);
- setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i16, Legal);
-
- setOperationAction(ISD::FP_ROUND, MVT::v8f16, Legal);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f16, Legal);
- setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f32, Legal);
- setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Legal);
+ setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8f16, Legal);
+ setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16f16, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v16i16, Legal);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i16, Legal);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Legal);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i16, Legal);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v16i16, Legal);
+ setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i16, Legal);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Legal);
+ setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i16, Legal);
+
+ setOperationAction(ISD::FP_ROUND, MVT::v8f16, Legal);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f16, Legal);
+ setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f32, Legal);
+ setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Legal);
// INSERT_VECTOR_ELT v8f16 extended to VECTOR_SHUFFLE
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8f16, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16f16, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8f16, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16f16, Custom);
- setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f16, Legal);
- setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v16f16, Legal);
- setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f16, Custom);
+ setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f16, Legal);
+ setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v16f16, Legal);
+ setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f16, Custom);
setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Legal);
setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Legal);
@@ -2545,7 +2547,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Legal);
// Need to custom widen these to prevent scalarization.
- setOperationAction(ISD::LOAD, MVT::v4f16, Custom);
+ setOperationAction(ISD::LOAD, MVT::v4f16, Custom);
setOperationAction(ISD::STORE, MVT::v4f16, Custom);
SetFPMinMaxAction(MVT::v8f16);
@@ -2635,52 +2637,52 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) {
- setTruncStoreAction(MVT::v4i64, MVT::v4i8, Legal);
+ setTruncStoreAction(MVT::v4i64, MVT::v4i8, Legal);
setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal);
setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal);
- setTruncStoreAction(MVT::v8i32, MVT::v8i8, Legal);
+ setTruncStoreAction(MVT::v8i32, MVT::v8i8, Legal);
setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal);
- setTruncStoreAction(MVT::v2i64, MVT::v2i8, Legal);
+ setTruncStoreAction(MVT::v2i64, MVT::v2i8, Legal);
setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal);
setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal);
- setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal);
+ setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal);
setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
if (Subtarget.hasBWI()) {
- setTruncStoreAction(MVT::v16i16, MVT::v16i8, Legal);
- setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
+ setTruncStoreAction(MVT::v16i16, MVT::v16i8, Legal);
+ setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
}
if (Subtarget.hasFP16()) {
// vcvttph2[u]dq v4f16 -> v4i32/64, v2f16 -> v2i32/64
- setOperationAction(ISD::FP_TO_SINT, MVT::v2f16, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v2f16, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f16, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v2f16, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v2f16, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f16, Custom);
- setOperationAction(ISD::FP_TO_SINT, MVT::v4f16, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v4f16, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4f16, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v4f16, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v4f16, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4f16, Custom);
// vcvt[u]dq2ph v4i32/64 -> v4f16, v2i32/64 -> v2f16
- setOperationAction(ISD::SINT_TO_FP, MVT::v2f16, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v2f16, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f16, Custom);
- setOperationAction(ISD::UINT_TO_FP, MVT::v2f16, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v2f16, Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f16, Custom);
- setOperationAction(ISD::SINT_TO_FP, MVT::v4f16, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v4f16, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4f16, Custom);
- setOperationAction(ISD::UINT_TO_FP, MVT::v4f16, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v4f16, Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4f16, Custom);
// vcvtps2phx v4f32 -> v4f16, v2f32 -> v2f16
- setOperationAction(ISD::FP_ROUND, MVT::v2f16, Custom);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f16, Custom);
- setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f16, Custom);
+ setOperationAction(ISD::FP_ROUND, MVT::v2f16, Custom);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f16, Custom);
+ setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f16, Custom);
// vcvtph2psx v4f16 -> v4f32, v2f16 -> v2f32
- setOperationAction(ISD::FP_EXTEND, MVT::v2f16, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f16, Custom);
- setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f16, Custom);
+ setOperationAction(ISD::FP_EXTEND, MVT::v2f16, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f16, Custom);
+ setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f16, Custom);
}
}
@@ -2702,7 +2704,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// FIXME: We really should do custom legalization for addition and
// subtraction on x86-32 once PR3203 is fixed. We really can't do much better
// than generic legalization for 64-bit multiplication-with-overflow, though.
- for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
+ for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
if (VT == MVT::i64 && !Subtarget.is64Bit())
continue;
// Add/Sub/Mul with overflow operations are custom lowered.
@@ -3051,8 +3053,9 @@ static bool isLogicOp(unsigned Opcode) {
}
static bool isTargetShuffle(unsigned Opcode) {
- switch(Opcode) {
- default: return false;
+ switch (Opcode) {
+ default:
+ return false;
case X86ISD::BLENDI:
case X86ISD::PSHUFB:
case X86ISD::PSHUFD:
@@ -3095,7 +3098,8 @@ static bool isTargetShuffle(unsigned Opcode) {
static bool isTargetShuffleVariableMask(unsigned Opcode) {
switch (Opcode) {
- default: return false;
+ default:
+ return false;
// Target Shuffles.
case X86ISD::PSHUFB:
case X86ISD::VPERMILPV:
@@ -3121,9 +3125,8 @@ SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
if (ReturnAddrIndex == 0) {
// Set up a frame object for the return address.
unsigned SlotSize = RegInfo->getSlotSize();
- ReturnAddrIndex = MF.getFrameInfo().CreateFixedObject(SlotSize,
- -(int64_t)SlotSize,
- false);
+ ReturnAddrIndex = MF.getFrameInfo().CreateFixedObject(
+ SlotSize, -(int64_t)SlotSize, false);
FuncInfo->setRAIndex(ReturnAddrIndex);
}
@@ -3181,7 +3184,7 @@ static bool isX86CCSigned(X86::CondCode X86CC) {
static X86::CondCode TranslateIntegerX86CC(ISD::CondCode SetCCOpcode) {
switch (SetCCOpcode) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Invalid integer condition!");
case ISD::SETEQ: return X86::COND_E;
case ISD::SETGT: return X86::COND_G;
@@ -3193,7 +3196,7 @@ static X86::CondCode TranslateIntegerX86CC(ISD::CondCode SetCCOpcode) {
case ISD::SETUGT: return X86::COND_A;
case ISD::SETULE: return X86::COND_BE;
case ISD::SETUGE: return X86::COND_AE;
- // clang-format on
+ // clang-format on
}
}
@@ -3231,14 +3234,14 @@ static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
// First determine if it is required or is profitable to flip the operands.
// If LHS is a foldable load, but RHS is not, flip the condition.
- if (ISD::isNON_EXTLoad(LHS.getNode()) &&
- !ISD::isNON_EXTLoad(RHS.getNode())) {
+ if (ISD::isNON_EXTLoad(LHS.getNode()) && !ISD::isNON_EXTLoad(RHS.getNode())) {
SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
std::swap(LHS, RHS);
}
switch (SetCCOpcode) {
- default: break;
+ default:
+ break;
case ISD::SETOLT:
case ISD::SETOLE:
case ISD::SETUGT:
@@ -3254,7 +3257,7 @@ static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
// 1 | 0 | 0 | X == Y
// 1 | 1 | 1 | unordered
switch (SetCCOpcode) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Condcode should be pre-legalized away");
case ISD::SETUEQ:
case ISD::SETEQ: return X86::COND_E;
@@ -3276,7 +3279,7 @@ static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
case ISD::SETO: return X86::COND_NP;
case ISD::SETOEQ:
case ISD::SETUNE: return X86::COND_INVALID;
- // clang-format on
+ // clang-format on
}
}
@@ -3311,7 +3314,7 @@ void X86TargetLowering::getTgtMemIntrinsic(
Info.flags = MachineMemOperand::MONone;
Info.offset = 0;
- const IntrinsicData* IntrData = getIntrinsicWithChain(Intrinsic);
+ const IntrinsicData *IntrData = getIntrinsicWithChain(Intrinsic);
if (!IntrData) {
switch (Intrinsic) {
case Intrinsic::x86_aesenc128kl:
@@ -3411,7 +3414,7 @@ void X86TargetLowering::getTgtMemIntrinsic(
case TRUNCATE_TO_MEM_VI32: {
Info.opc = ISD::INTRINSIC_VOID;
Info.ptrVal = I.getArgOperand(0);
- MVT VT = MVT::getVT(I.getArgOperand(1)->getType());
+ MVT VT = MVT::getVT(I.getArgOperand(1)->getType());
MVT ScalarVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
if (IntrData->Type == TRUNCATE_TO_MEM_VI8)
ScalarVT = MVT::i8;
@@ -3432,8 +3435,8 @@ void X86TargetLowering::getTgtMemIntrinsic(
Info.ptrVal = nullptr;
MVT DataVT = MVT::getVT(I.getType());
MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
- unsigned NumElts = std::min(DataVT.getVectorNumElements(),
- IndexVT.getVectorNumElements());
+ unsigned NumElts =
+ std::min(DataVT.getVectorNumElements(), IndexVT.getVectorNumElements());
Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
Info.align = Align(1);
Info.flags |= MachineMemOperand::MOLoad;
@@ -3445,8 +3448,8 @@ void X86TargetLowering::getTgtMemIntrinsic(
Info.ptrVal = nullptr;
MVT DataVT = MVT::getVT(I.getArgOperand(3)->getType());
MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
- unsigned NumElts = std::min(DataVT.getVectorNumElements(),
- IndexVT.getVectorNumElements());
+ unsigned NumElts =
+ std::min(DataVT.getVectorNumElements(), IndexVT.getVectorNumElements());
Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
Info.align = Align(1);
Info.flags |= MachineMemOperand::MOStore;
@@ -3696,9 +3699,9 @@ bool X86TargetLowering::isScalarFPTypeInSSEReg(EVT VT) const {
(VT == MVT::f32 && Subtarget.hasSSE1()) || VT == MVT::f16;
}
-bool X86TargetLowering::isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
- const SelectionDAG &DAG,
- const MachineMemOperand &MMO) const {
+bool X86TargetLowering::isLoadBitCastBeneficial(
+ EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG,
+ const MachineMemOperand &MMO) const {
if (!Subtarget.hasAVX512() && !LoadVT.isVector() && BitcastVT.isVector() &&
BitcastVT.getVectorElementType() == MVT::i1)
return false;
@@ -3740,9 +3743,7 @@ bool X86TargetLowering::canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
return true;
}
-bool X86TargetLowering::isCtlzFast() const {
- return Subtarget.hasFastLZCNT();
-}
+bool X86TargetLowering::isCtlzFast() const { return Subtarget.hasFastLZCNT(); }
bool X86TargetLowering::isMaskAndCmp0FoldingBeneficial(
const Instruction &AndI) const {
@@ -4167,8 +4168,7 @@ static bool canWidenShuffleElements(ArrayRef<int> Mask,
return true;
}
-static bool canWidenShuffleElements(ArrayRef<int> Mask,
- const APInt &Zeroable,
+static bool canWidenShuffleElements(ArrayRef<int> Mask, const APInt &Zeroable,
bool V2IsZero,
SmallVectorImpl<int> &WidenedMask) {
// Create an alternative mask with info about zeroable elements.
@@ -4252,7 +4252,7 @@ bool X86::isZeroNode(SDValue Elt) {
static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
const SDLoc &dl, bool IsMask = false) {
- SmallVector<SDValue, 32> Ops;
+ SmallVector<SDValue, 32> Ops;
bool Split = false;
MVT ConstVecVT = VT;
@@ -4266,12 +4266,12 @@ static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
MVT EltVT = ConstVecVT.getVectorElementType();
for (unsigned i = 0; i < NumElts; ++i) {
bool IsUndef = Values[i] < 0 && IsMask;
- SDValue OpNode = IsUndef ? DAG.getUNDEF(EltVT) :
- DAG.getConstant(Values[i], dl, EltVT);
+ SDValue OpNode =
+ IsUndef ? DAG.getUNDEF(EltVT) : DAG.getConstant(Values[i], dl, EltVT);
Ops.push_back(OpNode);
if (Split)
- Ops.push_back(IsUndef ? DAG.getUNDEF(EltVT) :
- DAG.getConstant(0, dl, EltVT));
+ Ops.push_back(IsUndef ? DAG.getUNDEF(EltVT)
+ : DAG.getConstant(0, dl, EltVT));
}
SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
if (Split)
@@ -4279,8 +4279,8 @@ static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
return ConstsNode;
}
-static SDValue getConstVector(ArrayRef<APInt> Bits, const APInt &Undefs,
- MVT VT, SelectionDAG &DAG, const SDLoc &dl) {
+static SDValue getConstVector(ArrayRef<APInt> Bits, const APInt &Undefs, MVT VT,
+ SelectionDAG &DAG, const SDLoc &dl) {
assert(Bits.size() == Undefs.getBitWidth() &&
"Unequal constant and undef arrays");
SmallVector<SDValue, 32> Ops;
@@ -4315,8 +4315,8 @@ static SDValue getConstVector(ArrayRef<APInt> Bits, const APInt &Undefs,
return DAG.getBitcast(VT, ConstsNode);
}
-static SDValue getConstVector(ArrayRef<APInt> Bits, MVT VT,
- SelectionDAG &DAG, const SDLoc &dl) {
+static SDValue getConstVector(ArrayRef<APInt> Bits, MVT VT, SelectionDAG &DAG,
+ const SDLoc &dl) {
APInt Undefs = APInt::getZero(Bits.size());
return getConstVector(Bits, Undefs, VT, DAG, dl);
}
@@ -4853,8 +4853,7 @@ static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
if (IdxVal == 0 && ISD::isBuildVectorAllZeros(Vec.getNode())) {
// May need to promote to a legal type.
Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
- DAG.getConstant(0, dl, WideOpVT),
- SubVec, Idx);
+ DAG.getConstant(0, dl, WideOpVT), SubVec, Idx);
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
}
@@ -4869,20 +4868,18 @@ static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
if (IdxVal == 0) {
// Zero lower bits of the Vec
SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8);
- Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec,
- ZeroIdx);
+ Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, ZeroIdx);
Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
// Merge them together, SubVec should be zero extended.
SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
- DAG.getConstant(0, dl, WideOpVT),
- SubVec, ZeroIdx);
+ DAG.getConstant(0, dl, WideOpVT), SubVec, ZeroIdx);
Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
}
- SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
- Undef, SubVec, ZeroIdx);
+ SubVec =
+ DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, SubVec, ZeroIdx);
if (Vec.isUndef()) {
assert(IdxVal != 0 && "Unexpected index");
@@ -4920,12 +4917,11 @@ static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
// isel to optimize when bits are known zero.
Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVecVT, Vec, ZeroIdx);
Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
- DAG.getConstant(0, dl, WideOpVT),
- Vec, ZeroIdx);
+ DAG.getConstant(0, dl, WideOpVT), Vec, ZeroIdx);
} else {
// Otherwise use explicit shifts to zero the bits.
- Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
- Undef, Vec, ZeroIdx);
+ Vec =
+ DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, ZeroIdx);
NumElems = WideOpVT.getVectorNumElements();
SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8);
Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
@@ -4978,9 +4974,9 @@ static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
// Isolate the bits after the last inserted bit.
unsigned HighShift = IdxVal + SubVecNumElems;
SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
- DAG.getTargetConstant(HighShift, dl, MVT::i8));
+ DAG.getTargetConstant(HighShift, dl, MVT::i8));
High = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, High,
- DAG.getTargetConstant(HighShift, dl, MVT::i8));
+ DAG.getTargetConstant(HighShift, dl, MVT::i8));
// Now OR all 3 pieces together.
Vec = DAG.getNode(ISD::OR, dl, WideOpVT, Low, High);
@@ -5229,8 +5225,8 @@ static SDValue getBitSelect(const SDLoc &DL, MVT VT, SDValue LHS, SDValue RHS,
return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
}
-void llvm::createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask,
- bool Lo, bool Unary) {
+void llvm::createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask, bool Lo,
+ bool Unary) {
assert(VT.getScalarType().isSimple() && (VT.getSizeInBits() % 128) == 0 &&
"Illegal vector type to unpack");
assert(Mask.empty() && "Expected an empty shuffle mask vector");
@@ -5367,13 +5363,12 @@ static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget,
/// This produces a shuffle where the low element of V2 is swizzled into the
/// zero/undef vector, landing at element Idx.
/// This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
-static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx,
- bool IsZero,
+static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx, bool IsZero,
const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
MVT VT = V2.getSimpleValueType();
- SDValue V1 = IsZero
- ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
+ SDValue V1 =
+ IsZero ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
int NumElems = VT.getVectorNumElements();
SmallVector<int, 16> MaskVec(NumElems);
for (int i = 0; i != NumElems; ++i)
@@ -6338,9 +6333,9 @@ static bool getTargetShuffleMask(SDValue N, bool AllowSentinelZero,
/// zero. Many x86 shuffles can zero lanes cheaply and we often want to handle
/// as many lanes with this technique as possible to simplify the remaining
/// shuffle.
-static void computeZeroableShuffleElements(ArrayRef<int> Mask,
- SDValue V1, SDValue V2,
- APInt &KnownUndef, APInt &KnownZero) {
+static void computeZeroableShuffleElements(ArrayRef<int> Mask, SDValue V1,
+ SDValue V2, APInt &KnownUndef,
+ APInt &KnownZero) {
int Size = Mask.size();
KnownUndef = KnownZero = APInt::getZero(Size);
@@ -6526,7 +6521,7 @@ static bool getTargetShuffleAndZeroables(SDValue N, SmallVectorImpl<int> &Mask,
static void resolveTargetShuffleFromZeroables(SmallVectorImpl<int> &Mask,
const APInt &KnownUndef,
const APInt &KnownZero,
- bool ResolveKnownZeros= true) {
+ bool ResolveKnownZeros = true) {
unsigned NumElts = Mask.size();
assert(KnownUndef.getBitWidth() == NumElts &&
KnownZero.getBitWidth() == NumElts && "Shuffle mask size mismatch");
@@ -7558,8 +7553,8 @@ static SDValue LowerBuildVectorv4x32(SDValue Op, const SDLoc &DL,
MVT EltVT = VT.getVectorElementType();
// Create a new build vector with the first 2 elements followed by undef
// padding, bitcast to v2f64, duplicate, and bitcast back.
- SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
- DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
+ SDValue Ops[4] = {Op.getOperand(0), Op.getOperand(1), DAG.getUNDEF(EltVT),
+ DAG.getUNDEF(EltVT)};
SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));
SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV);
return DAG.getBitcast(VT, Dup);
@@ -7607,7 +7602,7 @@ static SDValue LowerBuildVectorv4x32(SDValue Op, const SDLoc &DL,
for (EltIdx = 0; EltIdx < 4; ++EltIdx) {
if (Zeroable[EltIdx]) {
// The zero vector will be on the right hand side.
- Mask[EltIdx] = EltIdx+4;
+ Mask[EltIdx] = EltIdx + 4;
continue;
}
@@ -7837,7 +7832,7 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
APInt ZeroMask = APInt::getZero(NumElems);
APInt UndefMask = APInt::getZero(NumElems);
- SmallVector<LoadSDNode*, 8> Loads(NumElems, nullptr);
+ SmallVector<LoadSDNode *, 8> Loads(NumElems, nullptr);
SmallVector<int64_t, 8> ByteOffsets(NumElems, 0);
// For each element in the initializer, see if we've found a load, zero or an
@@ -7944,8 +7939,7 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, LoadSDNode *LDBase) {
auto MMOFlags = LDBase->getMemOperand()->getFlags();
- assert(LDBase->isSimple() &&
- "Cannot merge volatile or atomic loads.");
+ assert(LDBase->isSimple() && "Cannot merge volatile or atomic loads.");
SDValue NewLd =
DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
LDBase->getPointerInfo(), LDBase->getBaseAlign(), MMOFlags);
@@ -8033,7 +8027,7 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
VecVT = MVT::v4f32;
if (TLI.isTypeLegal(VecVT)) {
SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
- SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
+ SDValue Ops[] = {LDBase->getChain(), LDBase->getBasePtr()};
SDValue ResNode = DAG.getMemIntrinsicNode(
X86ISD::VZEXT_LOAD, DL, Tys, Ops, VecSVT, LDBase->getPointerInfo(),
LDBase->getBaseAlign(), MachineMemOperand::MOLoad);
@@ -8841,7 +8835,8 @@ static SDValue LowerBUILD_VECTORvXi1(SDValue Op, const SDLoc &dl,
/// Returns true if is possible to fold MUL and an idiom that has already been
/// recognized as ADDSUB/SUBADD(\p Opnd0, \p Opnd1) into
/// FMADDSUB/FMSUBADD(x, y, \p Opnd1). If (and only if) true is returned, the
-/// operands of FMADDSUB/FMSUBADD are written to parameters \p Opnd0, \p Opnd1, \p Opnd2.
+/// operands of FMADDSUB/FMSUBADD are written to parameters \p Opnd0, \p Opnd1,
+/// \p Opnd2.
///
/// Prior to calling this function it should be known that there is some
/// SDNode that potentially can be replaced with an X86ISD::ADDSUB operation
@@ -9430,8 +9425,8 @@ LowerBUILD_VECTORAsVariablePermute(SDValue V, const SDLoc &DL,
return createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);
}
-SDValue
-X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerBUILD_VECTOR(SDValue Op,
+ SelectionDAG &DAG) const {
SDLoc dl(Op);
MVT VT = Op.getSimpleValueType();
@@ -9653,14 +9648,13 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
}
// Is it a vector logical left shift?
- if (NumElems == 2 && Idx == 1 &&
- X86::isZeroNode(Op.getOperand(0)) &&
+ if (NumElems == 2 && Idx == 1 && X86::isZeroNode(Op.getOperand(0)) &&
!X86::isZeroNode(Op.getOperand(1))) {
unsigned NumBits = VT.getSizeInBits();
- return getVShift(true, VT,
- DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
- VT, Op.getOperand(1)),
- NumBits/2, DAG, *this, dl);
+ return getVShift(
+ true, VT,
+ DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(1)),
+ NumBits / 2, DAG, *this, dl);
}
if (IsAllConstants) // Otherwise, it's better to do a constpool load.
@@ -9673,7 +9667,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
// place.
if (EVTBits == 32) {
Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
- return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget, DAG);
+ return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget,
+ DAG);
}
}
@@ -9712,8 +9707,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
// build_vector and broadcast it.
// TODO: We could probably generalize this more.
if (Subtarget.hasAVX2() && EVTBits == 32 && Values.size() == 2) {
- SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
- DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
+ SDValue Ops[4] = {Op.getOperand(0), Op.getOperand(1), DAG.getUNDEF(EltVT),
+ DAG.getUNDEF(EltVT)};
auto CanSplat = [](SDValue Op, unsigned NumElems, ArrayRef<SDValue> Ops) {
// Make sure all the even/odd operands match.
for (unsigned i = 2; i != NumElems; ++i)
@@ -9729,8 +9724,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
DAG.getBuildVector(NarrowVT, dl, Ops));
// Broadcast from v2i64/v2f64 and cast to final VT.
MVT BcastVT = MVT::getVectorVT(WideEltVT, NumElems / 2);
- return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT,
- NewBV));
+ return DAG.getBitcast(
+ VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT, NewBV));
}
}
@@ -9743,7 +9738,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
SDValue Lower =
DAG.getBuildVector(HVT, dl, Op->ops().slice(0, NumElems / 2));
SDValue Upper = DAG.getBuildVector(
- HVT, dl, Op->ops().slice(NumElems / 2, NumElems /2));
+ HVT, dl, Op->ops().slice(NumElems / 2, NumElems / 2));
// Recreate the wider vector with the lower and upper part.
return concatSubVectors(Lower, Upper, DAG, dl);
@@ -9754,8 +9749,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
if (NumNonZero == 1) {
// One half is zero or undef.
unsigned Idx = NonZeroMask.countr_zero();
- SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
- Op.getOperand(Idx));
+ SDValue V2 =
+ DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(Idx));
return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG);
}
return SDValue();
@@ -9790,30 +9785,28 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
for (unsigned i = 0; i < 2; ++i) {
switch (NonZeroMask.extractBitsAsZExtValue(2, i * 2)) {
- default: llvm_unreachable("Unexpected NonZero count");
- case 0:
- Ops[i] = Ops[i*2]; // Must be a zero vector.
- break;
- case 1:
- Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2+1], Ops[i*2]);
- break;
- case 2:
- Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
- break;
- case 3:
- Ops[i] = getUnpackl(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
- break;
+ default:
+ llvm_unreachable("Unexpected NonZero count");
+ case 0:
+ Ops[i] = Ops[i * 2]; // Must be a zero vector.
+ break;
+ case 1:
+ Ops[i] = getMOVL(DAG, dl, VT, Ops[i * 2 + 1], Ops[i * 2]);
+ break;
+ case 2:
+ Ops[i] = getMOVL(DAG, dl, VT, Ops[i * 2], Ops[i * 2 + 1]);
+ break;
+ case 3:
+ Ops[i] = getUnpackl(DAG, dl, VT, Ops[i * 2], Ops[i * 2 + 1]);
+ break;
}
}
bool Reverse1 = NonZeroMask.extractBitsAsZExtValue(2, 0) == 2;
bool Reverse2 = NonZeroMask.extractBitsAsZExtValue(2, 2) == 2;
- int MaskVec[] = {
- Reverse1 ? 1 : 0,
- Reverse1 ? 0 : 1,
- static_cast<int>(Reverse2 ? NumElems+1 : NumElems),
- static_cast<int>(Reverse2 ? NumElems : NumElems+1)
- };
+ int MaskVec[] = {Reverse1 ? 1 : 0, Reverse1 ? 0 : 1,
+ static_cast<int>(Reverse2 ? NumElems + 1 : NumElems),
+ static_cast<int>(Reverse2 ? NumElems : NumElems + 1)};
return DAG.getVectorShuffle(VT, dl, Ops[0], Ops[1], MaskVec);
}
@@ -9832,7 +9825,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
Result = DAG.getUNDEF(VT);
for (unsigned i = 1; i < NumElems; ++i) {
- if (Op.getOperand(i).isUndef()) continue;
+ if (Op.getOperand(i).isUndef())
+ continue;
Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result,
Op.getOperand(i), DAG.getVectorIdxConstant(i, dl));
}
@@ -9857,14 +9851,14 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
for (unsigned Scale = 1; Scale < NumElems; Scale *= 2) {
// Generate scaled UNPCKL shuffle mask.
SmallVector<int, 16> Mask;
- for(unsigned i = 0; i != Scale; ++i)
+ for (unsigned i = 0; i != Scale; ++i)
Mask.push_back(i);
for (unsigned i = 0; i != Scale; ++i)
- Mask.push_back(NumElems+i);
+ Mask.push_back(NumElems + i);
Mask.append(NumElems - Mask.size(), SM_SentinelUndef);
for (unsigned i = 0, e = NumElems / (2 * Scale); i != e; ++i)
- Ops[i] = DAG.getVectorShuffle(VT, dl, Ops[2*i], Ops[(2*i)+1], Mask);
+ Ops[i] = DAG.getVectorShuffle(VT, dl, Ops[2 * i], Ops[(2 * i) + 1], Mask);
}
return Ops[0];
}
@@ -9890,15 +9884,14 @@ static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, const SDLoc &dl,
if (SubVec.isUndef())
continue;
if (ISD::isFreezeUndef(SubVec.getNode())) {
- // If the freeze(undef) has multiple uses then we must fold to zero.
- if (SubVec.hasOneUse()) {
- ++NumFreezeUndef;
- } else {
- ++NumZero;
- Undefs.insert(SubVec);
- }
- }
- else if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
+ // If the freeze(undef) has multiple uses then we must fold to zero.
+ if (SubVec.hasOneUse()) {
+ ++NumFreezeUndef;
+ } else {
+ ++NumZero;
+ Undefs.insert(SubVec);
+ }
+ } else if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
++NumZero;
else {
assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
@@ -9912,9 +9905,9 @@ static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, const SDLoc &dl,
MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
ArrayRef<SDUse> Ops = Op->ops();
SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
- Ops.slice(0, NumOperands/2));
+ Ops.slice(0, NumOperands / 2));
SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
- Ops.slice(NumOperands/2));
+ Ops.slice(NumOperands / 2));
return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
}
@@ -9947,7 +9940,7 @@ static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, const SDLoc &dl,
// TODO: Merge this with LowerAVXCONCAT_VECTORS?
static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op, const SDLoc &dl,
const X86Subtarget &Subtarget,
- SelectionDAG & DAG) {
+ SelectionDAG &DAG) {
MVT ResVT = Op.getSimpleValueType();
unsigned NumOperands = Op.getNumOperands();
assert(NumOperands > 1 && isPowerOf2_32(NumOperands) &&
@@ -10018,8 +10011,7 @@ static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op, const SDLoc &dl,
DAG.getVectorIdxConstant(NumElems / 2, dl));
}
-static SDValue LowerCONCAT_VECTORS(SDValue Op,
- const X86Subtarget &Subtarget,
+static SDValue LowerCONCAT_VECTORS(SDValue Op, const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
SDLoc DL(Op);
MVT VT = Op.getSimpleValueType();
@@ -10241,8 +10233,8 @@ static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,
// Ok, handle the in-lane shuffles by detecting if and when they repeat.
// Adjust second vector indices to start at LaneSize instead of Size.
- int LocalM = Mask[i] < Size ? Mask[i] % LaneSize
- : Mask[i] % LaneSize + LaneSize;
+ int LocalM =
+ Mask[i] < Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + LaneSize;
if (RepeatedMask[i % LaneSize] < 0)
// This is the first non-undef entry in this slot of a 128-bit lane.
RepeatedMask[i % LaneSize] = LocalM;
@@ -10260,8 +10252,7 @@ is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,
return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);
}
-static bool
-is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask) {
+static bool is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask) {
SmallVector<int, 32> RepeatedMask;
return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);
}
@@ -10560,8 +10551,8 @@ static SDValue getSHUFPDImmForMask(ArrayRef<int> Mask, const SDLoc &DL,
//
// The function looks for a sub-mask that the nonzero elements are in
// increasing order. If such sub-mask exist. The function returns true.
-static bool isNonZeroElementsInOrder(const APInt &Zeroable,
- ArrayRef<int> Mask, const EVT &VectorType,
+static bool isNonZeroElementsInOrder(const APInt &Zeroable, ArrayRef<int> Mask,
+ const EVT &VectorType,
bool &IsZeroSideLeft) {
int NextElement = -1;
// Check if the Mask's nonzero elements are in increasing order.
@@ -11348,7 +11339,7 @@ static bool matchShuffleAsBlend(MVT VT, SDValue V1, SDValue V2,
if (M == SM_SentinelUndef)
continue;
if (M == Elt || (0 <= M && M < NumElts &&
- IsElementEquivalent(NumElts, V1, V1, M, Elt))) {
+ IsElementEquivalent(NumElts, V1, V1, M, Elt))) {
Mask[Elt] = Elt;
LaneV1InUse = true;
continue;
@@ -11481,8 +11472,7 @@ static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
// If we have VPTERNLOG, we can use that as a bit blend.
if (Subtarget.hasVLX())
- if (SDValue BitBlend =
- lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
+ if (SDValue BitBlend = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
return BitBlend;
// Scale the blend by the number of bytes per element.
@@ -11790,9 +11780,11 @@ static SDValue lowerShuffleAsPermuteAndUnpack(const SDLoc &DL, MVT VT,
/// Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then
/// permuting the elements of the result in place.
-static SDValue lowerShuffleAsByteRotateAndPermute(
- const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
- const X86Subtarget &Subtarget, SelectionDAG &DAG) {
+static SDValue lowerShuffleAsByteRotateAndPermute(const SDLoc &DL, MVT VT,
+ SDValue V1, SDValue V2,
+ ArrayRef<int> Mask,
+ const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
if ((VT.is128BitVector() && !Subtarget.hasSSSE3()) ||
(VT.is256BitVector() && !Subtarget.hasAVX2()) ||
(VT.is512BitVector() && !Subtarget.hasBWI()))
@@ -11993,9 +11985,9 @@ static SDValue lowerShuffleAsDecomposedShuffleMerge(
// If either input vector provides only a single element which is repeated
// multiple times, unpacking from both input vectors would generate worse
// code. e.g. for
- // t5: v16i8 = vector_shuffle<16,0,16,1,16,2,16,3,16,4,16,5,16,6,16,7> t2, t4
- // it is better to process t4 first to create a vector of t4[0], then unpack
- // that vector with t2.
+ // t5: v16i8 = vector_shuffle<16,0,16,1,16,2,16,3,16,4,16,5,16,6,16,7> t2,
+ // t4 it is better to process t4 first to create a vector of t4[0], then
+ // unpack that vector with t2.
if (!V1Zero && !V2Zero && !isSingleElementRepeatedMask(V1Mask) &&
!isSingleElementRepeatedMask(V2Mask))
if (SDValue UnpackPerm =
@@ -12007,8 +11999,8 @@ static SDValue lowerShuffleAsDecomposedShuffleMerge(
return RotatePerm;
// Unpack/rotate failed - try again with variable blends.
- if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
- DAG))
+ if (SDValue BlendPerm =
+ lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask, DAG))
return BlendPerm;
if (EltSizeInBits >= 32)
@@ -12123,7 +12115,7 @@ static int matchShuffleAsElementRotate(SDValue &V1, SDValue &V2,
SDValue Lo, Hi;
for (int i = 0; i < NumElts; ++i) {
int M = Mask[i];
- assert((M == SM_SentinelUndef || (0 <= M && M < (2*NumElts))) &&
+ assert((M == SM_SentinelUndef || (0 <= M && M < (2 * NumElts))) &&
"Unexpected mask index.");
if (M < 0)
continue;
@@ -12245,8 +12237,7 @@ static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
"Rotate-based lowering only supports 128-bit lowering!");
assert(Mask.size() <= 16 &&
"Can shuffle at most 16 bytes in a 128-bit vector!");
- assert(ByteVT == MVT::v16i8 &&
- "SSE2 rotate lowering only needed for v16i8!");
+ assert(ByteVT == MVT::v16i8 && "SSE2 rotate lowering only needed for v16i8!");
// Default SSE2 implementation
int LoByteShift = 16 - ByteRotation;
@@ -12879,8 +12870,7 @@ static SDValue lowerShuffleAsSpecificExtension(const SDLoc &DL, MVT VT,
/// are both incredibly common and often quite performance sensitive.
static SDValue lowerShuffleAsZeroOrAnyExtend(
const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
- const APInt &Zeroable, const X86Subtarget &Subtarget,
- SelectionDAG &DAG) {
+ const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
int Bits = VT.getSizeInBits();
int NumLanes = Bits / 128;
int NumElements = VT.getVectorNumElements();
@@ -13013,7 +13003,8 @@ static SDValue getScalarValueForVectorElement(SDValue V, int Idx,
// If the bitcasts shift the element size, we can't extract an equivalent
// element from it.
MVT NewVT = V.getSimpleValueType();
- if (!NewVT.isVector() || NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
+ if (!NewVT.isVector() ||
+ NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
return SDValue();
if (V.getOpcode() == ISD::BUILD_VECTOR ||
@@ -13037,14 +13028,14 @@ static bool isShuffleFoldableLoad(SDValue V) {
ISD::isNON_EXTLoad(peekThroughOneUseBitcasts(V).getNode());
}
-template<typename T>
+template <typename T>
static bool isSoftF16(T VT, const X86Subtarget &Subtarget) {
T EltVT = VT.getScalarType();
return (EltVT == MVT::bf16 && !Subtarget.hasAVX10_2()) ||
(EltVT == MVT::f16 && !Subtarget.hasFP16());
}
-template<typename T>
+template <typename T>
static bool isBF16orSoftF16(T VT, const X86Subtarget &Subtarget) {
T EltVT = VT.getScalarType();
return EltVT == MVT::bf16 || (EltVT == MVT::f16 && !Subtarget.hasFP16());
@@ -13056,8 +13047,7 @@ static bool isBF16orSoftF16(T VT, const X86Subtarget &Subtarget) {
/// across all subtarget feature sets.
static SDValue lowerShuffleAsElementInsertion(
const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
- const APInt &Zeroable, const X86Subtarget &Subtarget,
- SelectionDAG &DAG) {
+ const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
MVT ExtVT = VT;
MVT EltVT = VT.getVectorElementType();
unsigned NumElts = VT.getVectorNumElements();
@@ -13090,8 +13080,8 @@ static SDValue lowerShuffleAsElementInsertion(
// all the smarts here sunk into that routine. However, the current
// lowering of BUILD_VECTOR makes that nearly impossible until the old
// vector shuffle lowering is dead.
- SDValue V2S = getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(),
- DAG);
+ SDValue V2S =
+ getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(), DAG);
if (V2S && DAG.getTargetLoweringInfo().isTypeLegal(V2S.getValueType())) {
// We need to zext the scalar if it is smaller than an i32.
V2S = DAG.getBitcast(EltVT, V2S);
@@ -13294,8 +13284,8 @@ static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
// Check that both sources are extracts of the same source vector.
if (N0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
N1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
- N0.getOperand(0) != N1.getOperand(0) ||
- !N0.hasOneUse() || !N1.hasOneUse())
+ N0.getOperand(0) != N1.getOperand(0) || !N0.hasOneUse() ||
+ !N1.hasOneUse())
return SDValue();
SDValue WideVec = N0.getOperand(0);
@@ -13325,8 +13315,8 @@ static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
NewMask.append(NumElts, -1);
// shuf (extract X, 0), (extract X, 4), M --> extract (shuf X, undef, M'), 0
- SDValue Shuf = DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT),
- NewMask);
+ SDValue Shuf =
+ DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT), NewMask);
// This is free: ymm -> xmm.
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuf,
DAG.getVectorIdxConstant(0, DL));
@@ -13525,8 +13515,8 @@ static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
if (!V.getValueType().isVector()) {
assert(V.getScalarValueSizeInBits() == NumEltBits &&
"Unexpected scalar size");
- MVT BroadcastVT = MVT::getVectorVT(V.getSimpleValueType(),
- VT.getVectorNumElements());
+ MVT BroadcastVT =
+ MVT::getVectorVT(V.getSimpleValueType(), VT.getVectorNumElements());
return DAG.getBitcast(VT, DAG.getNode(Opcode, DL, BroadcastVT, V));
}
@@ -13551,8 +13541,8 @@ static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
// elements are zeroable.
static bool matchShuffleAsInsertPS(SDValue &V1, SDValue &V2,
unsigned &InsertPSMask,
- const APInt &Zeroable,
- ArrayRef<int> Mask, SelectionDAG &DAG) {
+ const APInt &Zeroable, ArrayRef<int> Mask,
+ SelectionDAG &DAG) {
assert(V1.getSimpleValueType().is128BitVector() && "Bad operand type!");
assert(V2.getSimpleValueType().is128BitVector() && "Bad operand type!");
assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
@@ -14110,8 +14100,8 @@ static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// There are special ways we can lower some single-element blends.
if (NumV2Elements == 1)
- if (SDValue V = lowerShuffleAsElementInsertion(
- DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
+ if (SDValue V = lowerShuffleAsElementInsertion(DL, MVT::v4i32, V1, V2, Mask,
+ Zeroable, Subtarget, DAG))
return V;
// We have different paths for blend lowering, but they all must use the
@@ -14241,7 +14231,7 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
};
if ((NumHToL + NumHToH) == 0 || (NumLToL + NumLToH) == 0) {
- int PSHUFDMask[4] = { -1, -1, -1, -1 };
+ int PSHUFDMask[4] = {-1, -1, -1, -1};
SmallVector<std::pair<int, int>, 4> DWordPairs;
int DOffset = ((NumHToL + NumHToH) == 0 ? 0 : 2);
@@ -14345,7 +14335,8 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
int OneInput = ThreeAInputs ? BToAInputs[0] : AToAInputs[0];
int TripleInputSum = 0 + 1 + 2 + 3 + (4 * TripleInputOffset);
int TripleNonInputIdx =
- TripleInputSum - std::accumulate(TripleInputs.begin(), TripleInputs.end(), 0);
+ TripleInputSum -
+ std::accumulate(TripleInputs.begin(), TripleInputs.end(), 0);
TripleDWord = TripleNonInputIdx / 2;
// We use xor with one to compute the adjacent DWord to whichever one the
@@ -14423,9 +14414,9 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
// Adjust the mask to match the new locations of A and B.
for (int &M : Mask)
- if (M >= 0 && M/2 == ADWord)
+ if (M >= 0 && M / 2 == ADWord)
M = 2 * BDWord + M % 2;
- else if (M >= 0 && M/2 == BDWord)
+ else if (M >= 0 && M / 2 == BDWord)
M = 2 * ADWord + M % 2;
// Recurse back into this routine to re-compute state now that this isn't
@@ -14453,33 +14444,33 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
[&PSHUFDMask](ArrayRef<int> InPlaceInputs, ArrayRef<int> IncomingInputs,
MutableArrayRef<int> SourceHalfMask,
MutableArrayRef<int> HalfMask, int HalfOffset) {
- if (InPlaceInputs.empty())
- return;
- if (InPlaceInputs.size() == 1) {
- SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
- InPlaceInputs[0] - HalfOffset;
- PSHUFDMask[InPlaceInputs[0] / 2] = InPlaceInputs[0] / 2;
- return;
- }
- if (IncomingInputs.empty()) {
- // Just fix all of the in place inputs.
- for (int Input : InPlaceInputs) {
- SourceHalfMask[Input - HalfOffset] = Input - HalfOffset;
- PSHUFDMask[Input / 2] = Input / 2;
- }
- return;
- }
+ if (InPlaceInputs.empty())
+ return;
+ if (InPlaceInputs.size() == 1) {
+ SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
+ InPlaceInputs[0] - HalfOffset;
+ PSHUFDMask[InPlaceInputs[0] / 2] = InPlaceInputs[0] / 2;
+ return;
+ }
+ if (IncomingInputs.empty()) {
+ // Just fix all of the in place inputs.
+ for (int Input : InPlaceInputs) {
+ SourceHalfMask[Input - HalfOffset] = Input - HalfOffset;
+ PSHUFDMask[Input / 2] = Input / 2;
+ }
+ return;
+ }
- assert(InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!");
- SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
- InPlaceInputs[0] - HalfOffset;
- // Put the second input next to the first so that they are packed into
- // a dword. We find the adjacent index by toggling the low bit.
- int AdjIndex = InPlaceInputs[0] ^ 1;
- SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset;
- llvm::replace(HalfMask, InPlaceInputs[1], AdjIndex);
- PSHUFDMask[AdjIndex / 2] = AdjIndex / 2;
- };
+ assert(InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!");
+ SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
+ InPlaceInputs[0] - HalfOffset;
+ // Put the second input next to the first so that they are packed into
+ // a dword. We find the adjacent index by toggling the low bit.
+ int AdjIndex = InPlaceInputs[0] ^ 1;
+ SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset;
+ llvm::replace(HalfMask, InPlaceInputs[1], AdjIndex);
+ PSHUFDMask[AdjIndex / 2] = AdjIndex / 2;
+ };
fixInPlaceInputs(LToLInputs, HToLInputs, PSHUFLMask, LoMask, 0);
fixInPlaceInputs(HToHInputs, LToHInputs, PSHUFHMask, HiMask, 4);
@@ -14488,10 +14479,12 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
// FIXME: This operation could almost certainly be simplified dramatically to
// look more like the 3-1 fixing operation.
auto moveInputsToRightHalf = [&PSHUFDMask](
- MutableArrayRef<int> IncomingInputs, ArrayRef<int> ExistingInputs,
- MutableArrayRef<int> SourceHalfMask, MutableArrayRef<int> HalfMask,
- MutableArrayRef<int> FinalSourceHalfMask, int SourceOffset,
- int DestOffset) {
+ MutableArrayRef<int> IncomingInputs,
+ ArrayRef<int> ExistingInputs,
+ MutableArrayRef<int> SourceHalfMask,
+ MutableArrayRef<int> HalfMask,
+ MutableArrayRef<int> FinalSourceHalfMask,
+ int SourceOffset, int DestOffset) {
auto isWordClobbered = [](ArrayRef<int> SourceHalfMask, int Word) {
return SourceHalfMask[Word] >= 0 && SourceHalfMask[Word] != Word;
};
@@ -14687,9 +14680,11 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
/// Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the
/// blend if only one input is used.
-static SDValue lowerShuffleAsBlendOfPSHUFBs(
- const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
- const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) {
+static SDValue lowerShuffleAsBlendOfPSHUFBs(const SDLoc &DL, MVT VT, SDValue V1,
+ SDValue V2, ArrayRef<int> Mask,
+ const APInt &Zeroable,
+ SelectionDAG &DAG, bool &V1InUse,
+ bool &V2InUse) {
assert(!is128BitLaneCrossingShuffleMask(VT, Mask) &&
"Lane crossing shuffle masks not supported");
@@ -14784,8 +14779,8 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Broadcast;
// Try to use bit rotation instructions.
- if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v8i16, V1, Mask,
- Subtarget, DAG))
+ if (SDValue Rotate =
+ lowerShuffleAsBitRotate(DL, MVT::v8i16, V1, Mask, Subtarget, DAG))
return Rotate;
// Use dedicated unpack instructions for masks that match their pattern.
@@ -14820,14 +14815,14 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// See if we can use SSE4A Extraction / Insertion.
if (Subtarget.hasSSE4A())
- if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask,
- Zeroable, DAG))
+ if (SDValue V =
+ lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask, Zeroable, DAG))
return V;
// There are special ways we can lower some single-element blends.
if (NumV2Inputs == 1)
- if (SDValue V = lowerShuffleAsElementInsertion(
- DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
+ if (SDValue V = lowerShuffleAsElementInsertion(DL, MVT::v8i16, V1, V2, Mask,
+ Zeroable, Subtarget, DAG))
return V;
// We have different paths for blend lowering, but they all must use the
@@ -14943,8 +14938,8 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// can both shuffle and set up the inefficient blend.
if (!IsBlendSupported && Subtarget.hasSSSE3()) {
bool V1InUse, V2InUse;
- return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask,
- Zeroable, DAG, V1InUse, V2InUse);
+ return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask, Zeroable,
+ DAG, V1InUse, V2InUse);
}
// We can always bit-blend if we have to so the fallback strategy is to
@@ -15077,8 +15072,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// See if we can use SSE4A Extraction / Insertion.
if (Subtarget.hasSSE4A())
- if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask,
- Zeroable, DAG))
+ if (SDValue V =
+ lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG))
return V;
int NumV2Elements = count_if(Mask, [](int M) { return M >= 16; });
@@ -15091,8 +15086,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Broadcast;
// Try to use bit rotation instructions.
- if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v16i8, V1, Mask,
- Subtarget, DAG))
+ if (SDValue Rotate =
+ lowerShuffleAsBitRotate(DL, MVT::v16i8, V1, Mask, Subtarget, DAG))
return Rotate;
if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask, DAG))
@@ -15133,7 +15128,7 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
int PreDupI16Shuffle[] = {-1, -1, -1, -1, -1, -1, -1, -1};
SmallDenseMap<int, int, 8> LaneMap;
for (int I : InPlaceInputs) {
- PreDupI16Shuffle[I/2] = I/2;
+ PreDupI16Shuffle[I / 2] = I / 2;
LaneMap[I] = I;
}
int j = TargetLo ? 0 : 4, je = j + 4;
@@ -15147,7 +15142,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
++j;
if (j == je)
- // We can't place the inputs into a single half with a simple i16 shuffle, so bail.
+ // We can't place the inputs into a single half with a simple i16
+ // shuffle, so bail.
return SDValue();
// Map this input with the i16 shuffle.
@@ -15268,8 +15264,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// Use PALIGNR+Permute if possible - permute might become PSHUFB but the
// PALIGNR will be cheaper than the second PSHUFB+OR.
- if (SDValue V = lowerShuffleAsByteRotateAndPermute(
- DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
+ if (SDValue V = lowerShuffleAsByteRotateAndPermute(DL, MVT::v16i8, V1, V2,
+ Mask, Subtarget, DAG))
return V;
}
@@ -15278,8 +15274,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// There are special ways we can lower some single-element blends.
if (NumV2Elements == 1)
- if (SDValue V = lowerShuffleAsElementInsertion(
- DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
+ if (SDValue V = lowerShuffleAsElementInsertion(DL, MVT::v16i8, V1, V2, Mask,
+ Zeroable, Subtarget, DAG))
return V;
if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
@@ -15371,8 +15367,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
if (M >= 0)
M /= 2;
} else {
- // Otherwise just unpack the low half of V into VLoHalf and the high half into
- // VHiHalf so that we can blend them as i16s.
+ // Otherwise just unpack the low half of V into VLoHalf and the high half
+ // into VHiHalf so that we can blend them as i16s.
SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL);
VLoHalf = DAG.getBitcast(
@@ -15381,8 +15377,10 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
}
- SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
- SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);
+ SDValue LoV =
+ DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
+ SDValue HiV =
+ DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);
return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
}
@@ -15391,9 +15389,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
///
/// This routine breaks down the specific type of 128-bit shuffle and
/// dispatches to the lowering routines accordingly.
-static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
- MVT VT, SDValue V1, SDValue V2,
- const APInt &Zeroable,
+static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
+ SDValue V1, SDValue V2, const APInt &Zeroable,
const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
if (VT == MVT::v8bf16) {
@@ -15575,7 +15572,7 @@ static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
assert(!V2.isUndef() && "This routine must not be used to lower single-input "
- "shuffles as it could then recurse on itself.");
+ "shuffles as it could then recurse on itself.");
int Size = Mask.size();
// If this can be modeled as a broadcast of two elements followed by a blend,
@@ -15914,8 +15911,8 @@ static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
// instruction bytes needed to explicitly generate the zero vector.
// Blends are faster and handle all the non-lane-crossing cases.
- if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable,
- Subtarget, DAG))
+ if (SDValue Blend =
+ lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
return Blend;
// If either input operand is a zero vector, use VPERM2X128 because its mask
@@ -15941,8 +15938,8 @@ static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
// Try to use SHUF128 if possible.
if (Subtarget.hasVLX()) {
if (WidenedMask[0] < 2 && WidenedMask[1] >= 2) {
- unsigned PermMask = ((WidenedMask[0] % 2) << 0) |
- ((WidenedMask[1] % 2) << 1);
+ unsigned PermMask =
+ ((WidenedMask[0] % 2) << 0) | ((WidenedMask[1] % 2) << 1);
return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2,
DAG.getTargetConstant(PermMask, DL, MVT::i8));
}
@@ -15966,7 +15963,7 @@ static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
(WidenedMask[1] >= 0 || IsHighZero) && "Undef half?");
unsigned PermMask = 0;
- PermMask |= IsLowZero ? 0x08 : (WidenedMask[0] << 0);
+ PermMask |= IsLowZero ? 0x08 : (WidenedMask[0] << 0);
PermMask |= IsHighZero ? 0x80 : (WidenedMask[1] << 4);
// Check the immediate mask and replace unused sources with undef.
@@ -16157,9 +16154,9 @@ static SDValue lowerShuffleAsLanePermuteAndRepeatedMask(
/// adjusted to access the extracted halves of the original shuffle operands is
/// returned in HalfMask. HalfIdx1 and HalfIdx2 return whether the upper or
/// lower half of each input operand is accessed.
-static bool
-getHalfShuffleMask(ArrayRef<int> Mask, MutableArrayRef<int> HalfMask,
- int &HalfIdx1, int &HalfIdx2) {
+static bool getHalfShuffleMask(ArrayRef<int> Mask,
+ MutableArrayRef<int> HalfMask, int &HalfIdx1,
+ int &HalfIdx2) {
assert((Mask.size() == HalfMask.size() * 2) &&
"Expected input mask to be twice as long as output");
@@ -16212,7 +16209,8 @@ getHalfShuffleMask(ArrayRef<int> Mask, MutableArrayRef<int> HalfMask,
static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2,
ArrayRef<int> HalfMask, int HalfIdx1,
int HalfIdx2, bool UndefLower,
- SelectionDAG &DAG, bool UseConcat = false) {
+ SelectionDAG &DAG,
+ bool UseConcat = false) {
assert(V1.getValueType() == V2.getValueType() && "Different sized vectors?");
assert(V1.getValueType().isSimple() && "Expecting only simple types");
@@ -16574,7 +16572,7 @@ static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
assert(isUndefOrZeroOrInRange(Mask, 0, 2 * NumElts) &&
"Illegal shuffle mask");
- bool ZeroLane[2] = { true, true };
+ bool ZeroLane[2] = {true, true};
for (int i = 0; i < NumElts; ++i)
ZeroLane[i & 1] &= Zeroable[i];
@@ -16659,9 +16657,9 @@ static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT,
// The VTRUNCs will put 0s in the upper 12 bytes. Use them to put zeroes in
// the upper bits of the result using an unpckldq.
- SDValue Unpack = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2,
- { 0, 1, 2, 3, 16, 17, 18, 19,
- 4, 5, 6, 7, 20, 21, 22, 23 });
+ SDValue Unpack = DAG.getVectorShuffle(
+ MVT::v16i8, DL, V1, V2,
+ {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23});
// Insert the unpckldq into a zero vector to widen to v32i8.
return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v32i8,
DAG.getConstant(0, DL, MVT::v32i8), Unpack,
@@ -16898,8 +16896,8 @@ static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Blend;
// Check for being able to broadcast a single element.
- if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask,
- Subtarget, DAG))
+ if (SDValue Broadcast =
+ lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
return Broadcast;
// Try to use shift instructions if fast.
@@ -17006,8 +17004,8 @@ static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Blend;
// Check for being able to broadcast a single element.
- if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask,
- Subtarget, DAG))
+ if (SDValue Broadcast =
+ lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
return Broadcast;
if (!Subtarget.hasAVX2()) {
@@ -17154,8 +17152,8 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Blend;
// Check for being able to broadcast a single element.
- if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask,
- Subtarget, DAG))
+ if (SDValue Broadcast =
+ lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
return Broadcast;
// Try to use shift instructions if fast.
@@ -17322,7 +17320,8 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// Try to produce a fixed cross-128-bit lane permute followed by unpack
// because that should be faster than the variable permute alternatives.
- if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, V1, V2, Mask, DAG))
+ if (SDValue V =
+ lowerShuffleWithUNPCK256(DL, MVT::v16i16, V1, V2, Mask, DAG))
return V;
// There are no generalized cross-lane shuffle operations available on i16
@@ -17341,8 +17340,8 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// As this is a single-input shuffle, the repeated mask should be
// a strictly valid v8i16 mask that we can pass through to the v8i16
// lowering to handle even the v16 case.
- return lowerV8I16GeneralSingleInputShuffle(
- DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG);
+ return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v16i16, V1,
+ RepeatedMask, Subtarget, DAG);
}
}
@@ -17361,8 +17360,8 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Result;
// Try to permute the lanes and then use a per-lane permute.
- if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
- DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
+ if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v16i16, V1, V2,
+ Mask, DAG, Subtarget))
return V;
// Try to match an interleave of two v16i16s and lower them as unpck and
@@ -17398,8 +17397,8 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return ZExt;
// Check for being able to broadcast a single element.
- if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask,
- Subtarget, DAG))
+ if (SDValue Broadcast =
+ lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
return Broadcast;
if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
@@ -17451,8 +17450,8 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, V1, V2, Mask, DAG))
return V;
- if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
- DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
+ if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v32i8, V1, V2,
+ Mask, DAG, Subtarget))
return V;
return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v32i8, V1, V2, Mask,
@@ -17474,16 +17473,16 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Result;
// Try to permute the lanes and then use a per-lane permute.
- if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
- DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
+ if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v32i8, V1, V2,
+ Mask, DAG, Subtarget))
return V;
// Look for {0, 8, 16, 24, 32, 40, 48, 56 } in the first 8 elements. Followed
// by zeroable elements in the remaining 24 elements. Turn this into two
// vmovqb instructions shuffled together.
if (Subtarget.hasVLX())
- if (SDValue V = lowerShuffleAsVTRUNCAndUnpack(DL, MVT::v32i8, V1, V2,
- Mask, Zeroable, DAG))
+ if (SDValue V = lowerShuffleAsVTRUNCAndUnpack(DL, MVT::v32i8, V1, V2, Mask,
+ Zeroable, DAG))
return V;
// Try to match an interleave of two v32i8s and lower them as unpck and
@@ -17538,7 +17537,8 @@ static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
return V;
if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
return V;
- return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
+ return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
+ /*SimpleOnly*/ false);
}
MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits),
@@ -17793,8 +17793,7 @@ static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// If we have a single input shuffle with different shuffle patterns in the
// 128-bit lanes and don't lane cross, use variable mask VPERMILPS.
- if (V2.isUndef() &&
- !is128BitLaneCrossingShuffleMask(MVT::v16f32, Mask)) {
+ if (V2.isUndef() && !is128BitLaneCrossingShuffleMask(MVT::v16f32, Mask)) {
SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true);
return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v16f32, V1, VPermMask);
}
@@ -18066,8 +18065,8 @@ static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// Whenever we can lower this as a zext, that instruction is strictly faster
// than any alternative. It also allows us to fold memory operands into the
// shuffle in many cases.
- if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
- DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
+ if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v64i8, V1, V2, Mask,
+ Zeroable, Subtarget, DAG))
return ZExt;
// Use dedicated unpack instructions for masks that match their pattern.
@@ -18161,13 +18160,11 @@ static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
/// This routine either breaks down the specific type of a 512-bit x86 vector
/// shuffle or splits it into two 256-bit shuffles and fuses the results back
/// together based on the available instructions.
-static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
- MVT VT, SDValue V1, SDValue V2,
- const APInt &Zeroable,
+static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
+ SDValue V1, SDValue V2, const APInt &Zeroable,
const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
- assert(Subtarget.hasAVX512() &&
- "Cannot lower 512-bit vectors w/ basic ISA!");
+ assert(Subtarget.hasAVX512() && "Cannot lower 512-bit vectors w/ basic ISA!");
// If we have a single input to the zero element, insert that into V1 if we
// can do so cheaply.
@@ -18185,8 +18182,8 @@ static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
return V;
// Check for being able to broadcast a single element.
- if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask,
- Subtarget, DAG))
+ if (SDValue Broadcast =
+ lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask, Subtarget, DAG))
return Broadcast;
if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) {
@@ -18197,7 +18194,8 @@ static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
return V;
- return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
+ return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
+ /*SimpleOnly*/ false);
}
if (VT == MVT::v32f16 || VT == MVT::v32bf16) {
@@ -18304,14 +18302,12 @@ static int match1BitShuffleAsKSHIFT(unsigned &Opcode, ArrayRef<int> Mask,
return -1;
}
-
// Lower vXi1 vector shuffles.
// There is no a dedicated instruction on AVX-512 that shuffles the masks.
// The only way to shuffle bits is to sign-extend the mask vector to SIMD
// vector, shuffle and then truncate it back.
-static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
- MVT VT, SDValue V1, SDValue V2,
- const APInt &Zeroable,
+static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
+ SDValue V1, SDValue V2, const APInt &Zeroable,
const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
assert(Subtarget.hasAVX512() &&
@@ -18460,8 +18456,8 @@ static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
int NumElems = VT.getVectorNumElements();
if ((Subtarget.hasBWI() && (NumElems >= 32)) ||
(Subtarget.hasDQI() && (NumElems < 32)))
- return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT),
- Shuffle, ISD::SETGT);
+ return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT), Shuffle,
+ ISD::SETGT);
return DAG.getNode(ISD::TRUNCATE, DL, VT, Shuffle);
}
@@ -18590,7 +18586,7 @@ static SDValue canonicalizeShuffleMaskWithHorizOp(
unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG,
const X86Subtarget &Subtarget);
- /// Top-level lowering for x86 vector shuffles.
+/// Top-level lowering for x86 vector shuffles.
///
/// This handles decomposition, canonicalization, and lowering of all x86
/// vector shuffles. Most of the specific lowering strategies are encapsulated
@@ -18666,8 +18662,8 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, const X86Subtarget &Subtarget,
// by obfuscating the operands with bitcasts.
// TODO: Avoid lowering directly from this top-level function: make this
// a query (canLowerAsBroadcast) and defer lowering to the type-based calls.
- if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask,
- Subtarget, DAG))
+ if (SDValue Broadcast =
+ lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask, Subtarget, DAG))
return Broadcast;
MVT NewEltVT = VT.isFloatingPoint()
@@ -18894,8 +18890,7 @@ SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
// Build a mask by testing the condition against zero.
MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
SDValue Mask = DAG.getSetCC(dl, MaskVT, Cond,
- DAG.getConstant(0, dl, CondVT),
- ISD::SETNE);
+ DAG.getConstant(0, dl, CondVT), ISD::SETNE);
// Now return a new VSELECT using the mask.
return DAG.getSelect(dl, VT, Mask, LHS, RHS);
}
@@ -19002,7 +18997,7 @@ static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
}
if (VT == MVT::i32 || VT == MVT::i64)
- return Op;
+ return Op;
return SDValue();
}
@@ -19015,7 +19010,7 @@ static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
SDLoc dl(Vec);
MVT VecVT = Vec.getSimpleValueType();
SDValue Idx = Op.getOperand(1);
- auto* IdxC = dyn_cast<ConstantSDNode>(Idx);
+ auto *IdxC = dyn_cast<ConstantSDNode>(Idx);
MVT EltVT = Op.getSimpleValueType();
assert((VecVT.getVectorNumElements() <= 16 || Subtarget.hasBWI()) &&
@@ -19030,7 +19025,8 @@ static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
if (NumElts == 1) {
Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
MVT IntVT = MVT::getIntegerVT(Vec.getValueType().getVectorNumElements());
- return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, DAG.getBitcast(IntVT, Vec));
+ return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8,
+ DAG.getBitcast(IntVT, Vec));
}
MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;
MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);
@@ -19088,14 +19084,13 @@ static APInt getExtractedDemandedElts(SDNode *N) {
return DemandedElts;
}
-SDValue
-X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
+ SelectionDAG &DAG) const {
SDLoc dl(Op);
SDValue Vec = Op.getOperand(0);
MVT VecVT = Vec.getSimpleValueType();
SDValue Idx = Op.getOperand(1);
- auto* IdxC = dyn_cast<ConstantSDNode>(Idx);
+ auto *IdxC = dyn_cast<ConstantSDNode>(Idx);
if (VecVT.getVectorElementType() == MVT::i1)
return ExtractBitFromMaskVector(Op, DAG, Subtarget);
@@ -19126,10 +19121,10 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
// | | Ports pressure in cycles | |
// |Uops| 1 | 2 - D |3 - D | 4 | 5 | |
// ---------------------------------------------------------
- // |2^ | | 0.5 | 0.5 |1.0| |CP| vmovaps xmmword ptr [rsp-0x18], xmm0
- // |1 |0.5| | | |0.5| | lea rax, ptr [rsp-0x18]
- // |1 | |0.5, 0.5|0.5, 0.5| | |CP| mov al, byte ptr [rdi+rax*1]
- // Total Num Of Uops: 4
+ // |2^ | | 0.5 | 0.5 |1.0| |CP| vmovaps xmmword ptr [rsp-0x18],
+ // xmm0 |1 |0.5| | | |0.5| | lea rax, ptr [rsp-0x18] |1
+ // | |0.5, 0.5|0.5, 0.5| | |CP| mov al, byte ptr [rdi+rax*1] Total Num
+ // Of Uops: 4
return SDValue();
}
@@ -19234,7 +19229,7 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
// UNPCKHPD the element to the lowest double word, then movsd.
// Note if the lower 64 bits of the result of the UNPCKHPD is then stored
// to a f64mem, the whole operation is folded into a single MOVHPDmr.
- int Mask[2] = { 1, -1 };
+ int Mask[2] = {1, -1};
Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask);
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
DAG.getVectorIdxConstant(0, dl));
@@ -19259,9 +19254,10 @@ static SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG,
unsigned NumElts = VecVT.getVectorNumElements();
MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;
MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);
- SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
- DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec),
- DAG.getNode(ISD::SIGN_EXTEND, dl, ExtEltVT, Elt), Idx);
+ SDValue ExtOp =
+ DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
+ DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec),
+ DAG.getNode(ISD::SIGN_EXTEND, dl, ExtEltVT, Elt), Idx);
return DAG.getNode(ISD::TRUNCATE, dl, VecVT, ExtOp);
}
@@ -19288,9 +19284,9 @@ SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
if (EltVT == MVT::bf16) {
MVT IVT = VT.changeVectorElementTypeToInteger();
- SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVT,
- DAG.getBitcast(IVT, N0),
- DAG.getBitcast(MVT::i16, N1), N2);
+ SDValue Res =
+ DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVT, DAG.getBitcast(IVT, N0),
+ DAG.getBitcast(MVT::i16, N1), N2);
return DAG.getBitcast(VT, Res);
}
@@ -19625,8 +19621,9 @@ static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
}
// Returns the appropriate wrapper opcode for a global reference.
-unsigned X86TargetLowering::getGlobalWrapperKind(
- const GlobalValue *GV, const unsigned char OpFlags) const {
+unsigned
+X86TargetLowering::getGlobalWrapperKind(const GlobalValue *GV,
+ const unsigned char OpFlags) const {
// References to absolute symbols are never PC-relative.
if (GV && GV->isAbsoluteSymbolRef())
return X86ISD::Wrapper;
@@ -19650,8 +19647,8 @@ unsigned X86TargetLowering::getGlobalWrapperKind(
// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
// be used to form addressing mode. These wrapped nodes will be selected
// into MOV32ri.
-SDValue
-X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerConstantPool(SDValue Op,
+ SelectionDAG &DAG) const {
ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
// In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
@@ -19701,11 +19698,10 @@ SDValue X86TargetLowering::LowerExternalSymbol(SDValue Op,
return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false, nullptr);
}
-SDValue
-X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerBlockAddress(SDValue Op,
+ SelectionDAG &DAG) const {
// Create the TargetBlockAddressAddress node.
- unsigned char OpFlags =
- Subtarget.classifyBlockAddressReference();
+ unsigned char OpFlags = Subtarget.classifyBlockAddressReference();
const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
SDLoc dl(Op);
@@ -19810,8 +19806,8 @@ SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
return Result;
}
-SDValue
-X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerGlobalAddress(SDValue Op,
+ SelectionDAG &DAG) const {
return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false, nullptr);
}
@@ -19889,24 +19885,24 @@ static SDValue GetTLSADDR(SelectionDAG &DAG, GlobalAddressSDNode *GA,
}
// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit
-static SDValue
-LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
- const EVT PtrVT) {
+static SDValue LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA,
+ SelectionDAG &DAG,
+ const EVT PtrVT) {
return GetTLSADDR(DAG, GA, PtrVT, X86::EAX, X86II::MO_TLSGD,
/*LoadGlobalBaseReg=*/true);
}
// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit LP64
-static SDValue
-LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
- const EVT PtrVT) {
+static SDValue LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA,
+ SelectionDAG &DAG,
+ const EVT PtrVT) {
return GetTLSADDR(DAG, GA, PtrVT, X86::RAX, X86II::MO_TLSGD);
}
// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit ILP32
-static SDValue
-LowerToTLSGeneralDynamicModelX32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
- const EVT PtrVT) {
+static SDValue LowerToTLSGeneralDynamicModelX32(GlobalAddressSDNode *GA,
+ SelectionDAG &DAG,
+ const EVT PtrVT) {
return GetTLSADDR(DAG, GA, PtrVT, X86::EAX, X86II::MO_TLSGD);
}
@@ -19938,9 +19934,8 @@ static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
// Build x at dtpoff.
unsigned char OperandFlags = X86II::MO_DTPOFF;
unsigned WrapperKind = X86ISD::Wrapper;
- SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
- GA->getValueType(0),
- GA->getOffset(), OperandFlags);
+ SDValue TGA = DAG.getTargetGlobalAddress(
+ GA->getGlobal(), dl, GA->getValueType(0), GA->getOffset(), OperandFlags);
SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
// Add x at dtpoff with the base.
@@ -19981,9 +19976,8 @@ static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
// emit "addl x at ntpoff,%eax" (local exec)
// or "addl x at indntpoff,%eax" (initial exec)
// or "addl x at gotntpoff(%ebx) ,%eax" (initial exec, 32-bit pic)
- SDValue TGA =
- DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
- GA->getOffset(), OperandFlags);
+ SDValue TGA = DAG.getTargetGlobalAddress(
+ GA->getGlobal(), dl, GA->getValueType(0), GA->getOffset(), OperandFlags);
SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
if (model == TLSModel::InitialExec) {
@@ -20002,8 +19996,8 @@ static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
}
-SDValue
-X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerGlobalTLSAddress(SDValue Op,
+ SelectionDAG &DAG) const {
GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
@@ -20024,20 +20018,20 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
if (model == TLSModel::GeneralDynamic || model == TLSModel::LocalDynamic)
model = TLSModel::InitialExec;
switch (model) {
- case TLSModel::GeneralDynamic:
- if (Subtarget.is64Bit()) {
- if (Subtarget.isTarget64BitLP64())
- return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT);
- return LowerToTLSGeneralDynamicModelX32(GA, DAG, PtrVT);
- }
- return LowerToTLSGeneralDynamicModel32(GA, DAG, PtrVT);
- case TLSModel::LocalDynamic:
- return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT, Subtarget.is64Bit(),
- Subtarget.isTarget64BitLP64());
- case TLSModel::InitialExec:
- case TLSModel::LocalExec:
- return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget.is64Bit(),
- PositionIndependent);
+ case TLSModel::GeneralDynamic:
+ if (Subtarget.is64Bit()) {
+ if (Subtarget.isTarget64BitLP64())
+ return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT);
+ return LowerToTLSGeneralDynamicModelX32(GA, DAG, PtrVT);
+ }
+ return LowerToTLSGeneralDynamicModel32(GA, DAG, PtrVT);
+ case TLSModel::LocalDynamic:
+ return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT, Subtarget.is64Bit(),
+ Subtarget.isTarget64BitLP64());
+ case TLSModel::InitialExec:
+ case TLSModel::LocalExec:
+ return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget.is64Bit(),
+ PositionIndependent);
}
llvm_unreachable("Unknown TLS model.");
}
@@ -20058,9 +20052,8 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
WrapperKind = X86ISD::WrapperRIP;
}
SDLoc DL(Op);
- SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
- GA->getValueType(0),
- GA->getOffset(), OpFlag);
+ SDValue Result = DAG.getTargetGlobalAddress(
+ GA->getGlobal(), DL, GA->getValueType(0), GA->getOffset(), OpFlag);
SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result);
// With PIC32, the address is actually $g + Offset.
@@ -20074,7 +20067,7 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
SDValue Chain = DAG.getEntryNode();
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
- SDValue Args[] = { Chain, Offset };
+ SDValue Args[] = {Chain, Offset};
Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args);
Chain = DAG.getCALLSEQ_END(Chain, 0, 0, Chain.getValue(1), DL);
@@ -20142,9 +20135,9 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
res = DAG.getLoad(PtrVT, dl, Chain, res, MachinePointerInfo());
// Get the offset of start of .tls section
- SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
- GA->getValueType(0),
- GA->getOffset(), X86II::MO_SECREL);
+ SDValue TGA =
+ DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
+ GA->getOffset(), X86II::MO_SECREL);
SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA);
// The address of the thread local variable is the add of the thread
@@ -20204,8 +20197,8 @@ static SDValue LowerI64IntToFP_AVX512DQ(SDValue Op, const SDLoc &dl,
MVT SrcVT = Src.getSimpleValueType();
MVT VT = Op.getSimpleValueType();
- if (!Subtarget.hasDQI() || SrcVT != MVT::i64 || Subtarget.is64Bit() ||
- (VT != MVT::f32 && VT != MVT::f64))
+ if (!Subtarget.hasDQI() || SrcVT != MVT::i64 || Subtarget.is64Bit() ||
+ (VT != MVT::f32 && VT != MVT::f64))
return SDValue();
// Pack the i64 into a vector, do the operation and extract.
@@ -20270,22 +20263,22 @@ static SDValue LowerI64IntToFP16(SDValue Op, const SDLoc &dl, SelectionDAG &DAG,
static bool useVectorCast(unsigned Opcode, MVT FromVT, MVT ToVT,
const X86Subtarget &Subtarget) {
switch (Opcode) {
- case ISD::SINT_TO_FP:
- // TODO: Handle wider types with AVX/AVX512.
- if (!Subtarget.hasSSE2() || FromVT != MVT::v4i32)
- return false;
- // CVTDQ2PS or (V)CVTDQ2PD
- return ToVT == MVT::v4f32 || (Subtarget.hasAVX() && ToVT == MVT::v4f64);
-
- case ISD::UINT_TO_FP:
- // TODO: Handle wider types and i64 elements.
- if (!Subtarget.hasAVX512() || FromVT != MVT::v4i32)
- return false;
- // VCVTUDQ2PS or VCVTUDQ2PD
- return ToVT == MVT::v4f32 || ToVT == MVT::v4f64;
+ case ISD::SINT_TO_FP:
+ // TODO: Handle wider types with AVX/AVX512.
+ if (!Subtarget.hasSSE2() || FromVT != MVT::v4i32)
+ return false;
+ // CVTDQ2PS or (V)CVTDQ2PD
+ return ToVT == MVT::v4f32 || (Subtarget.hasAVX() && ToVT == MVT::v4f64);
- default:
+ case ISD::UINT_TO_FP:
+ // TODO: Handle wider types and i64 elements.
+ if (!Subtarget.hasAVX512() || FromVT != MVT::v4i32)
return false;
+ // VCVTUDQ2PS or VCVTUDQ2PD
+ return ToVT == MVT::v4f32 || ToVT == MVT::v4f64;
+
+ default:
+ return false;
}
}
@@ -20465,7 +20458,7 @@ static SDValue lowerINT_TO_FP_vXi64(SDValue Op, const SDLoc &DL,
return SDValue();
SDValue Zero = DAG.getConstant(0, DL, MVT::v4i64);
- SDValue One = DAG.getConstant(1, DL, MVT::v4i64);
+ SDValue One = DAG.getConstant(1, DL, MVT::v4i64);
SDValue Sign = DAG.getNode(ISD::OR, DL, MVT::v4i64,
DAG.getNode(ISD::SRL, DL, MVT::v4i64, Src, One),
DAG.getNode(ISD::AND, DL, MVT::v4i64, Src, One));
@@ -20685,7 +20678,7 @@ std::pair<SDValue, SDValue> X86TargetLowering::BuildFILD(
Chain = Result.getValue(1);
}
- return { Result, Chain };
+ return {Result, Chain};
}
/// Horizontal vector math instructions may be slower than normal math with
@@ -20722,18 +20715,18 @@ static SDValue LowerUINT_TO_FP_i64(SDValue Op, const SDLoc &dl,
LLVMContext *Context = DAG.getContext();
// Build some magic constants.
- static const uint32_t CV0[] = { 0x43300000, 0x45300000, 0, 0 };
+ static const uint32_t CV0[] = {0x43300000, 0x45300000, 0, 0};
Constant *C0 = ConstantDataVector::get(*Context, CV0);
auto PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
SDValue CPIdx0 = DAG.getConstantPool(C0, PtrVT, Align(16));
- SmallVector<Constant*,2> CV1;
+ SmallVector<Constant *, 2> CV1;
CV1.push_back(
- ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
- APInt(64, 0x4330000000000000ULL))));
+ ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
+ APInt(64, 0x4330000000000000ULL))));
CV1.push_back(
- ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
- APInt(64, 0x4530000000000000ULL))));
+ ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
+ APInt(64, 0x4530000000000000ULL))));
Constant *C1 = ConstantVector::get(CV1);
SDValue CPIdx1 = DAG.getConstantPool(C1, PtrVT, Align(16));
@@ -20754,11 +20747,10 @@ static SDValue LowerUINT_TO_FP_i64(SDValue Op, const SDLoc &dl,
SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
SDValue Result;
- if (Subtarget.hasSSE3() &&
- shouldUseHorizontalOp(true, DAG, Subtarget)) {
+ if (Subtarget.hasSSE3() && shouldUseHorizontalOp(true, DAG, Subtarget)) {
Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);
} else {
- SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1});
+ SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1, -1});
Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuffle, Sub);
}
Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result,
@@ -20784,8 +20776,7 @@ static SDValue LowerUINT_TO_FP_i32(SDValue Op, const SDLoc &dl,
// Or the load with the bias.
SDValue Or = DAG.getNode(
- ISD::OR, dl, MVT::v2i64,
- DAG.getBitcast(MVT::v2i64, Load),
+ ISD::OR, dl, MVT::v2i64, DAG.getBitcast(MVT::v2i64, Load),
DAG.getBitcast(MVT::v2i64,
DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Bias)));
Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
@@ -20989,8 +20980,9 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, const SDLoc &DL,
SDValue VecBitcast = DAG.getBitcast(VecI16VT, V);
// Low will be bitcasted right away, so do not bother bitcasting back to its
// original type.
- Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
- VecCstLowBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
+ Low =
+ DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast, VecCstLowBitcast,
+ DAG.getTargetConstant(0xaa, DL, MVT::i8));
// uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
// (uint4) 0x53000000, 0xaa);
SDValue VecCstHighBitcast = DAG.getBitcast(VecI16VT, VecCstHigh);
@@ -20998,7 +20990,8 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, const SDLoc &DL,
// High will be bitcasted right away, so do not bother bitcasting back to
// its original type.
High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
- VecCstHighBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
+ VecCstHighBitcast,
+ DAG.getTargetConstant(0xaa, DL, MVT::i8));
} else {
SDValue VecCstMask = DAG.getConstant(0xffff, DL, VecIntVT);
// uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
@@ -21034,7 +21027,8 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, const SDLoc &DL,
return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh);
}
-static SDValue lowerUINT_TO_FP_vec(SDValue Op, const SDLoc &dl, SelectionDAG &DAG,
+static SDValue lowerUINT_TO_FP_vec(SDValue Op, const SDLoc &dl,
+ SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0;
SDValue N0 = Op.getOperand(OpNo);
@@ -21248,8 +21242,7 @@ SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
DstTy = MVT::i64;
}
- assert(DstTy.getSimpleVT() <= MVT::i64 &&
- DstTy.getSimpleVT() >= MVT::i16 &&
+ assert(DstTy.getSimpleVT() <= MVT::i64 && DstTy.getSimpleVT() >= MVT::i16 &&
"Unknown FP_TO_INT to lower!");
// We lower FP->int64 into FISTP64 followed by a load from a temporary
@@ -21287,8 +21280,8 @@ SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
bool LosesInfo = false;
if (TheVT == MVT::f64)
// The rounding mode is irrelevant as the conversion should be exact.
- Status = Thresh.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
- &LosesInfo);
+ Status = Thresh.convert(APFloat::IEEEdouble(),
+ APFloat::rmNearestTiesToEven, &LosesInfo);
else if (TheVT == MVT::f80)
Status = Thresh.convert(APFloat::x87DoubleExtended(),
APFloat::rmNearestTiesToEven, &LosesInfo);
@@ -21298,8 +21291,8 @@ SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
SDValue ThreshVal = DAG.getConstantFP(Thresh, DL, TheVT);
- EVT ResVT = getSetCCResultType(DAG.getDataLayout(),
- *DAG.getContext(), TheVT);
+ EVT ResVT =
+ getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), TheVT);
SDValue Cmp;
if (IsStrict) {
Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE, Chain,
@@ -21328,8 +21321,8 @@ SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
DAG.getConstantFP(0.0, DL, TheVT));
if (IsStrict) {
- Value = DAG.getNode(ISD::STRICT_FSUB, DL, { TheVT, MVT::Other},
- { Chain, Value, FltOfs });
+ Value = DAG.getNode(ISD::STRICT_FSUB, DL, {TheVT, MVT::Other},
+ {Chain, Value, FltOfs});
Chain = Value.getValue(1);
} else
Value = DAG.getNode(ISD::FSUB, DL, TheVT, Value, FltOfs);
@@ -21343,7 +21336,7 @@ SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!");
Chain = DAG.getStore(Chain, DL, Value, StackSlot, MPI);
SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
- SDValue Ops[] = { Chain, StackSlot };
+ SDValue Ops[] = {Chain, StackSlot};
unsigned FLDSize = TheVT.getStoreSize();
assert(FLDSize <= MemSize && "Stack slot not big enough");
@@ -21356,10 +21349,9 @@ SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
// Build the FP_TO_INT*_IN_MEM
MachineMemOperand *MMO = MF.getMachineMemOperand(
MPI, MachineMemOperand::MOStore, MemSize, Align(MemSize));
- SDValue Ops[] = { Chain, Value, StackSlot };
- SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL,
- DAG.getVTList(MVT::Other),
- Ops, DstTy, MMO);
+ SDValue Ops[] = {Chain, Value, StackSlot};
+ SDValue FIST = DAG.getMemIntrinsicNode(
+ X86ISD::FP_TO_INT_IN_MEM, DL, DAG.getVTList(MVT::Other), Ops, DstTy, MMO);
SDValue Res = DAG.getLoad(Op.getValueType(), DL, FIST, StackSlot, MPI);
Chain = Res.getValue(1);
@@ -21538,7 +21530,7 @@ static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
return In;
unsigned NumElems = SrcVT.getVectorNumElements();
- if (NumElems < 2 || !isPowerOf2_32(NumElems) )
+ if (NumElems < 2 || !isPowerOf2_32(NumElems))
return SDValue();
unsigned DstSizeInBits = DstVT.getSizeInBits();
@@ -21609,7 +21601,7 @@ static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
// Scale shuffle mask to avoid bitcasts and help ComputeNumSignBits.
SmallVector<int, 64> Mask;
int Scale = 64 / OutVT.getScalarSizeInBits();
- narrowShuffleMaskElts(Scale, { 0, 2, 1, 3 }, Mask);
+ narrowShuffleMaskElts(Scale, {0, 2, 1, 3}, Mask);
Res = DAG.getVectorShuffle(OutVT, DL, Res, Res, Mask);
if (DstVT.is256BitVector())
@@ -21853,14 +21845,12 @@ static SDValue LowerTruncateVecI1(SDValue Op, const SDLoc &DL,
if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) {
// We need to shift to get the lsb into sign position.
// Shift packed bytes not supported natively, bitcast to word
- MVT ExtVT = MVT::getVectorVT(MVT::i16, InVT.getSizeInBits()/16);
- In = DAG.getNode(ISD::SHL, DL, ExtVT,
- DAG.getBitcast(ExtVT, In),
+ MVT ExtVT = MVT::getVectorVT(MVT::i16, InVT.getSizeInBits() / 16);
+ In = DAG.getNode(ISD::SHL, DL, ExtVT, DAG.getBitcast(ExtVT, In),
DAG.getConstant(ShiftInx, DL, ExtVT));
In = DAG.getBitcast(InVT, In);
}
- return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT),
- In, ISD::SETGT);
+ return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), In, ISD::SETGT);
}
// Use TESTD/Q, extended vector to packed dword/qword.
assert((InVT.is256BitVector() || InVT.is128BitVector()) &&
@@ -21898,7 +21888,8 @@ static SDValue LowerTruncateVecI1(SDValue Op, const SDLoc &DL,
// We either have 8 elements or we're allowed to use 512-bit vectors.
// If we have VLX, we want to use the narrowest vector that can get the
// job done so we use vXi32.
- MVT EltVT = Subtarget.hasVLX() ? MVT::i32 : MVT::getIntegerVT(512/NumElts);
+ MVT EltVT =
+ Subtarget.hasVLX() ? MVT::i32 : MVT::getIntegerVT(512 / NumElts);
MVT ExtVT = MVT::getVectorVT(EltVT, NumElts);
In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In);
InVT = ExtVT;
@@ -22012,10 +22003,9 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
// On AVX2, v8i32 -> v8i16 becomes PSHUFB.
if (Subtarget.hasInt256()) {
// The PSHUFB mask:
- static const int ShufMask1[] = { 0, 1, 4, 5, 8, 9, 12, 13,
- -1, -1, -1, -1, -1, -1, -1, -1,
- 16, 17, 20, 21, 24, 25, 28, 29,
- -1, -1, -1, -1, -1, -1, -1, -1 };
+ static const int ShufMask1[] = {
+ 0, 1, 4, 5, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1,
+ 16, 17, 20, 21, 24, 25, 28, 29, -1, -1, -1, -1, -1, -1, -1, -1};
In = DAG.getBitcast(MVT::v32i8, In);
In = DAG.getVectorShuffle(MVT::v32i8, DL, In, In, ShufMask1);
In = DAG.getBitcast(MVT::v4i64, In);
@@ -22205,8 +22195,8 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
dl, {NVT, MVT::Other}, {Chain, Src});
Chain = Res.getValue(1);
} else {
- Res = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl,
- NVT, Src);
+ Res = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, NVT,
+ Src);
}
// TODO: Need to add exception check code for strict FP.
@@ -22308,8 +22298,8 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
DAG.getUNDEF(MVT::v2f32));
if (IsStrict) {
- unsigned Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI
- : X86ISD::STRICT_CVTTP2UI;
+ unsigned Opc =
+ IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op->getOperand(0), Tmp});
}
unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
@@ -22434,7 +22424,7 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
if (IsStrict)
- return DAG.getMergeValues({ Tmp.first, Tmp.second }, dl);
+ return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
return Tmp.first;
}
@@ -22498,7 +22488,7 @@ SDValue X86TargetLowering::LRINT_LLRINTHelper(SDNode *N,
assert(DstVT == MVT::i64 && "Invalid LRINT/LLRINT to lower!");
Chain = DAG.getStore(Chain, DL, Src, StackPtr, MPI);
SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
- SDValue Ops[] = { Chain, StackPtr };
+ SDValue Ops[] = {Chain, StackPtr};
Src = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, SrcVT, MPI,
/*Align*/ std::nullopt,
@@ -22506,7 +22496,7 @@ SDValue X86TargetLowering::LRINT_LLRINTHelper(SDNode *N,
Chain = Src.getValue(1);
}
- SDValue StoreOps[] = { Chain, Src, StackPtr };
+ SDValue StoreOps[] = {Chain, Src, StackPtr};
Chain = DAG.getMemIntrinsicNode(X86ISD::FIST, DL, DAG.getVTList(MVT::Other),
StoreOps, DstVT, MPI, /*Align*/ std::nullopt,
MachineMemOperand::MOStore);
@@ -22514,8 +22504,8 @@ SDValue X86TargetLowering::LRINT_LLRINTHelper(SDNode *N,
return DAG.getLoad(DstVT, DL, Chain, StackPtr, MPI);
}
-SDValue
-X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
+ SelectionDAG &DAG) const {
// This is based on the TargetLowering::expandFP_TO_INT_SAT implementation,
// but making use of X86 specifics to produce better instruction sequences.
SDNode *Node = Op.getNode();
@@ -22585,12 +22575,12 @@ X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const {
APFloat MinFloat(Sem);
APFloat MaxFloat(Sem);
- APFloat::opStatus MinStatus = MinFloat.convertFromAPInt(
- MinInt, IsSigned, APFloat::rmTowardZero);
- APFloat::opStatus MaxStatus = MaxFloat.convertFromAPInt(
- MaxInt, IsSigned, APFloat::rmTowardZero);
- bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact)
- && !(MaxStatus & APFloat::opStatus::opInexact);
+ APFloat::opStatus MinStatus =
+ MinFloat.convertFromAPInt(MinInt, IsSigned, APFloat::rmTowardZero);
+ APFloat::opStatus MaxStatus =
+ MaxFloat.convertFromAPInt(MaxInt, IsSigned, APFloat::rmTowardZero);
+ bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
+ !(MaxStatus & APFloat::opStatus::opInexact);
SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);
@@ -22617,8 +22607,8 @@ X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const {
SDValue MinClamped =
DAG.getNode(ISD::PSEUDO_FMAX, dl, SrcVT, Src, MinFloatNode);
// Clamp by MaxFloat from above. NaN cannot occur.
- SDValue BothClamped = DAG.getNode(
- X86ISD::FMINC, dl, SrcVT, MinClamped, MaxFloatNode);
+ SDValue BothClamped =
+ DAG.getNode(X86ISD::FMINC, dl, SrcVT, MinClamped, MaxFloatNode);
// Convert clamped value to integer.
SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, DstVT, BothClamped);
@@ -22630,8 +22620,8 @@ X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const {
// Otherwise, select zero if Src is NaN.
SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
- return DAG.getSelectCC(
- dl, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
+ return DAG.getSelectCC(dl, Src, Src, ZeroInt, FpToInt,
+ ISD::CondCode::SETUO);
}
SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
@@ -22662,8 +22652,8 @@ X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const {
}
// If Src OGT MaxFloat, select MaxInt.
- Select = DAG.getSelectCC(
- dl, Src, MaxFloatNode, MaxIntNode, Select, ISD::CondCode::SETOGT);
+ Select = DAG.getSelectCC(dl, Src, MaxFloatNode, MaxIntNode, Select,
+ ISD::CondCode::SETOGT);
// In the unsigned case we are done, because we mapped NaN to MinInt, which
// is already zero. The promoted case was already handled above.
@@ -22673,8 +22663,7 @@ X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const {
// Otherwise, select 0 if Src is NaN.
SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
- return DAG.getSelectCC(
- dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);
+ return DAG.getSelectCC(dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);
}
SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
@@ -22743,7 +22732,7 @@ SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
std::move(Args));
SDValue Res;
- std::tie(Res,Chain) = LowerCallTo(CLI);
+ std::tie(Res, Chain) = LowerCallTo(CLI);
if (IsStrict)
Res = DAG.getMergeValues({Res, Chain}, DL);
@@ -23017,14 +23006,14 @@ static SDValue lowerAddSubToHorizontalOp(SDValue Op, const SDLoc &DL,
// TODO: Allow commuted (f)sub by negating the result of (F)HSUB?
unsigned HOpcode;
switch (Op.getOpcode()) {
- // clang-format off
+ // clang-format off
case ISD::ADD: HOpcode = X86ISD::HADD; break;
case ISD::SUB: HOpcode = X86ISD::HSUB; break;
case ISD::FADD: HOpcode = X86ISD::FHADD; break;
case ISD::FSUB: HOpcode = X86ISD::FHSUB; break;
default:
llvm_unreachable("Trying to lower unsupported opcode to horizontal op");
- // clang-format on
+ // clang-format on
}
unsigned LExtIndex = LHS.getConstantOperandVal(1);
unsigned RExtIndex = RHS.getConstantOperandVal(1);
@@ -23082,7 +23071,7 @@ static SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) {
bool Ignored;
APFloat Point5Pred = APFloat(0.5f);
Point5Pred.convert(Sem, APFloat::rmNearestTiesToEven, &Ignored);
- Point5Pred.next(/*nextDown*/true);
+ Point5Pred.next(/*nextDown*/ true);
SDValue Adder = DAG.getNode(ISD::FCOPYSIGN, dl, VT,
DAG.getConstantFP(Point5Pred, dl, VT), N0);
@@ -23132,16 +23121,16 @@ static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
unsigned EltBits = VT.getScalarSizeInBits();
// For FABS, mask is 0x7f...; for FNEG, mask is 0x80...
- APInt MaskElt = IsFABS ? APInt::getSignedMaxValue(EltBits) :
- APInt::getSignMask(EltBits);
+ APInt MaskElt =
+ IsFABS ? APInt::getSignedMaxValue(EltBits) : APInt::getSignMask(EltBits);
const fltSemantics &Sem = VT.getFltSemantics();
SDValue Mask = DAG.getConstantFP(APFloat(Sem, MaskElt), dl, LogicVT);
SDValue Op0 = Op.getOperand(0);
bool IsFNABS = !IsFABS && (Op0.getOpcode() == ISD::FABS);
- unsigned LogicOp = IsFABS ? X86ISD::FAND :
- IsFNABS ? X86ISD::FOR :
- X86ISD::FXOR;
+ unsigned LogicOp = IsFABS ? X86ISD::FAND
+ : IsFNABS ? X86ISD::FOR
+ : X86ISD::FXOR;
SDValue Operand = IsFNABS ? Op0.getOperand(0) : Op0;
if (VT.isVector() || IsF128)
@@ -23244,7 +23233,8 @@ static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
}
/// Helper for attempting to create a X86ISD::BT node.
-static SDValue getBT(SDValue Src, SDValue BitNo, const SDLoc &DL, SelectionDAG &DAG) {
+static SDValue getBT(SDValue Src, SDValue BitNo, const SDLoc &DL,
+ SelectionDAG &DAG) {
// If Src is i8, promote it to i32 with any_extend. There is no i8 BT
// instruction. Since the shift amount is in-range-or-undefined, we know
// that doing a bittest on the i32 value is ok. We extend to i32 because
@@ -23853,8 +23843,7 @@ static bool hasNonFlagsUse(SDValue Op) {
// the node alone and emit a 'cmp' or 'test' instruction.
static bool isProfitableToUseFlagOp(SDValue Op) {
for (SDNode *U : Op->users())
- if (U->getOpcode() != ISD::CopyToReg &&
- U->getOpcode() != ISD::SETCC &&
+ if (U->getOpcode() != ISD::CopyToReg && U->getOpcode() != ISD::SETCC &&
U->getOpcode() != ISD::STORE)
return false;
@@ -23870,14 +23859,20 @@ static SDValue EmitTest(SDValue Op, X86::CondCode X86CC, const SDLoc &dl,
bool NeedCF = false;
bool NeedOF = false;
switch (X86CC) {
- default: break;
- case X86::COND_A: case X86::COND_AE:
- case X86::COND_B: case X86::COND_BE:
+ default:
+ break;
+ case X86::COND_A:
+ case X86::COND_AE:
+ case X86::COND_B:
+ case X86::COND_BE:
NeedCF = true;
break;
- case X86::COND_G: case X86::COND_GE:
- case X86::COND_L: case X86::COND_LE:
- case X86::COND_O: case X86::COND_NO: {
+ case X86::COND_G:
+ case X86::COND_GE:
+ case X86::COND_L:
+ case X86::COND_LE:
+ case X86::COND_O:
+ case X86::COND_NO: {
// Check if we really need to set the
// Overflow flag. If NoSignedWrap is present
// that is not actually needed.
@@ -23929,14 +23924,14 @@ static SDValue EmitTest(SDValue Op, X86::CondCode X86CC, const SDLoc &dl,
// Otherwise use a regular EFLAGS-setting instruction.
switch (ArithOp.getOpcode()) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("unexpected operator!");
case ISD::ADD: Opcode = X86ISD::ADD; break;
case ISD::SUB: Opcode = X86ISD::SUB; break;
case ISD::XOR: Opcode = X86ISD::XOR; break;
case ISD::AND: Opcode = X86ISD::AND; break;
case ISD::OR: Opcode = X86ISD::OR; break;
- // clang-format on
+ // clang-format on
}
NumOperands = 2;
@@ -23951,8 +23946,9 @@ static SDValue EmitTest(SDValue Op, X86::CondCode X86CC, const SDLoc &dl,
case ISD::USUBO: {
// /USUBO/SSUBO will become a X86ISD::SUB and we can use its Z flag.
SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
- return DAG.getNode(X86ISD::SUB, dl, VTs, Op->getOperand(0),
- Op->getOperand(1)).getValue(1);
+ return DAG
+ .getNode(X86ISD::SUB, dl, VTs, Op->getOperand(0), Op->getOperand(1))
+ .getValue(1);
}
default:
break;
@@ -23981,8 +23977,9 @@ static SDValue EmitCmp(SDValue Op0, SDValue Op1, X86::CondCode X86CC,
EVT CmpVT = Op0.getValueType();
- assert((CmpVT == MVT::i8 || CmpVT == MVT::i16 ||
- CmpVT == MVT::i32 || CmpVT == MVT::i64) && "Unexpected VT!");
+ assert((CmpVT == MVT::i8 || CmpVT == MVT::i16 || CmpVT == MVT::i32 ||
+ CmpVT == MVT::i64) &&
+ "Unexpected VT!");
// Only promote the compare up to I32 if it is a 16 bit operation
// with an immediate. 16 bit immediates are to be avoided unless the target
@@ -24109,9 +24106,8 @@ bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const {
/// The minimum architected relative accuracy is 2^-12. We need one
/// Newton-Raphson step to have a good float result (24 bits of precision).
-SDValue X86TargetLowering::getSqrtEstimate(SDValue Op,
- SelectionDAG &DAG, int Enabled,
- int &RefinementSteps,
+SDValue X86TargetLowering::getSqrtEstimate(SDValue Op, SelectionDAG &DAG,
+ int Enabled, int &RefinementSteps,
bool &UseOneConstNR,
bool Reciprocal) const {
SDLoc DL(Op);
@@ -24218,9 +24214,7 @@ SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
/// This is because we still need one division to calculate the reciprocal and
/// then we need two multiplies by that reciprocal as replacements for the
/// original divisions.
-unsigned X86TargetLowering::combineRepeatedFPDivisors() const {
- return 2;
-}
+unsigned X86TargetLowering::combineRepeatedFPDivisors() const { return 2; }
SDValue
X86TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
@@ -24228,7 +24222,7 @@ X86TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
SmallVectorImpl<SDNode *> &Created) const {
AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
if (isIntDivCheap(N->getValueType(0), Attr))
- return SDValue(N,0); // Lower SDIV as SDIV
+ return SDValue(N, 0); // Lower SDIV as SDIV
assert((Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()) &&
"Unexpected divisor!");
@@ -24338,7 +24332,7 @@ static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
// 6 - NLE
// 7 - ORD
switch (SetCCOpcode) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected SETCC condition");
case ISD::SETOEQ:
case ISD::SETEQ: SSECC = 0; break;
@@ -24360,7 +24354,7 @@ static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
case ISD::SETO: SSECC = 7; break;
case ISD::SETUEQ: SSECC = 8; break;
case ISD::SETONE: SSECC = 12; break;
- // clang-format on
+ // clang-format on
}
if (Swap)
std::swap(Op0, Op1);
@@ -24649,13 +24643,13 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
Cmp1 = DAG.getNode(
Opc, dl, {VT, MVT::Other},
{Chain, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8)});
- Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Cmp0.getValue(1),
- Cmp1.getValue(1));
+ Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
+ Cmp0.getValue(1), Cmp1.getValue(1));
} else {
- Cmp0 = DAG.getNode(
- Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8));
- Cmp1 = DAG.getNode(
- Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8));
+ Cmp0 = DAG.getNode(Opc, dl, VT, Op0, Op1,
+ DAG.getTargetConstant(CC0, dl, MVT::i8));
+ Cmp1 = DAG.getNode(Opc, dl, VT, Op0, Op1,
+ DAG.getTargetConstant(CC1, dl, MVT::i8));
}
Cmp = DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);
} else {
@@ -24665,8 +24659,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
{Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
Chain = Cmp.getValue(1);
} else
- Cmp = DAG.getNode(
- Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
+ Cmp = DAG.getNode(Opc, dl, VT, Op0, Op1,
+ DAG.getTargetConstant(SSECC, dl, MVT::i8));
}
} else {
// Handle all other FP comparisons here.
@@ -24678,8 +24672,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
{Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
Chain = Cmp.getValue(1);
} else
- Cmp = DAG.getNode(
- Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
+ Cmp = DAG.getNode(Opc, dl, VT, Op0, Op1,
+ DAG.getTargetConstant(SSECC, dl, MVT::i8));
}
if (VT.getFixedSizeInBits() >
@@ -24730,7 +24724,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
// Translate compare code to XOP PCOM compare mode.
unsigned CmpMode = 0;
switch (Cond) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected SETCC condition");
case ISD::SETULT:
case ISD::SETLT: CmpMode = 0x00; break;
@@ -24742,7 +24736,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
case ISD::SETGE: CmpMode = 0x03; break;
case ISD::SETEQ: CmpMode = 0x04; break;
case ISD::SETNE: CmpMode = 0x05; break;
- // clang-format on
+ // clang-format on
}
// Are we comparing unsigned or signed integers?
@@ -24840,13 +24834,13 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
bool Invert = false;
unsigned Opc;
switch (Cond) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected condition code");
case ISD::SETUGT: Invert = true; [[fallthrough]];
case ISD::SETULE: Opc = ISD::UMIN; break;
case ISD::SETULT: Invert = true; [[fallthrough]];
case ISD::SETUGE: Opc = ISD::UMAX; break;
- // clang-format on
+ // clang-format on
}
SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
@@ -24870,10 +24864,10 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
// operations may be required for some comparisons.
unsigned Opc = (Cond == ISD::SETEQ || Cond == ISD::SETNE) ? X86ISD::PCMPEQ
: X86ISD::PCMPGT;
- bool Swap = Cond == ISD::SETLT || Cond == ISD::SETULT ||
- Cond == ISD::SETGE || Cond == ISD::SETUGE;
- bool Invert = Cond == ISD::SETNE ||
- (Cond != ISD::SETEQ && ISD::isTrueWhenEqual(Cond));
+ bool Swap = Cond == ISD::SETLT || Cond == ISD::SETULT || Cond == ISD::SETGE ||
+ Cond == ISD::SETUGE;
+ bool Invert =
+ Cond == ISD::SETNE || (Cond != ISD::SETEQ && ISD::isTrueWhenEqual(Cond));
if (Swap)
std::swap(Op0, Op1);
@@ -24891,7 +24885,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
Op1 = DAG.getBitcast(MVT::v4i32, Op1);
SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
- static const int MaskHi[] = { 1, 1, 3, 3 };
+ static const int MaskHi[] = {1, 1, 3, 3};
SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
return DAG.getBitcast(VT, Result);
@@ -24902,7 +24896,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
Op1 = DAG.getAllOnesConstant(dl, MVT::v4i32);
SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
- static const int MaskHi[] = { 1, 1, 3, 3 };
+ static const int MaskHi[] = {1, 1, 3, 3};
SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
return DAG.getBitcast(VT, Result);
@@ -24941,8 +24935,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
// Create masks for only the low parts/high parts of the 64 bit integers.
- static const int MaskHi[] = { 1, 1, 3, 3 };
- static const int MaskLo[] = { 0, 0, 2, 2 };
+ static const int MaskHi[] = {1, 1, 3, 3};
+ static const int MaskLo[] = {0, 0, 2, 2};
SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);
SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
@@ -24969,7 +24963,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
// Make sure the lower and upper halves are both all-ones.
- static const int Mask[] = { 1, 0, 3, 2 };
+ static const int Mask[] = {1, 0, 3, 2};
SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);
Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf);
@@ -24984,8 +24978,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
// bits of the inputs before performing those operations.
if (FlipSigns) {
MVT EltVT = VT.getVectorElementType();
- SDValue SM = DAG.getConstant(APInt::getSignMask(EltVT.getSizeInBits()), dl,
- VT);
+ SDValue SM =
+ DAG.getConstant(APInt::getSignMask(EltVT.getSizeInBits()), dl, VT);
Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SM);
Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SM);
}
@@ -25002,8 +24996,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
// Try to select this as a KORTEST+SETCC or KTEST+SETCC if possible.
static SDValue EmitAVX512Test(SDValue Op0, SDValue Op1, ISD::CondCode CC,
const SDLoc &dl, SelectionDAG &DAG,
- const X86Subtarget &Subtarget,
- SDValue &X86CC) {
+ const X86Subtarget &Subtarget, SDValue &X86CC) {
assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
// Must be a bitcast from vXi1.
@@ -25150,7 +25143,8 @@ SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
Op.getOpcode() == ISD::STRICT_FSETCCS;
MVT VT = Op->getSimpleValueType(0);
- if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
+ if (VT.isVector())
+ return LowerVSETCC(Op, Subtarget, DAG);
assert(VT == MVT::i8 && "SetCC type must be 8-bit integer");
SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
@@ -25245,7 +25239,8 @@ SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
}
-SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op,
+ SelectionDAG &DAG) const {
SDValue LHS = Op.getOperand(0);
SDValue RHS = Op.getOperand(1);
SDValue Carry = Op.getOperand(2);
@@ -25257,8 +25252,8 @@ SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const
// Recreate the carry if needed.
EVT CarryVT = Carry.getValueType();
- Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
- Carry, DAG.getAllOnesConstant(DL, CarryVT));
+ Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32), Carry,
+ DAG.getAllOnesConstant(DL, CarryVT));
SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1));
@@ -25278,7 +25273,8 @@ getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) {
unsigned BaseOp = 0;
SDLoc DL(Op);
switch (Op.getOpcode()) {
- default: llvm_unreachable("Unknown ovf instruction!");
+ default:
+ llvm_unreachable("Unknown ovf instruction!");
case ISD::SADDO:
BaseOp = X86ISD::ADD;
Cond = X86::COND_O;
@@ -25352,7 +25348,8 @@ static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG) {
SDValue VOp0 = V.getOperand(0);
unsigned InBits = VOp0.getValueSizeInBits();
unsigned Bits = V.getValueSizeInBits();
- return DAG.MaskedValueIsZero(VOp0, APInt::getHighBitsSet(InBits,InBits-Bits));
+ return DAG.MaskedValueIsZero(VOp0,
+ APInt::getHighBitsSet(InBits, InBits - Bits));
}
// Lower various (select (icmp CmpVal, 0), LHS, RHS) custom patterns.
@@ -25687,7 +25684,7 @@ static SDValue emitConjunctionForCCMP(SDValue Val, X86::CondCode &OutCC,
SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
bool AddTest = true;
- SDValue Cond = Op.getOperand(0);
+ SDValue Cond = Op.getOperand(0);
SDValue Op1 = Op.getOperand(1);
SDValue Op2 = Op.getOperand(2);
SDLoc DL(Op);
@@ -25850,14 +25847,13 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
// If condition flag is set by a X86ISD::CMP, then use it as the condition
// setting operand in place of the X86ISD::SETCC.
unsigned CondOpcode = Cond.getOpcode();
- if (CondOpcode == X86ISD::SETCC ||
- CondOpcode == X86ISD::SETCC_CARRY) {
+ if (CondOpcode == X86ISD::SETCC || CondOpcode == X86ISD::SETCC_CARRY) {
CC = Cond.getOperand(0);
SDValue Cmp = Cond.getOperand(1);
bool IllegalFPCMov = false;
- if (VT.isFloatingPoint() && !VT.isVector() &&
- !isScalarFPTypeInSSEReg(VT) && Subtarget.canUseCMOV()) // FPStack?
+ if (VT.isFloatingPoint() && !VT.isVector() && !isScalarFPTypeInSSEReg(VT) &&
+ Subtarget.canUseCMOV()) // FPStack?
IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue());
if ((isX86LogicalCmp(Cmp) && !IllegalFPCMov) ||
@@ -25920,14 +25916,15 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
// X86 doesn't have an i8 cmov. If both operands are the result of a truncate
// widen the cmov and push the truncate through. This avoids introducing a new
// branch during isel and doesn't add any extensions.
- if (Op.getValueType() == MVT::i8 &&
- Op1.getOpcode() == ISD::TRUNCATE && Op2.getOpcode() == ISD::TRUNCATE) {
+ if (Op.getValueType() == MVT::i8 && Op1.getOpcode() == ISD::TRUNCATE &&
+ Op2.getOpcode() == ISD::TRUNCATE) {
SDValue T1 = Op1.getOperand(0), T2 = Op2.getOperand(0);
if (T1.getValueType() == T2.getValueType() &&
// Exclude CopyFromReg to avoid partial register stalls.
- T1.getOpcode() != ISD::CopyFromReg && T2.getOpcode()!=ISD::CopyFromReg){
- SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1,
- CC, Cond);
+ T1.getOpcode() != ISD::CopyFromReg &&
+ T2.getOpcode() != ISD::CopyFromReg) {
+ SDValue Cmov =
+ DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1, CC, Cond);
return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
}
}
@@ -25943,14 +25940,14 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
!X86::mayFoldLoad(Op2, Subtarget))) {
Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
- SDValue Ops[] = { Op2, Op1, CC, Cond };
+ SDValue Ops[] = {Op2, Op1, CC, Cond};
SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops);
return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
}
// X86ISD::CMOV means set the result (which is operand 1) to the RHS if
// condition is true.
- SDValue Ops[] = { Op2, Op1, CC, Cond };
+ SDValue Ops[] = {Op2, Op1, CC, Cond};
return DAG.getNode(X86ISD::CMOV, DL, Op.getValueType(), Ops, Op->getFlags());
}
@@ -26060,9 +26057,9 @@ static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op,
InVT = In.getSimpleValueType();
}
- // SSE41 targets can use the pmov[sz]x* instructions directly for 128-bit results,
- // so are legal and shouldn't occur here. AVX2/AVX512 pmovsx* instructions still
- // need to be handled here for 256/512-bit results.
+ // SSE41 targets can use the pmov[sz]x* instructions directly for 128-bit
+ // results, so are legal and shouldn't occur here. AVX2/AVX512 pmovsx*
+ // instructions still need to be handled here for 256/512-bit results.
if (Subtarget.hasInt256()) {
assert(VT.getSizeInBits() > 128 && "Unexpected 128-bit vector extension");
@@ -26071,9 +26068,8 @@ static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op,
// FIXME: Apparently we create inreg operations that could be regular
// extends.
- unsigned ExtOpc =
- Opc == ISD::SIGN_EXTEND_VECTOR_INREG ? ISD::SIGN_EXTEND
- : ISD::ZERO_EXTEND;
+ unsigned ExtOpc = Opc == ISD::SIGN_EXTEND_VECTOR_INREG ? ISD::SIGN_EXTEND
+ : ISD::ZERO_EXTEND;
return DAG.getNode(ExtOpc, dl, VT, In);
}
@@ -26191,9 +26187,9 @@ static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
SDValue OpLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, In);
unsigned NumElems = InVT.getVectorNumElements();
- SmallVector<int,8> ShufMask(NumElems, -1);
- for (unsigned i = 0; i != NumElems/2; ++i)
- ShufMask[i] = i + NumElems/2;
+ SmallVector<int, 8> ShufMask(NumElems, -1);
+ for (unsigned i = 0; i != NumElems / 2; ++i)
+ ShufMask[i] = i + NumElems / 2;
SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
OpHi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, OpHi);
@@ -26355,11 +26351,10 @@ static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
// TODO: It is possible to support ZExt by zeroing the undef values during
// the shuffle phase or after the shuffle.
static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget,
- SelectionDAG &DAG) {
+ SelectionDAG &DAG) {
MVT RegVT = Op.getSimpleValueType();
assert(RegVT.isVector() && "We only custom lower vector loads.");
- assert(RegVT.isInteger() &&
- "We only custom lower integer vector loads.");
+ assert(RegVT.isInteger() && "We only custom lower integer vector loads.");
LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
SDLoc dl(Ld);
@@ -26403,7 +26398,7 @@ static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
SDValue X86TargetLowering::LowerConditionalBranch(SDValue Op,
SelectionDAG &DAG) const {
SDValue Chain = Op.getOperand(0);
- SDValue Cond = Op.getOperand(1);
+ SDValue Cond = Op.getOperand(1);
SDValue Dest;
if (Op.getOpcode() == ISD::BRCOND)
Dest = Op.getOperand(2);
@@ -26463,7 +26458,7 @@ SDValue X86TargetLowering::LowerConditionalBranch(SDValue Op,
if (User->getOpcode() == ISD::BR) {
SDValue FalseBB = User->getOperand(1);
SDNode *NewBR =
- DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
+ DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
assert(NewBR == User);
(void)NewBR;
Dest = FalseBB;
@@ -26527,9 +26522,8 @@ SDValue X86TargetLowering::LowerConditionalBranch(SDValue Op,
// bytes in one go. Touching the stack at 4K increments is necessary to ensure
// that the guard pages used by the OS virtual memory manager are allocated in
// correct sequence.
-SDValue
-X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
+ SelectionDAG &DAG) const {
MachineFunction &MF = DAG.getMachineFunction();
bool SplitStack = MF.shouldSplitStack();
bool EmitStackProbeCall = hasStackProbeSymbol(MF);
@@ -26540,7 +26534,7 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
// Get the inputs.
SDNode *Node = Op.getNode();
SDValue Chain = Op.getOperand(0);
- SDValue Size = Op.getOperand(1);
+ SDValue Size = Op.getOperand(1);
MaybeAlign Alignment(Op.getConstantOperandVal(2));
EVT VT = Node->getValueType(0);
@@ -26663,8 +26657,9 @@ SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
MemOps.push_back(Store);
// Store ptr to reg_save_area.
- FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(
- Subtarget.isTarget64BitLP64() ? 8 : 4, DL));
+ FIN = DAG.getNode(
+ ISD::ADD, DL, PtrVT, FIN,
+ DAG.getIntPtrConstant(Subtarget.isTarget64BitLP64() ? 8 : 4, DL));
SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT);
Store = DAG.getStore(
Op.getOperand(0), DL, RSFIN, FIN,
@@ -26674,8 +26669,7 @@ SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
}
SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
- assert(Subtarget.is64Bit() &&
- "LowerVAARG only handles 64-bit va_arg!");
+ assert(Subtarget.is64Bit() && "LowerVAARG only handles 64-bit va_arg!");
assert(Op.getNumOperands() == 4);
MachineFunction &MF = DAG.getMachineFunction();
@@ -26699,11 +26693,11 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
// selection mechanism works only for the basic types.
assert(ArgVT != MVT::f80 && "va_arg for f80 not yet implemented");
if (ArgVT.isFloatingPoint() && ArgSize <= 16 /*bytes*/) {
- ArgMode = 2; // Argument passed in XMM register. Use fp_offset.
+ ArgMode = 2; // Argument passed in XMM register. Use fp_offset.
} else {
assert(ArgVT.isInteger() && ArgSize <= 32 /*bytes*/ &&
"Unhandled argument type in LowerVAARG");
- ArgMode = 1; // Argument passed in GPR64 register(s). Use gp_offset.
+ ArgMode = 1; // Argument passed in GPR64 register(s). Use gp_offset.
}
if (ArgMode == 2) {
@@ -26737,7 +26731,7 @@ static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
// where a va_list is still an i8*.
assert(Subtarget.is64Bit() && "This code only handles 64-bit va_copy!");
if (Subtarget.isCallingConvWin64(
- DAG.getMachineFunction().getFunction().getCallingConv()))
+ DAG.getMachineFunction().getFunction().getCallingConv()))
// Probably a Win64 va_copy.
return DAG.expandVACopy(Op.getNode());
@@ -26830,9 +26824,12 @@ static int getSEHRegistrationNodeSize(const Function *Fn) {
// The RegNodeSize is 6 32-bit words for SEH and 4 for C++ EH. See
// WinEHStatePass for the full struct definition.
switch (classifyEHPersonality(Fn->getPersonalityFn())) {
- case EHPersonality::MSVC_X86SEH: return 24;
- case EHPersonality::MSVC_CXX: return 16;
- default: break;
+ case EHPersonality::MSVC_X86SEH:
+ return 24;
+ case EHPersonality::MSVC_CXX:
+ return 16;
+ default:
+ break;
}
report_fatal_error(
"can only recover FP for 32-bit MSVC EH personality functions");
@@ -26922,13 +26919,13 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SDLoc dl(Op);
unsigned IntNo = Op.getConstantOperandVal(0);
MVT VT = Op.getSimpleValueType();
- const IntrinsicData* IntrData = getIntrinsicWithoutChain(IntNo);
+ const IntrinsicData *IntrData = getIntrinsicWithoutChain(IntNo);
// Propagate flags from original node to transformed node(s).
SelectionDAG::FlagInserter FlagsInserter(DAG, Op->getFlags());
if (IntrData) {
- switch(IntrData->Type) {
+ switch (IntrData->Type) {
case INTR_TYPE_1OP: {
// We specify 2 possible opcodes for intrinsics with rounding modes.
// First, we check if the intrinsic may have non-default rounding mode,
@@ -27054,9 +27051,8 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
if (!isRoundModeCurDirection(Rnd))
return SDValue();
}
- return getVectorMaskingNode(
- DAG.getNode(IntrData->Opc0, dl, VT, Src), Mask, PassThru,
- Subtarget, DAG);
+ return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src),
+ Mask, PassThru, Subtarget, DAG);
}
case INTR_TYPE_1OP_MASK_SAE: {
SDValue Src = Op.getOperand(1);
@@ -27097,9 +27093,9 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
if (!isRoundModeCurDirection(Rnd))
return SDValue();
}
- return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1,
- Src2),
- Mask, passThru, Subtarget, DAG);
+ return getScalarMaskingNode(
+ DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2), Mask, passThru,
+ Subtarget, DAG);
}
assert(Op.getNumOperands() == (6U + HasRounding) &&
@@ -27113,9 +27109,9 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
else if (!isRoundModeCurDirection(Sae))
return SDValue();
}
- return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1,
- Src2, RoundingMode),
- Mask, passThru, Subtarget, DAG);
+ return getScalarMaskingNode(
+ DAG.getNode(Opc, dl, VT, Src1, Src2, RoundingMode), Mask, passThru,
+ Subtarget, DAG);
}
case INTR_TYPE_SCALAR_MASK_RND: {
SDValue Src1 = Op.getOperand(1);
@@ -27150,8 +27146,8 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
else
return SDValue();
- return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
- Mask, passThru, Subtarget, DAG);
+ return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2), Mask,
+ passThru, Subtarget, DAG);
}
case INTR_TYPE_2OP_MASK: {
SDValue Src1 = Op.getOperand(1);
@@ -27187,8 +27183,8 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
return SDValue();
}
- return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
- Mask, PassThru, Subtarget, DAG);
+ return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2), Mask,
+ PassThru, Subtarget, DAG);
}
case INTR_TYPE_3OP_SCALAR_MASK_SAE: {
SDValue Src1 = Op.getOperand(1);
@@ -27237,12 +27233,12 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
// Reverse the operands to match VSELECT order.
return DAG.getNode(IntrData->Opc0, dl, VT, Src3, Src2, Src1);
}
- case VPERM_2OP : {
+ case VPERM_2OP: {
SDValue Src1 = Op.getOperand(1);
SDValue Src2 = Op.getOperand(2);
// Swap Src1 and Src2 in the node creation
- return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1);
+ return DAG.getNode(IntrData->Opc0, dl, VT, Src2, Src1);
}
case CFMA_OP_MASKZ:
case CFMA_OP_MASK: {
@@ -27286,8 +27282,8 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SDValue Imm = Op.getOperand(2);
SDValue Mask = Op.getOperand(3);
SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm);
- SDValue FPclassMask = getScalarMaskingNode(FPclass, Mask, SDValue(),
- Subtarget, DAG);
+ SDValue FPclassMask =
+ getScalarMaskingNode(FPclass, Mask, SDValue(), Subtarget, DAG);
// Need to fill with zeros to ensure the bitcast will produce zeroes
// for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
@@ -27312,7 +27308,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
if (!isRoundModeCurDirection(Sae))
return SDValue();
}
- //default rounding mode
+ // default rounding mode
return DAG.getNode(IntrData->Opc0, dl, MaskVT,
{Op.getOperand(1), Op.getOperand(2), CC, Mask});
}
@@ -27331,12 +27327,12 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
else if (!isRoundModeCurDirection(Sae))
return SDValue();
}
- //default rounding mode
+ // default rounding mode
if (!Cmp.getNode())
Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Src2, CC);
- SDValue CmpMask = getScalarMaskingNode(Cmp, Mask, SDValue(),
- Subtarget, DAG);
+ SDValue CmpMask =
+ getScalarMaskingNode(Cmp, Mask, SDValue(), Subtarget, DAG);
// Need to fill with zeros to ensure the bitcast will produce zeroes
// for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
@@ -27505,8 +27501,8 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
assert(IntrData->Opc0 == X86ISD::BEXTRI && "Unexpected opcode");
uint64_t Imm = Op.getConstantOperandVal(2);
- SDValue Control = DAG.getTargetConstant(Imm & 0xffff, dl,
- Op.getValueType());
+ SDValue Control =
+ DAG.getTargetConstant(Imm & 0xffff, dl, Op.getValueType());
return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
Op.getOperand(1), Control);
}
@@ -27528,7 +27524,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
Op.getOperand(3), GenCF.getValue(1));
}
SDValue SetCC = getSETCC(X86::COND_B, Res.getValue(1), dl, DAG);
- SDValue Results[] = { SetCC, Res };
+ SDValue Results[] = {SetCC, Res};
return DAG.getMergeValues(Results, dl);
}
case CVTPD2PS_MASK:
@@ -27611,7 +27607,8 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
}
switch (IntNo) {
- default: return SDValue(); // Don't custom lower most intrinsics.
+ default:
+ return SDValue(); // Don't custom lower most intrinsics.
// ptest and testp intrinsics. The intrinsic these come from are designed to
// return an integer value, not just an instruction so lower it to the ptest
@@ -27645,7 +27642,8 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
unsigned TestOpc = X86ISD::PTEST;
X86::CondCode X86CC;
switch (IntNo) {
- default: llvm_unreachable("Bad fallthrough in Intrinsic lowering.");
+ default:
+ llvm_unreachable("Bad fallthrough in Intrinsic lowering.");
case Intrinsic::x86_avx512_ktestc_b:
case Intrinsic::x86_avx512_ktestc_w:
case Intrinsic::x86_avx512_ktestc_d:
@@ -27716,7 +27714,8 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
unsigned Opcode;
X86::CondCode X86CC;
switch (IntNo) {
- default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
+ default:
+ llvm_unreachable("Impossible intrinsic"); // Can't reach here.
case Intrinsic::x86_sse42_pcmpistria128:
Opcode = X86ISD::PCMPISTR;
X86CC = X86::COND_A;
@@ -27886,7 +27885,8 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
unsigned NewIntrinsic;
switch (IntNo) {
- default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
+ default:
+ llvm_unreachable("Impossible intrinsic"); // Can't reach here.
case Intrinsic::x86_mmx_pslli_w:
NewIntrinsic = Intrinsic::x86_mmx_psll_w;
break;
@@ -27963,16 +27963,16 @@ static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
- SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
+ SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale};
SDValue Res =
DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops,
MemIntr->getMemoryVT(), MemIntr->getMemOperand());
return DAG.getMergeValues({Res, Res.getValue(1)}, dl);
}
-static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
- SDValue Src, SDValue Mask, SDValue Base,
- SDValue Index, SDValue ScaleOp, SDValue Chain,
+static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG, SDValue Src,
+ SDValue Mask, SDValue Base, SDValue Index,
+ SDValue ScaleOp, SDValue Chain,
const X86Subtarget &Subtarget) {
MVT VT = Op.getSimpleValueType();
SDLoc dl(Op);
@@ -28001,7 +28001,7 @@ static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
- SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
+ SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale};
SDValue Res =
DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops,
MemIntr->getMemoryVT(), MemIntr->getMemOperand());
@@ -28009,9 +28009,9 @@ static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
}
static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
- SDValue Src, SDValue Mask, SDValue Base,
- SDValue Index, SDValue ScaleOp, SDValue Chain,
- const X86Subtarget &Subtarget) {
+ SDValue Src, SDValue Mask, SDValue Base,
+ SDValue Index, SDValue ScaleOp, SDValue Chain,
+ const X86Subtarget &Subtarget) {
SDLoc dl(Op);
auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
// Scale must be constant.
@@ -28053,8 +28053,8 @@ static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
TLI.getPointerTy(DAG.getDataLayout()));
SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
SDValue Segment = DAG.getRegister(0, MVT::i32);
- MVT MaskVT =
- MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements());
+ MVT MaskVT = MVT::getVectorVT(
+ MVT::i1, Index.getSimpleValueType().getVectorNumElements());
SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
SDValue Ops[] = {VMask, Base, Scale, Index, Disp, Segment, Chain};
SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops);
@@ -28070,11 +28070,11 @@ static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
/// expanded intrinsics implicitly defines extra registers (i.e. not just
/// EDX:EAX).
static SDValue expandIntrinsicWChainHelper(SDNode *N, const SDLoc &DL,
- SelectionDAG &DAG,
- unsigned TargetOpcode,
- unsigned SrcReg,
- const X86Subtarget &Subtarget,
- SmallVectorImpl<SDValue> &Results) {
+ SelectionDAG &DAG,
+ unsigned TargetOpcode,
+ unsigned SrcReg,
+ const X86Subtarget &Subtarget,
+ SmallVectorImpl<SDValue> &Results) {
SDValue Chain = N->getOperand(0);
SDValue Glue;
@@ -28114,7 +28114,7 @@ static SDValue expandIntrinsicWChainHelper(SDNode *N, const SDLoc &DL,
}
// Use a buildpair to merge the two 32-bit values into a 64-bit one.
- SDValue Ops[] = { LO, HI };
+ SDValue Ops[] = {LO, HI};
SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
Results.push_back(Pair);
Results.push_back(Chain);
@@ -28131,9 +28131,9 @@ static void getReadTimeStampCounter(SDNode *N, const SDLoc &DL, unsigned Opcode,
// The processor's time-stamp counter (a 64-bit MSR) is stored into the
// EDX:EAX registers. EDX is loaded with the high-order 32 bits of the MSR
// and the EAX register is loaded with the low-order 32 bits.
- SDValue Glue = expandIntrinsicWChainHelper(N, DL, DAG, Opcode,
- /* NoRegister */0, Subtarget,
- Results);
+ SDValue Glue =
+ expandIntrinsicWChainHelper(N, DL, DAG, Opcode,
+ /* NoRegister */ 0, Subtarget, Results);
if (Opcode != X86::RDTSCP)
return;
@@ -28191,10 +28191,9 @@ static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG) {
}
/// Emit Truncating Store with signed or unsigned saturation.
-static SDValue
-EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val,
- SDValue Ptr, EVT MemVT, MachineMemOperand *MMO,
- SelectionDAG &DAG) {
+static SDValue EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &DL,
+ SDValue Val, SDValue Ptr, EVT MemVT,
+ MachineMemOperand *MMO, SelectionDAG &DAG) {
SDVTList VTs = DAG.getVTList(MVT::Other);
SDValue Ops[] = {Chain, Val, Ptr};
unsigned Opc = SignedSat ? X86ISD::VTRUNCSTORES : X86ISD::VTRUNCSTOREUS;
@@ -28203,11 +28202,12 @@ EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val,
/// Emit Masked Truncating Store with signed or unsigned saturation.
static SDValue EmitMaskedTruncSStore(bool SignedSat, SDValue Chain,
- const SDLoc &DL,
- SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT,
- MachineMemOperand *MMO, SelectionDAG &DAG) {
+ const SDLoc &DL, SDValue Val, SDValue Ptr,
+ SDValue Mask, EVT MemVT,
+ MachineMemOperand *MMO,
+ SelectionDAG &DAG) {
SDVTList VTs = DAG.getVTList(MVT::Other);
- SDValue Ops[] = { Chain, Val, Ptr, Mask };
+ SDValue Ops[] = {Chain, Val, Ptr, Mask};
unsigned Opc = SignedSat ? X86ISD::VMTRUNCSTORES : X86ISD::VMTRUNCSTOREUS;
return DAG.getMemIntrinsicNode(Opc, DL, VTs, Ops, MemVT, MMO);
}
@@ -28275,9 +28275,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
SDLoc dl(Op);
// Create a WRPKRU node, pass the input to the EAX parameter, and pass 0
// to the EDX and ECX parameters.
- return DAG.getNode(X86ISD::WRPKRU, dl, MVT::Other,
- Op.getOperand(0), Op.getOperand(2),
- DAG.getConstant(0, dl, MVT::i32),
+ return DAG.getNode(X86ISD::WRPKRU, dl, MVT::Other, Op.getOperand(0),
+ Op.getOperand(2), DAG.getConstant(0, dl, MVT::i32),
DAG.getConstant(0, dl, MVT::i32));
}
case llvm::Intrinsic::asan_check_memaccess: {
@@ -28308,7 +28307,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
unsigned Opcode;
switch (IntNo) {
- default: llvm_unreachable("Impossible intrinsic");
+ default:
+ llvm_unreachable("Impossible intrinsic");
case Intrinsic::x86_umwait:
Opcode = X86ISD::UMWAIT;
break;
@@ -28321,9 +28321,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
break;
}
- SDValue Operation =
- DAG.getNode(Opcode, dl, VTs, Chain, Op->getOperand(2),
- Op->getOperand(3), Op->getOperand(4));
+ SDValue Operation = DAG.getNode(Opcode, dl, VTs, Chain, Op->getOperand(2),
+ Op->getOperand(3), Op->getOperand(4));
SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
Operation.getValue(1));
@@ -28335,7 +28334,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
unsigned Opcode;
switch (IntNo) {
- default: llvm_unreachable("Impossible intrinsic!");
+ default:
+ llvm_unreachable("Impossible intrinsic!");
case Intrinsic::x86_enqcmd:
Opcode = X86ISD::ENQCMD;
break;
@@ -28359,7 +28359,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
unsigned Opcode;
switch (IntNo) {
- default: llvm_unreachable("Impossible intrinsic");
+ default:
+ llvm_unreachable("Impossible intrinsic");
case Intrinsic::x86_aesenc128kl:
Opcode = X86ISD::AESENC128KL;
break;
@@ -28397,7 +28398,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
unsigned Opcode;
switch (IntNo) {
- default: llvm_unreachable("Impossible intrinsic");
+ default:
+ llvm_unreachable("Impossible intrinsic");
case Intrinsic::x86_aesencwide128kl:
Opcode = X86ISD::AESENCWIDE128KL;
break;
@@ -28578,8 +28580,9 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
}
SDLoc dl(Op);
- switch(IntrData->Type) {
- default: llvm_unreachable("Unknown Intrinsic Type");
+ switch (IntrData->Type) {
+ default:
+ llvm_unreachable("Unknown Intrinsic Type");
case RDSEED:
case RDRAND: {
// Emit the node with the right value type.
@@ -28600,32 +28603,32 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
}
case GATHER_AVX2: {
SDValue Chain = Op.getOperand(0);
- SDValue Src = Op.getOperand(2);
- SDValue Base = Op.getOperand(3);
+ SDValue Src = Op.getOperand(2);
+ SDValue Base = Op.getOperand(3);
SDValue Index = Op.getOperand(4);
- SDValue Mask = Op.getOperand(5);
+ SDValue Mask = Op.getOperand(5);
SDValue Scale = Op.getOperand(6);
return getAVX2GatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
Scale, Chain, Subtarget);
}
case GATHER: {
- //gather(v1, mask, index, base, scale);
+ // gather(v1, mask, index, base, scale);
SDValue Chain = Op.getOperand(0);
- SDValue Src = Op.getOperand(2);
- SDValue Base = Op.getOperand(3);
+ SDValue Src = Op.getOperand(2);
+ SDValue Base = Op.getOperand(3);
SDValue Index = Op.getOperand(4);
- SDValue Mask = Op.getOperand(5);
+ SDValue Mask = Op.getOperand(5);
SDValue Scale = Op.getOperand(6);
- return getGatherNode(Op, DAG, Src, Mask, Base, Index, Scale,
- Chain, Subtarget);
+ return getGatherNode(Op, DAG, Src, Mask, Base, Index, Scale, Chain,
+ Subtarget);
}
case SCATTER: {
- //scatter(base, mask, index, v1, scale);
+ // scatter(base, mask, index, v1, scale);
SDValue Chain = Op.getOperand(0);
- SDValue Base = Op.getOperand(2);
- SDValue Mask = Op.getOperand(3);
+ SDValue Base = Op.getOperand(2);
+ SDValue Mask = Op.getOperand(3);
SDValue Index = Op.getOperand(4);
- SDValue Src = Op.getOperand(5);
+ SDValue Src = Op.getOperand(5);
SDValue Scale = Op.getOperand(6);
return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
Scale, Chain, Subtarget);
@@ -28636,9 +28639,9 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
"Wrong prefetch hint in intrinsic: should be 2 or 3");
unsigned Opcode = (HintVal == 2 ? IntrData->Opc1 : IntrData->Opc0);
SDValue Chain = Op.getOperand(0);
- SDValue Mask = Op.getOperand(2);
+ SDValue Mask = Op.getOperand(2);
SDValue Index = Op.getOperand(3);
- SDValue Base = Op.getOperand(4);
+ SDValue Base = Op.getOperand(4);
SDValue Scale = Op.getOperand(5);
return getPrefetchNode(Opcode, Op, DAG, Mask, Base, Index, Scale, Chain,
Subtarget);
@@ -28673,8 +28676,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
SDValue SetCC = getSETCC(X86::COND_NE, InTrans, dl, DAG);
SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
- return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
- Ret, SDValue(InTrans.getNode(), 1));
+ return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Ret,
+ SDValue(InTrans.getNode(), 1));
}
case TRUNCATE_TO_MEM_VI8:
case TRUNCATE_TO_MEM_VI16:
@@ -28687,7 +28690,7 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
MemIntrinsicSDNode *MemIntr = dyn_cast<MemIntrinsicSDNode>(Op);
assert(MemIntr && "Expected MemIntrinsicSDNode!");
- EVT MemVT = MemIntr->getMemoryVT();
+ EVT MemVT = MemIntr->getMemoryVT();
uint16_t TruncationOp = IntrData->Opc0;
switch (TruncationOp) {
@@ -28782,7 +28785,7 @@ SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
Register FrameReg =
RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction());
- SDLoc dl(Op); // FIXME probably not meaningful
+ SDLoc dl(Op); // FIXME probably not meaningful
unsigned Depth = Op.getConstantOperandVal(0);
assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
(FrameReg == X86::EBP && VT == MVT::i32)) &&
@@ -28799,7 +28802,7 @@ SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
// FIXME? Maybe this could be a TableGen attribute on some registers and
// this table could be generated automatically from RegInfo.
-Register X86TargetLowering::getRegisterByName(const char* RegName, LLT VT,
+Register X86TargetLowering::getRegisterByName(const char *RegName, LLT VT,
const MachineFunction &MF) const {
const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
@@ -28863,10 +28866,10 @@ bool X86TargetLowering::needsFixedCatchObjects() const {
}
SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
- SDValue Chain = Op.getOperand(0);
- SDValue Offset = Op.getOperand(1);
- SDValue Handler = Op.getOperand(2);
- SDLoc dl (Op);
+ SDValue Chain = Op.getOperand(0);
+ SDValue Offset = Op.getOperand(1);
+ SDValue Handler = Op.getOperand(2);
+ SDLoc dl(Op);
EVT PtrVT = getPointerTy(DAG.getDataLayout());
const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
@@ -28877,9 +28880,9 @@ SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT);
Register StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX;
- SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
- DAG.getIntPtrConstant(RegInfo->getSlotSize(),
- dl));
+ SDValue StoreAddr =
+ DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
+ DAG.getIntPtrConstant(RegInfo->getSlotSize(), dl));
StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StoreAddr, Offset);
Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());
Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr);
@@ -28902,19 +28905,20 @@ SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
(void)TII->getGlobalBaseReg(&DAG.getMachineFunction());
}
return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
- DAG.getVTList(MVT::i32, MVT::Other),
- Op.getOperand(0), Op.getOperand(1));
+ DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0),
+ Op.getOperand(1));
}
SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
- return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
- Op.getOperand(0), Op.getOperand(1));
+ return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other, Op.getOperand(0),
+ Op.getOperand(1));
}
-SDValue X86TargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue
+X86TargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
+ SelectionDAG &DAG) const {
SDLoc DL(Op);
return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other,
Op.getOperand(0));
@@ -28930,7 +28934,7 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
SDValue Trmp = Op.getOperand(1); // trampoline
SDValue FPtr = Op.getOperand(2); // nested function
SDValue Nest = Op.getOperand(3); // 'nest' parameter value
- SDLoc dl (Op);
+ SDLoc dl(Op);
const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
@@ -28939,7 +28943,7 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
SDValue OutChains[6];
// Large code-model.
- const unsigned char JMP64r = 0xFF; // 64-bit jmp through register opcode.
+ const unsigned char JMP64r = 0xFF; // 64-bit jmp through register opcode.
const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode.
const unsigned char N86R10 = TRI->getEncodingValue(X86::R10) & 0x7;
@@ -28987,7 +28991,7 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
} else {
const Function *Func =
- cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
+ cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
CallingConv::ID CC = Func->getCallingConv();
unsigned NestReg;
@@ -29009,7 +29013,8 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
unsigned Idx = 0;
for (FunctionType::param_iterator I = FTy->param_begin(),
- E = FTy->param_end(); I != E; ++I, ++Idx)
+ E = FTy->param_end();
+ I != E; ++I, ++Idx)
if (Attrs.hasParamAttr(Idx, Attribute::InReg)) {
const DataLayout &DL = DAG.getDataLayout();
// FIXME: should only count parameters that are lowered to integers.
@@ -29115,18 +29120,16 @@ SDValue X86TargetLowering::LowerGET_ROUNDING(SDValue Op,
Chain = CWD.getValue(1);
// Mask and turn the control bits into a shift for the lookup table.
- SDValue Shift =
- DAG.getNode(ISD::SRL, DL, MVT::i16,
- DAG.getNode(ISD::AND, DL, MVT::i16,
- CWD, DAG.getConstant(0xc00, DL, MVT::i16)),
- DAG.getConstant(9, DL, MVT::i8));
+ SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i16,
+ DAG.getNode(ISD::AND, DL, MVT::i16, CWD,
+ DAG.getConstant(0xc00, DL, MVT::i16)),
+ DAG.getConstant(9, DL, MVT::i8));
Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Shift);
SDValue LUT = DAG.getConstant(0x2d, DL, MVT::i32);
- SDValue RetVal =
- DAG.getNode(ISD::AND, DL, MVT::i32,
- DAG.getNode(ISD::SRL, DL, MVT::i32, LUT, Shift),
- DAG.getConstant(3, DL, MVT::i32));
+ SDValue RetVal = DAG.getNode(ISD::AND, DL, MVT::i32,
+ DAG.getNode(ISD::SRL, DL, MVT::i32, LUT, Shift),
+ DAG.getConstant(3, DL, MVT::i32));
RetVal = DAG.getZExtOrTrunc(RetVal, DL, VT);
@@ -29434,17 +29437,15 @@ static SDValue LowerVectorCTLZ_AVX512CDI(SDValue Op, SelectionDAG &DAG,
MVT EltVT = VT.getVectorElementType();
unsigned NumElems = VT.getVectorNumElements();
- assert((EltVT == MVT::i8 || EltVT == MVT::i16) &&
- "Unsupported element type");
+ assert((EltVT == MVT::i8 || EltVT == MVT::i16) && "Unsupported element type");
// Split vector, it's Lo and Hi parts will be handled in next iteration.
- if (NumElems > 16 ||
- (NumElems == 16 && !Subtarget.canExtendTo512DQ()))
+ if (NumElems > 16 || (NumElems == 16 && !Subtarget.canExtendTo512DQ()))
return splitVectorIntUnary(Op, DAG, dl);
MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);
assert((NewVT.is256BitVector() || NewVT.is512BitVector()) &&
- "Unsupported value type for operation");
+ "Unsupported value type for operation");
// Use native supported vector instruction vplzcntd.
Op = DAG.getNode(ISD::ZERO_EXTEND, dl, NewVT, Op.getOperand(0));
@@ -30400,7 +30401,7 @@ static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
// Merge the two vectors back together with a shuffle. This expands into 2
// shuffles.
- static const int ShufMask[] = { 0, 4, 2, 6 };
+ static const int ShufMask[] = {0, 4, 2, 6};
return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask);
}
@@ -30537,7 +30538,7 @@ static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
//
// Place the odd value at an even position (basically, shift all values 1
// step to the left):
- const int Mask[] = {1, -1, 3, -1, 5, -1, 7, -1,
+ const int Mask[] = {1, -1, 3, -1, 5, -1, 7, -1,
9, -1, 11, -1, 13, -1, 15, -1};
// <a|b|c|d> => <b|undef|d|undef>
SDValue Odd0 =
@@ -30587,7 +30588,7 @@ static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
// Only i8 vectors should need custom lowering after this.
assert((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) ||
- (VT == MVT::v64i8 && Subtarget.hasBWI())) &&
+ (VT == MVT::v64i8 && Subtarget.hasBWI())) &&
"Unsupported vector type");
// Lower v16i8/v32i8 as extension to v8i16/v16i16 vector pairs, multiply,
@@ -30742,7 +30743,8 @@ static SDValue LowerMULO(SDValue Op, const X86Subtarget &Subtarget,
return DAG.getMergeValues({Low, Ovf}, dl);
}
-SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op,
+ SelectionDAG &DAG) const {
assert(Subtarget.isTargetWin64() && "Unexpected target");
EVT VT = Op.getValueType();
assert(VT.isInteger() && VT.getSizeInBits() == 128 &&
@@ -30757,13 +30759,13 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) cons
RTLIB::Libcall LC;
bool isSigned;
switch (Op->getOpcode()) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected request for libcall!");
case ISD::SDIV: isSigned = true; LC = RTLIB::SDIV_I128; break;
case ISD::UDIV: isSigned = false; LC = RTLIB::UDIV_I128; break;
case ISD::SREM: isSigned = true; LC = RTLIB::SREM_I128; break;
case ISD::UREM: isSigned = false; LC = RTLIB::UREM_I128; break;
- // clang-format on
+ // clang-format on
}
SDLoc dl(Op);
@@ -30905,9 +30907,9 @@ static bool supportedVectorShiftWithImm(EVT VT, const X86Subtarget &Subtarget,
// The shift amount is a variable, but it is the same for all vector lanes.
// These instructions are defined together with shift-immediate.
-static
-bool supportedVectorShiftWithBaseAmnt(EVT VT, const X86Subtarget &Subtarget,
- unsigned Opcode) {
+static bool supportedVectorShiftWithBaseAmnt(EVT VT,
+ const X86Subtarget &Subtarget,
+ unsigned Opcode) {
return supportedVectorShiftWithImm(VT, Subtarget, Opcode);
}
@@ -30936,7 +30938,7 @@ static bool supportedVectorVarShift(EVT VT, const X86Subtarget &Subtarget,
return true;
bool LShift = VT.is128BitVector() || VT.is256BitVector();
- bool AShift = LShift && VT != MVT::v2i64 && VT != MVT::v4i64;
+ bool AShift = LShift && VT != MVT::v2i64 && VT != MVT::v4i64;
return (Opcode == ISD::SRA) ? AShift : LShift;
}
@@ -33062,7 +33064,8 @@ X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
auto SSID = AI->getSyncScopeID();
// We must restrict the ordering to avoid generating loads with Release or
// ReleaseAcquire orderings.
- auto Order = AtomicCmpXchgInst::getStrongestFailureOrdering(AI->getOrdering());
+ auto Order =
+ AtomicCmpXchgInst::getStrongestFailureOrdering(AI->getOrdering());
// Before the load we need a fence. Here is an example lifted from
// http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf showing why a fence
@@ -33131,31 +33134,28 @@ static SDValue emitLockedStackOp(SelectionDAG &DAG,
if (Subtarget.is64Bit()) {
SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
- SDValue Ops[] = {
- DAG.getRegister(X86::RSP, MVT::i64), // Base
- DAG.getTargetConstant(1, DL, MVT::i8), // Scale
- DAG.getRegister(0, MVT::i64), // Index
- DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
- DAG.getRegister(0, MVT::i16), // Segment.
- Zero,
- Chain};
- SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
- MVT::Other, Ops);
+ SDValue Ops[] = {DAG.getRegister(X86::RSP, MVT::i64), // Base
+ DAG.getTargetConstant(1, DL, MVT::i8), // Scale
+ DAG.getRegister(0, MVT::i64), // Index
+ DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
+ DAG.getRegister(0, MVT::i16), // Segment.
+ Zero,
+ Chain};
+ SDNode *Res =
+ DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32, MVT::Other, Ops);
return SDValue(Res, 1);
}
SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
- SDValue Ops[] = {
- DAG.getRegister(X86::ESP, MVT::i32), // Base
- DAG.getTargetConstant(1, DL, MVT::i8), // Scale
- DAG.getRegister(0, MVT::i32), // Index
- DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
- DAG.getRegister(0, MVT::i16), // Segment.
- Zero,
- Chain
- };
- SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
- MVT::Other, Ops);
+ SDValue Ops[] = {DAG.getRegister(X86::ESP, MVT::i32), // Base
+ DAG.getTargetConstant(1, DL, MVT::i8), // Scale
+ DAG.getRegister(0, MVT::i32), // Index
+ DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
+ DAG.getRegister(0, MVT::i16), // Segment.
+ Zero,
+ Chain};
+ SDNode *Res =
+ DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32, MVT::Other, Ops);
return SDValue(Res, 1);
}
@@ -33188,36 +33188,44 @@ static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget &Subtarget,
SDLoc DL(Op);
unsigned Reg = 0;
unsigned size = 0;
- switch(T.SimpleTy) {
- default: llvm_unreachable("Invalid value type!");
- case MVT::i8: Reg = X86::AL; size = 1; break;
- case MVT::i16: Reg = X86::AX; size = 2; break;
- case MVT::i32: Reg = X86::EAX; size = 4; break;
+ switch (T.SimpleTy) {
+ default:
+ llvm_unreachable("Invalid value type!");
+ case MVT::i8:
+ Reg = X86::AL;
+ size = 1;
+ break;
+ case MVT::i16:
+ Reg = X86::AX;
+ size = 2;
+ break;
+ case MVT::i32:
+ Reg = X86::EAX;
+ size = 4;
+ break;
case MVT::i64:
assert(Subtarget.is64Bit() && "Node not type legal!");
- Reg = X86::RAX; size = 8;
+ Reg = X86::RAX;
+ size = 8;
break;
}
- SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
- Op.getOperand(2), SDValue());
- SDValue Ops[] = { cpIn.getValue(0),
- Op.getOperand(1),
- Op.getOperand(3),
- DAG.getTargetConstant(size, DL, MVT::i8),
- cpIn.getValue(1) };
+ SDValue cpIn =
+ DAG.getCopyToReg(Op.getOperand(0), DL, Reg, Op.getOperand(2), SDValue());
+ SDValue Ops[] = {cpIn.getValue(0), Op.getOperand(1), Op.getOperand(3),
+ DAG.getTargetConstant(size, DL, MVT::i8), cpIn.getValue(1)};
SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
- SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
- Ops, T, MMO);
+ SDValue Result =
+ DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys, Ops, T, MMO);
SDValue cpOut =
- DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
+ DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
MVT::i32, cpOut.getValue(2));
SDValue Success = getSETCC(X86::COND_E, EFLAGS, DL, DAG);
- return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
- cpOut, Success, EFLAGS.getValue(1));
+ return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), cpOut, Success,
+ EFLAGS.getValue(1));
}
// Create X86ISD::MOVMSK, taking into account whether we need to split.
@@ -33285,7 +33293,8 @@ static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget,
}
assert((SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8 ||
- SrcVT == MVT::i64) && "Unexpected VT!");
+ SrcVT == MVT::i64) &&
+ "Unexpected VT!");
assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
if (!(DstVT == MVT::f64 && SrcVT == MVT::i64) &&
@@ -33299,8 +33308,7 @@ static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget,
// Example: from MVT::v2i32 to MVT::v4i32.
MVT NewVT = MVT::getVectorVT(SrcVT.getVectorElementType(),
SrcVT.getVectorNumElements() * 2);
- Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT, Src,
- DAG.getUNDEF(SrcVT));
+ Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT, Src, DAG.getUNDEF(SrcVT));
} else {
assert(SrcVT == MVT::i64 && !Subtarget.is64Bit() &&
"Unexpected source type in LowerBITCAST");
@@ -33446,7 +33454,8 @@ static SDValue LowerVectorCTPOP(SDValue Op, const SDLoc &DL,
if (Subtarget.hasVPOPCNTDQ()) {
unsigned NumElems = VT.getVectorNumElements();
assert((VT.getVectorElementType() == MVT::i8 ||
- VT.getVectorElementType() == MVT::i16) && "Unexpected type");
+ VT.getVectorElementType() == MVT::i16) &&
+ "Unexpected type");
if (NumElems < 16 || (NumElems == 16 && Subtarget.canExtendTo512DQ())) {
MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);
Op = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, Op0);
@@ -33898,16 +33907,16 @@ static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG,
SDValue NewChain = DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Chain);
assert(!N->hasAnyUseOfValue(0));
// NOTE: The getUNDEF is needed to give something for the unused result 0.
- return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
- DAG.getUNDEF(VT), NewChain);
+ return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), DAG.getUNDEF(VT),
+ NewChain);
}
SDValue LockOp = lowerAtomicArithWithLOCK(N, DAG, Subtarget);
// RAUW the chain, but don't worry about the result, as it's unused.
assert(!N->hasAnyUseOfValue(0));
// NOTE: The getUNDEF is needed to give something for the unused result 0.
- return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
- DAG.getUNDEF(VT), LockOp.getValue(1));
+ return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), DAG.getUNDEF(VT),
+ LockOp.getValue(1));
}
static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
@@ -34014,17 +34023,17 @@ static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) {
// Set the carry flag.
SDValue Carry = Op.getOperand(2);
EVT CarryVT = Carry.getValueType();
- Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
- Carry, DAG.getAllOnesConstant(DL, CarryVT));
+ Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32), Carry,
+ DAG.getAllOnesConstant(DL, CarryVT));
bool IsAdd = Opc == ISD::UADDO_CARRY || Opc == ISD::SADDO_CARRY;
- SDValue Sum = DAG.getNode(IsAdd ? X86ISD::ADC : X86ISD::SBB, DL, VTs,
- Op.getOperand(0), Op.getOperand(1),
- Carry.getValue(1));
+ SDValue Sum =
+ DAG.getNode(IsAdd ? X86ISD::ADC : X86ISD::SBB, DL, VTs, Op.getOperand(0),
+ Op.getOperand(1), Carry.getValue(1));
bool IsSigned = Opc == ISD::SADDO_CARRY || Opc == ISD::SSUBO_CARRY;
- SDValue SetCC = getSETCC(IsSigned ? X86::COND_O : X86::COND_B,
- Sum.getValue(1), DL, DAG);
+ SDValue SetCC =
+ getSETCC(IsSigned ? X86::COND_O : X86::COND_B, Sum.getValue(1), DL, DAG);
if (N->getValueType(1) == MVT::i1)
SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
@@ -34120,8 +34129,8 @@ static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget,
if (!Subtarget.hasVLX() && !VT.is512BitVector() &&
!Index.getSimpleValueType().is512BitVector()) {
// Determine how much we need to widen by to get a 512-bit type.
- unsigned Factor = std::min(512/VT.getSizeInBits(),
- 512/IndexVT.getSizeInBits());
+ unsigned Factor =
+ std::min(512 / VT.getSizeInBits(), 512 / IndexVT.getSizeInBits());
unsigned NumElts = VT.getVectorNumElements() * Factor;
VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
@@ -34163,7 +34172,7 @@ static SDValue LowerMLOAD(SDValue Op, const X86Subtarget &Subtarget,
N->isExpandingLoad());
// Emit a blend.
SDValue Select = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
- return DAG.getMergeValues({ Select, NewLoad.getValue(1) }, dl);
+ return DAG.getMergeValues({Select, NewLoad.getValue(1)}, dl);
}
assert((!N->isExpandingLoad() || Subtarget.hasAVX512()) &&
@@ -34230,7 +34239,7 @@ static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget,
// This operation is legal for targets with VLX, but without
// VLX the vector should be widened to 512 bit
- unsigned NumEltsInWideVec = 512/VT.getScalarSizeInBits();
+ unsigned NumEltsInWideVec = 512 / VT.getScalarSizeInBits();
MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec);
// Mask element has to be i1.
@@ -34272,8 +34281,8 @@ static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget,
if (Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&
!IndexVT.is512BitVector()) {
// Determine how much we need to widen by to get a 512-bit type.
- unsigned Factor = std::min(512/VT.getSizeInBits(),
- 512/IndexVT.getSizeInBits());
+ unsigned Factor =
+ std::min(512 / VT.getSizeInBits(), 512 / IndexVT.getSizeInBits());
unsigned NumElts = VT.getVectorNumElements() * Factor;
@@ -34290,8 +34299,8 @@ static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget,
if (PassThru.isUndef())
PassThru = getZeroVector(VT, Subtarget, DAG, dl);
- SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
- N->getScale() };
+ SDValue Ops[] = {N->getChain(), PassThru, Mask,
+ N->getBasePtr(), Index, N->getScale()};
SDValue NewGather = DAG.getMemIntrinsicNode(
X86ISD::MGATHER, dl, DAG.getVTList(VT, MVT::Other), Ops, N->getMemoryVT(),
N->getMemOperand());
@@ -34471,7 +34480,7 @@ SDValue X86TargetLowering::visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL,
/// Provide custom lowering hooks for some operations.
SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
switch (Op.getOpcode()) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Should not custom lower this!");
case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, Subtarget, DAG);
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
@@ -34646,7 +34655,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
/// Replace a node with an illegal result type with a new node built out of
/// custom code.
void X86TargetLowering::ReplaceNodeResults(SDNode *N,
- SmallVectorImpl<SDValue>&Results,
+ SmallVectorImpl<SDValue> &Results,
SelectionDAG &DAG) const {
SDLoc dl(N);
unsigned Opc = N->getOpcode();
@@ -35182,8 +35191,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
SDValue Ovf = DAG.getSetCC(dl, N->getValueType(1), Hi, HiCmp, ISD::SETNE);
// Widen the result with by padding with undef.
- Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res,
- DAG.getUNDEF(VT));
+ Res =
+ DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res, DAG.getUNDEF(VT));
Results.push_back(Res);
Results.push_back(Ovf);
return;
@@ -35200,11 +35209,10 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
"Unexpected type action!");
unsigned NumConcat = 128 / InVT.getSizeInBits();
- EVT InWideVT = EVT::getVectorVT(*DAG.getContext(),
- InVT.getVectorElementType(),
- NumConcat * InVT.getVectorNumElements());
- EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
- VT.getVectorElementType(),
+ EVT InWideVT =
+ EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(),
+ NumConcat * InVT.getVectorNumElements());
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
NumConcat * VT.getVectorNumElements());
SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
@@ -35266,7 +35274,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
case ISD::UDIV:
case ISD::SREM:
case ISD::UREM: {
- SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG);
+ SDValue V = LowerWin64_i128OP(SDValue(N, 0), DAG);
Results.push_back(V);
return;
}
@@ -35344,9 +35352,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
Lo = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Lo);
Hi = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Hi);
- SDValue Res = DAG.getVectorShuffle(MVT::v16i8, dl, Lo, Hi,
- { 0, 1, 2, 3, 16, 17, 18, 19,
- -1, -1, -1, -1, -1, -1, -1, -1 });
+ SDValue Res = DAG.getVectorShuffle(
+ MVT::v16i8, dl, Lo, Hi,
+ {0, 1, 2, 3, 16, 17, 18, 19, -1, -1, -1, -1, -1, -1, -1, -1});
Results.push_back(Res);
return;
}
@@ -35378,7 +35386,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
SDValue In = N->getOperand(0);
EVT InVT = In.getValueType();
if (!Subtarget.hasSSE41() && VT == MVT::v4i64 &&
- (InVT == MVT::v4i16 || InVT == MVT::v4i8)){
+ (InVT == MVT::v4i16 || InVT == MVT::v4i8)) {
assert(getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector &&
"Unexpected type action!");
assert(Opc == ISD::SIGN_EXTEND && "Unexpected opcode");
@@ -35394,11 +35402,11 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
// Create an unpackl and unpackh to interleave the sign bits then bitcast
// to v2i64.
- SDValue Lo = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
- {0, 4, 1, 5});
+ SDValue Lo =
+ DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits, {0, 4, 1, 5});
Lo = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Lo);
- SDValue Hi = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
- {2, 6, 3, 7});
+ SDValue Hi =
+ DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits, {2, 6, 3, 7});
Hi = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Hi);
SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
@@ -35617,7 +35625,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
return;
}
-
if (VT == MVT::v2i32) {
assert((!IsStrict || IsSigned || Subtarget.hasAVX512()) &&
"Strict unsigned conversion requires AVX512");
@@ -35702,9 +35709,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
}
SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
- SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT,
- DAG.getConstantFP(0.0, dl, VecInVT), Src,
- ZeroIdx);
+ SDValue Res =
+ DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT,
+ DAG.getConstantFP(0.0, dl, VecInVT), Src, ZeroIdx);
SDValue Chain;
if (IsStrict) {
SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
@@ -35791,8 +35798,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
EVT SrcVT = Src.getValueType();
if (Subtarget.hasDQI() && Subtarget.hasVLX() && SrcVT == MVT::v2i64) {
if (IsStrict) {
- unsigned Opc = IsSigned ? X86ISD::STRICT_CVTSI2P
- : X86ISD::STRICT_CVTUI2P;
+ unsigned Opc =
+ IsSigned ? X86ISD::STRICT_CVTSI2P : X86ISD::STRICT_CVTUI2P;
SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},
{N->getOperand(0), Src});
Results.push_back(Res);
@@ -35806,7 +35813,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
if (SrcVT == MVT::v2i64 && !IsSigned && Subtarget.is64Bit() &&
Subtarget.hasSSE41() && !Subtarget.hasAVX512()) {
SDValue Zero = DAG.getConstant(0, dl, SrcVT);
- SDValue One = DAG.getConstant(1, dl, SrcVT);
+ SDValue One = DAG.getConstant(1, dl, SrcVT);
SDValue Sign = DAG.getNode(ISD::OR, dl, SrcVT,
DAG.getNode(ISD::SRL, dl, SrcVT, Src, One),
DAG.getNode(ISD::AND, dl, SrcVT, Src, One));
@@ -35872,9 +35879,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
if (IsStrict) {
SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other},
{N->getOperand(0), Or, VBias});
- SDValue Res = DAG.getNode(X86ISD::STRICT_VFPROUND, dl,
- {MVT::v4f32, MVT::Other},
- {Sub.getValue(1), Sub});
+ SDValue Res =
+ DAG.getNode(X86ISD::STRICT_VFPROUND, dl, {MVT::v4f32, MVT::Other},
+ {Sub.getValue(1), Sub});
Results.push_back(Res);
Results.push_back(Res.getValue(1));
} else {
@@ -35959,8 +35966,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
case ISD::INTRINSIC_W_CHAIN: {
unsigned IntNo = N->getConstantOperandVal(1);
switch (IntNo) {
- default : llvm_unreachable("Do not know how to custom type "
- "legalize this intrinsic operation!");
+ default:
+ llvm_unreachable("Do not know how to custom type "
+ "legalize this intrinsic operation!");
case Intrinsic::x86_rdtsc:
return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget,
Results);
@@ -35973,7 +35981,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
return;
case Intrinsic::x86_rdpru:
expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPRU, X86::ECX, Subtarget,
- Results);
+ Results);
return;
case Intrinsic::x86_xgetbv:
expandIntrinsicWChainHelper(N, dl, DAG, X86::XGETBV, X86::ECX, Subtarget,
@@ -36030,12 +36038,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
}
SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
- Regs64bit ? X86::RAX : X86::EAX,
- HalfT, Result.getValue(1));
+ Regs64bit ? X86::RAX : X86::EAX, HalfT,
+ Result.getValue(1));
SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl,
- Regs64bit ? X86::RDX : X86::EDX,
- HalfT, cpOutL.getValue(2));
- SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
+ Regs64bit ? X86::RDX : X86::EDX, HalfT,
+ cpOutL.getValue(2));
+ SDValue OpsF[] = {cpOutL.getValue(0), cpOutH.getValue(0)};
SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS,
MVT::i32, cpOutH.getValue(2));
@@ -36074,7 +36082,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
// Then extract the lower 64-bits.
MVT LdVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32;
SDVTList Tys = DAG.getVTList(LdVT, MVT::Other);
- SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
+ SDValue Ops[] = {Node->getChain(), Node->getBasePtr()};
SDValue Ld = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
MVT::i64, Node->getMemOperand());
if (Subtarget.hasSSE2()) {
@@ -36098,10 +36106,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
// First load this into an 80-bit X87 register. This will put the whole
// integer into the significand.
SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
- SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
- SDValue Result = DAG.getMemIntrinsicNode(X86ISD::FILD,
- dl, Tys, Ops, MVT::i64,
- Node->getMemOperand());
+ SDValue Ops[] = {Node->getChain(), Node->getBasePtr()};
+ SDValue Result = DAG.getMemIntrinsicNode(
+ X86ISD::FILD, dl, Tys, Ops, MVT::i64, Node->getMemOperand());
SDValue Chain = Result.getValue(1);
// Now store the X87 register to a stack temporary and convert to i64.
@@ -36112,7 +36119,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
MachinePointerInfo MPI =
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
- SDValue StoreOps[] = { Chain, Result, StackPtr };
+ SDValue StoreOps[] = {Chain, Result, StackPtr};
Chain = DAG.getMemIntrinsicNode(
X86ISD::FIST, dl, DAG.getVTList(MVT::Other), StoreOps, MVT::i64,
MPI, std::nullopt /*Align*/, MachineMemOperand::MOStore);
@@ -36170,8 +36177,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
assert(getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector &&
"Unexpected type action!");
EVT WideVT = getTypeToTransformTo(*DAG.getContext(), DstVT);
- SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64,
- N->getOperand(0));
+ SDValue Res =
+ DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, N->getOperand(0));
Res = DAG.getBitcast(WideVT, Res);
Results.push_back(Res);
return;
@@ -36193,8 +36200,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
SDValue Mask = Gather->getMask();
assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");
SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT,
- Gather->getPassThru(),
- DAG.getUNDEF(VT));
+ Gather->getPassThru(), DAG.getUNDEF(VT));
if (!Subtarget.hasVLX()) {
// We need to widen the mask, but the instruction will only use 2
// of its elements. So we can use undef.
@@ -36202,8 +36208,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
DAG.getUNDEF(MVT::v2i1));
Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Mask);
}
- SDValue Ops[] = { Gather->getChain(), PassThru, Mask,
- Gather->getBasePtr(), Index, Gather->getScale() };
+ SDValue Ops[] = {Gather->getChain(), PassThru, Mask,
+ Gather->getBasePtr(), Index, Gather->getScale()};
SDValue Res = DAG.getMemIntrinsicNode(
X86ISD::MGATHER, dl, DAG.getVTList(WideVT, MVT::Other), Ops,
Gather->getMemoryVT(), Gather->getMemOperand());
@@ -36248,7 +36254,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
return;
}
case ISD::ADDRSPACECAST: {
- SDValue V = LowerADDRSPACECAST(SDValue(N,0), DAG);
+ SDValue V = LowerADDRSPACECAST(SDValue(N, 0), DAG);
Results.push_back(V);
return;
}
@@ -36342,7 +36348,7 @@ bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
if (AM.HasBaseReg)
return false;
break;
- default: // Other stuff never works.
+ default: // Other stuff never works.
return false;
}
@@ -36445,12 +36451,13 @@ bool X86TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
if (Val.getOpcode() != ISD::LOAD)
return false;
- if (!VT1.isSimple() || !VT1.isInteger() ||
- !VT2.isSimple() || !VT2.isInteger())
+ if (!VT1.isSimple() || !VT1.isInteger() || !VT2.isSimple() ||
+ !VT2.isInteger())
return false;
switch (VT1.getSimpleVT().SimpleTy) {
- default: break;
+ default:
+ break;
case MVT::i8:
case MVT::i16:
case MVT::i32:
@@ -36686,8 +36693,10 @@ static MachineBasicBlock *emitXBegin(MachineInstr &MI, MachineBasicBlock *MBB,
// sinkMBB:
// DstReg := phi(mainDstReg/mainBB, fallDstReg/fallBB)
BuildMI(*sinkMBB, sinkMBB->begin(), MIMD, TII->get(X86::PHI), DstReg)
- .addReg(mainDstReg).addMBB(mainMBB)
- .addReg(fallDstReg).addMBB(fallMBB);
+ .addReg(mainDstReg)
+ .addMBB(mainMBB)
+ .addReg(fallDstReg)
+ .addMBB(fallMBB);
MI.eraseFromParent();
return sinkMBB;
@@ -36753,8 +36762,8 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
unsigned TotalNumXMMRegs = 8;
bool UseGPOffset = (ArgMode == 1);
bool UseFPOffset = (ArgMode == 2);
- unsigned MaxOffset = TotalNumIntRegs * 8 +
- (UseFPOffset ? TotalNumXMMRegs * 16 : 0);
+ unsigned MaxOffset =
+ TotalNumIntRegs * 8 + (UseFPOffset ? TotalNumXMMRegs * 16 : 0);
/* Align ArgSize to a multiple of 8 */
unsigned ArgSizeA8 = (ArgSize + 7) & ~7;
@@ -36832,13 +36841,14 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
// Check if there is enough room left to pull this argument.
BuildMI(thisMBB, MIMD, TII->get(X86::CMP32ri))
- .addReg(OffsetReg)
- .addImm(MaxOffset + 8 - ArgSizeA8);
+ .addReg(OffsetReg)
+ .addImm(MaxOffset + 8 - ArgSizeA8);
// Branch to "overflowMBB" if offset >= max
// Fall through to "offsetMBB" otherwise
BuildMI(thisMBB, MIMD, TII->get(X86::JCC_1))
- .addMBB(overflowMBB).addImm(X86::COND_AE);
+ .addMBB(overflowMBB)
+ .addImm(X86::COND_AE);
}
// In offsetMBB, emit code to use the reg_save_area.
@@ -36879,8 +36889,8 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
// Compute the offset for the next argument
Register NextOffsetReg = MRI.createVirtualRegister(OffsetRegClass);
BuildMI(offsetMBB, MIMD, TII->get(X86::ADD32ri), NextOffsetReg)
- .addReg(OffsetReg)
- .addImm(UseFPOffset ? 16 : 8);
+ .addReg(OffsetReg)
+ .addImm(UseFPOffset ? 16 : 8);
// Store it back into the va_list.
BuildMI(offsetMBB, MIMD, TII->get(X86::MOV32mr))
@@ -36893,8 +36903,7 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
.setMemRefs(StoreOnlyMMO);
// Jump to endMBB
- BuildMI(offsetMBB, MIMD, TII->get(X86::JMP_1))
- .addMBB(endMBB);
+ BuildMI(offsetMBB, MIMD, TII->get(X86::JMP_1)).addMBB(endMBB);
}
//
@@ -36935,7 +36944,7 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
.addImm(~(uint64_t)(Alignment.value() - 1));
} else {
BuildMI(overflowMBB, MIMD, TII->get(TargetOpcode::COPY), OverflowDestReg)
- .addReg(OverflowAddrReg);
+ .addReg(OverflowAddrReg);
}
// Compute the next overflow address after this argument.
@@ -36961,10 +36970,11 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
// If we branched, emit the PHI to the front of endMBB.
if (offsetMBB) {
- BuildMI(*endMBB, endMBB->begin(), MIMD,
- TII->get(X86::PHI), DestReg)
- .addReg(OffsetDestReg).addMBB(offsetMBB)
- .addReg(OverflowDestReg).addMBB(overflowMBB);
+ BuildMI(*endMBB, endMBB->begin(), MIMD, TII->get(X86::PHI), DestReg)
+ .addReg(OffsetDestReg)
+ .addMBB(offsetMBB)
+ .addReg(OverflowDestReg)
+ .addMBB(overflowMBB);
}
// Erase the pseudo instruction
@@ -36979,8 +36989,8 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
// kill marker, and set it if it should. Returns the correct kill
// marker value.
static bool checkAndUpdateEFLAGSKill(MachineBasicBlock::iterator SelectItr,
- MachineBasicBlock* BB,
- const TargetRegisterInfo* TRI) {
+ MachineBasicBlock *BB,
+ const TargetRegisterInfo *TRI) {
if (isPhysRegUsedAfter(X86::EFLAGS, SelectItr))
return false;
@@ -37447,11 +37457,21 @@ X86TargetLowering::EmitLoweredProbedAlloca(MachineInstr &MI,
//
// + ---- <- ------------ <- ------------- <- ------------ +
// | |
- // [free probe] -> [page alloc] -> [alloc probe] -> [tail alloc] + -> [dyn probe] -> [page alloc] -> [dyn probe] -> [tail alloc] +
- // | |
- // + <- ----------- <- ------------ <- ----------- <- ------------ +
+ // [free probe] -> [page alloc] -> [alloc probe] -> [tail alloc] + -> [dyn
+ // probe] -> [page alloc] -> [dyn probe] -> [tail alloc] +
+ // | |
+ // + <-
+ // -----------
+ // <-
+ // ------------
+ // <-
+ // -----------
+ // <-
+ // ------------
+ // +
//
- // The property we want to enforce is to never have more than [page alloc] between two probes.
+ // The property we want to enforce is to never have more than [page alloc]
+ // between two probes.
const unsigned XORMIOpc =
TFI.Uses64BitFramePtr ? X86::XOR64mi32 : X86::XOR32mi;
@@ -37543,56 +37563,61 @@ X86TargetLowering::EmitLoweredSegAlloca(MachineInstr &MI,
// Add code to the main basic block to check if the stack limit has been hit,
// and if so, jump to mallocMBB otherwise to bumpMBB.
BuildMI(BB, MIMD, TII->get(TargetOpcode::COPY), tmpSPVReg).addReg(physSPReg);
- BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::SUB64rr:X86::SUB32rr), SPLimitVReg)
- .addReg(tmpSPVReg).addReg(sizeVReg);
- BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::CMP64mr:X86::CMP32mr))
- .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg)
- .addReg(SPLimitVReg);
+ BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::SUB64rr : X86::SUB32rr), SPLimitVReg)
+ .addReg(tmpSPVReg)
+ .addReg(sizeVReg);
+ BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::CMP64mr : X86::CMP32mr))
+ .addReg(0)
+ .addImm(1)
+ .addReg(0)
+ .addImm(TlsOffset)
+ .addReg(TlsReg)
+ .addReg(SPLimitVReg);
BuildMI(BB, MIMD, TII->get(X86::JCC_1)).addMBB(mallocMBB).addImm(X86::COND_G);
// bumpMBB simply decreases the stack pointer, since we know the current
// stacklet has enough space.
BuildMI(bumpMBB, MIMD, TII->get(TargetOpcode::COPY), physSPReg)
- .addReg(SPLimitVReg);
+ .addReg(SPLimitVReg);
BuildMI(bumpMBB, MIMD, TII->get(TargetOpcode::COPY), bumpSPPtrVReg)
- .addReg(SPLimitVReg);
+ .addReg(SPLimitVReg);
BuildMI(bumpMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB);
// Calls into a routine in libgcc to allocate more space from the heap.
const uint32_t *RegMask =
Subtarget.getRegisterInfo()->getCallPreservedMask(*MF, CallingConv::C);
if (IsLP64) {
- BuildMI(mallocMBB, MIMD, TII->get(X86::MOV64rr), X86::RDI)
- .addReg(sizeVReg);
+ BuildMI(mallocMBB, MIMD, TII->get(X86::MOV64rr), X86::RDI).addReg(sizeVReg);
BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32))
- .addExternalSymbol("__morestack_allocate_stack_space")
- .addRegMask(RegMask)
- .addReg(X86::RDI, RegState::Implicit)
- .addReg(X86::RAX, RegState::ImplicitDefine);
+ .addExternalSymbol("__morestack_allocate_stack_space")
+ .addRegMask(RegMask)
+ .addReg(X86::RDI, RegState::Implicit)
+ .addReg(X86::RAX, RegState::ImplicitDefine);
} else if (Is64Bit) {
- BuildMI(mallocMBB, MIMD, TII->get(X86::MOV32rr), X86::EDI)
- .addReg(sizeVReg);
+ BuildMI(mallocMBB, MIMD, TII->get(X86::MOV32rr), X86::EDI).addReg(sizeVReg);
BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32))
- .addExternalSymbol("__morestack_allocate_stack_space")
- .addRegMask(RegMask)
- .addReg(X86::EDI, RegState::Implicit)
- .addReg(X86::EAX, RegState::ImplicitDefine);
+ .addExternalSymbol("__morestack_allocate_stack_space")
+ .addRegMask(RegMask)
+ .addReg(X86::EDI, RegState::Implicit)
+ .addReg(X86::EAX, RegState::ImplicitDefine);
} else {
- BuildMI(mallocMBB, MIMD, TII->get(X86::SUB32ri), physSPReg).addReg(physSPReg)
- .addImm(12);
+ BuildMI(mallocMBB, MIMD, TII->get(X86::SUB32ri), physSPReg)
+ .addReg(physSPReg)
+ .addImm(12);
BuildMI(mallocMBB, MIMD, TII->get(X86::PUSH32r)).addReg(sizeVReg);
BuildMI(mallocMBB, MIMD, TII->get(X86::CALLpcrel32))
- .addExternalSymbol("__morestack_allocate_stack_space")
- .addRegMask(RegMask)
- .addReg(X86::EAX, RegState::ImplicitDefine);
+ .addExternalSymbol("__morestack_allocate_stack_space")
+ .addRegMask(RegMask)
+ .addReg(X86::EAX, RegState::ImplicitDefine);
}
if (!Is64Bit)
- BuildMI(mallocMBB, MIMD, TII->get(X86::ADD32ri), physSPReg).addReg(physSPReg)
- .addImm(16);
+ BuildMI(mallocMBB, MIMD, TII->get(X86::ADD32ri), physSPReg)
+ .addReg(physSPReg)
+ .addImm(16);
BuildMI(mallocMBB, MIMD, TII->get(TargetOpcode::COPY), mallocPtrVReg)
- .addReg(IsLP64 ? X86::RAX : X86::EAX);
+ .addReg(IsLP64 ? X86::RAX : X86::EAX);
BuildMI(mallocMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB);
// Set up the CFG correctly.
@@ -37647,7 +37672,8 @@ X86TargetLowering::EmitLoweredCatchRet(MachineInstr &MI,
RestoreMBB->setIsEHPad(true);
auto RestoreMBBI = RestoreMBB->begin();
- BuildMI(*RestoreMBB, RestoreMBBI, MIMD, TII.get(X86::JMP_4)).addMBB(TargetMBB);
+ BuildMI(*RestoreMBB, RestoreMBBI, MIMD, TII.get(X86::JMP_4))
+ .addMBB(TargetMBB);
return BB;
}
@@ -37669,9 +37695,10 @@ X86TargetLowering::EmitLoweredTLSCall(MachineInstr &MI,
// FIXME: The 32-bit calls have non-standard calling conventions. Use a
// proper register mask.
const uint32_t *RegMask =
- Subtarget.is64Bit() ?
- Subtarget.getRegisterInfo()->getDarwinTLSCallPreservedMask() :
- Subtarget.getRegisterInfo()->getCallPreservedMask(*F, CallingConv::C);
+ Subtarget.is64Bit()
+ ? Subtarget.getRegisterInfo()->getDarwinTLSCallPreservedMask()
+ : Subtarget.getRegisterInfo()->getCallPreservedMask(*F,
+ CallingConv::C);
if (Subtarget.is64Bit()) {
MachineInstrBuilder MIB =
BuildMI(*BB, MI, MIMD, TII->get(X86::MOV64rm), X86::RDI)
@@ -37927,8 +37954,7 @@ X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
MemOpndSlot = CurOp;
MVT PVT = getPointerTy(MF->getDataLayout());
- assert((PVT == MVT::i64 || PVT == MVT::i32) &&
- "Invalid Pointer Size!");
+ assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
// For v = setjmp(buf), we generate
//
@@ -37976,19 +38002,19 @@ X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
LabelReg = MRI.createVirtualRegister(PtrRC);
if (Subtarget.is64Bit()) {
MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA64r), LabelReg)
- .addReg(X86::RIP)
- .addImm(0)
- .addReg(0)
- .addMBB(restoreMBB)
- .addReg(0);
+ .addReg(X86::RIP)
+ .addImm(0)
+ .addReg(0)
+ .addMBB(restoreMBB)
+ .addReg(0);
} else {
- const X86InstrInfo *XII = static_cast<const X86InstrInfo*>(TII);
+ const X86InstrInfo *XII = static_cast<const X86InstrInfo *>(TII);
MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA32r), LabelReg)
- .addReg(XII->getGlobalBaseReg(MF))
- .addImm(0)
- .addReg(0)
- .addMBB(restoreMBB, Subtarget.classifyBlockAddressReference())
- .addReg(0);
+ .addReg(XII->getGlobalBaseReg(MF))
+ .addImm(0)
+ .addReg(0)
+ .addMBB(restoreMBB, Subtarget.classifyBlockAddressReference())
+ .addReg(0);
}
} else
PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
@@ -38012,7 +38038,7 @@ X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
// Setup
MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::EH_SjLj_Setup))
- .addMBB(restoreMBB);
+ .addMBB(restoreMBB);
const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
MIB.addRegMask(RegInfo->getNoPreservedMask());
@@ -38039,9 +38065,9 @@ X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
Register FramePtr = RegInfo->getFrameRegister(*MF);
Register BasePtr = RegInfo->getBaseRegister();
unsigned Opm = Uses64BitFramePtr ? X86::MOV64rm : X86::MOV32rm;
- addRegOffset(BuildMI(restoreMBB, MIMD, TII->get(Opm), BasePtr),
- FramePtr, true, X86FI->getRestoreBasePointerOffset())
- .setMIFlag(MachineInstr::FrameSetup);
+ addRegOffset(BuildMI(restoreMBB, MIMD, TII->get(Opm), BasePtr), FramePtr,
+ true, X86FI->getRestoreBasePointerOffset())
+ .setMIFlag(MachineInstr::FrameSetup);
}
BuildMI(restoreMBB, MIMD, TII->get(X86::MOV32ri), restoreDstReg).addImm(1);
BuildMI(restoreMBB, MIMD, TII->get(X86::JMP_1)).addMBB(sinkMBB);
@@ -38256,11 +38282,10 @@ X86TargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands());
MVT PVT = getPointerTy(MF->getDataLayout());
- assert((PVT == MVT::i64 || PVT == MVT::i32) &&
- "Invalid Pointer Size!");
+ assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
const TargetRegisterClass *RC =
- (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
+ (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
Register Tmp = MRI.createVirtualRegister(RC);
// Since FP is only updated here but NOT referenced, it's treated as GPR.
const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
@@ -38640,7 +38665,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
const MIMetadata MIMD(MI);
auto TMMImmToTMMReg = [](unsigned Imm) {
- assert (Imm < 8 && "Illegal tmm index");
+ assert(Imm < 8 && "Illegal tmm index");
return X86::TMM0 + Imm;
};
switch (MI.getOpcode()) {
@@ -38776,29 +38801,30 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
// OR 0b11 into bit 10 and 11. 0b11 is the encoding for round toward zero.
Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
BuildMI(*BB, MI, MIMD, TII->get(X86::OR32ri), NewCW)
- .addReg(OldCW, RegState::Kill).addImm(0xC00);
+ .addReg(OldCW, RegState::Kill)
+ .addImm(0xC00);
// Extract to 16 bits.
Register NewCW16 =
MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass);
BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), NewCW16)
- .addReg(NewCW, RegState::Kill, X86::sub_16bit);
+ .addReg(NewCW, RegState::Kill, X86::sub_16bit);
// Prepare memory for FLDCW.
int NewCWFrameIdx =
MF->getFrameInfo().CreateStackObject(2, Align(2), false);
addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOV16mr)),
NewCWFrameIdx)
- .addReg(NewCW16, RegState::Kill);
+ .addReg(NewCW16, RegState::Kill);
// Reload the modified control word now...
- addFrameReference(BuildMI(*BB, MI, MIMD,
- TII->get(X86::FLDCW16m)), NewCWFrameIdx);
+ addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
+ NewCWFrameIdx);
// Get the X86 opcode to use.
unsigned Opc;
switch (MI.getOpcode()) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("illegal opcode!");
case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break;
case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break;
@@ -38809,7 +38835,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break;
case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break;
case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break;
- // clang-format on
+ // clang-format on
}
X86AddressMode AM = getAddressFromInstr(&MI, 0);
@@ -39025,7 +39051,8 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case X86::PTDPHF8PS: {
unsigned Opc;
switch (MI.getOpcode()) {
- default: llvm_unreachable("illegal opcode!");
+ default:
+ llvm_unreachable("illegal opcode!");
// clang-format off
case X86::PTDPBSSD: Opc = X86::TDPBSSD; break;
case X86::PTDPBSUD: Opc = X86::TDPBSUD; break;
@@ -39071,7 +39098,8 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case X86::PTILESTORED: {
unsigned Opc;
switch (MI.getOpcode()) {
- default: llvm_unreachable("illegal opcode!");
+ default:
+ llvm_unreachable("illegal opcode!");
#define GET_EGPR_IF_ENABLED(OPC) (Subtarget.hasEGPR() ? OPC##_EVEX : OPC)
case X86::PTILELOADD:
Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
@@ -39193,11 +39221,9 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
// X86 Optimization Hooks
//===----------------------------------------------------------------------===//
-bool
-X86TargetLowering::targetShrinkDemandedConstant(SDValue Op,
- const APInt &DemandedBits,
- const APInt &DemandedElts,
- TargetLoweringOpt &TLO) const {
+bool X86TargetLowering::targetShrinkDemandedConstant(
+ SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
+ TargetLoweringOpt &TLO) const {
EVT VT = Op.getValueType();
unsigned Opcode = Op.getOpcode();
unsigned EltSize = VT.getScalarSizeInBits();
@@ -39382,16 +39408,15 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
unsigned NumElts = DemandedElts.getBitWidth();
unsigned Opc = Op.getOpcode();
EVT VT = Op.getValueType();
- assert((Opc >= ISD::BUILTIN_OP_END ||
- Opc == ISD::INTRINSIC_WO_CHAIN ||
- Opc == ISD::INTRINSIC_W_CHAIN ||
- Opc == ISD::INTRINSIC_VOID) &&
+ assert((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN ||
+ Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) &&
"Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!");
Known.resetAll();
switch (Opc) {
- default: break;
+ default:
+ break;
case X86ISD::MUL_IMM: {
KnownBits Known2;
Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
@@ -39630,7 +39655,7 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
SDValue Op0 = Op.getOperand(0);
SDValue Op1 = Op.getOperand(1);
- if (auto* Cst1 = dyn_cast<ConstantSDNode>(Op1)) {
+ if (auto *Cst1 = dyn_cast<ConstantSDNode>(Op1)) {
unsigned Shift = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 0);
unsigned Length = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 8);
@@ -39856,7 +39881,8 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
unsigned NumElts = VT.getVectorNumElements();
if (Mask.size() == NumElts) {
SmallVector<APInt, 2> DemandedOps(NumOps, APInt(NumElts, 0));
- Known.Zero.setAllBits(); Known.One.setAllBits();
+ Known.Zero.setAllBits();
+ Known.One.setAllBits();
for (unsigned i = 0; i != NumElts; ++i) {
if (!DemandedElts[i])
continue;
@@ -40001,16 +40027,18 @@ unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
case X86ISD::ANDNP: {
unsigned Tmp0 =
DAG.ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
- if (Tmp0 == 1) return 1; // Early out.
+ if (Tmp0 == 1)
+ return 1; // Early out.
unsigned Tmp1 =
DAG.ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
return std::min(Tmp0, Tmp1);
}
case X86ISD::CMOV: {
- unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth+1);
- if (Tmp0 == 1) return 1; // Early out.
- unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth+1);
+ unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
+ if (Tmp0 == 1)
+ return 1; // Early out.
+ unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth + 1);
return std::min(Tmp0, Tmp1);
}
}
@@ -40382,7 +40410,6 @@ static bool matchUnaryPermuteShuffle(MVT MaskVT, ArrayRef<int> Mask,
PermuteImm = (unsigned)ShiftAmt;
return true;
}
-
}
}
@@ -40442,7 +40469,8 @@ static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
// Attempt to match against either an unary or binary PACKSS/PACKUS shuffle.
if (((MaskVT == MVT::v8i16 || MaskVT == MVT::v16i8) && Subtarget.hasSSE2()) ||
- ((MaskVT == MVT::v16i16 || MaskVT == MVT::v32i8) && Subtarget.hasInt256()) ||
+ ((MaskVT == MVT::v16i16 || MaskVT == MVT::v32i8) &&
+ Subtarget.hasInt256()) ||
((MaskVT == MVT::v32i16 || MaskVT == MVT::v64i8) && Subtarget.hasBWI())) {
if (matchShuffleWithPACK(MaskVT, SrcVT, V1, V2, Shuffle, Mask, DAG,
Subtarget)) {
@@ -41015,9 +41043,9 @@ static SDValue combineX86ShuffleChain(
SDValue LHS = isInRange(Mask[0], 0, 2) ? V1 : V2;
SDValue RHS = isInRange(Mask[1], 0, 2) ? V1 : V2;
return DAG.getNode(X86ISD::VPERM2X128, DL, RootVT,
- CanonicalizeShuffleInput(RootVT, LHS),
- CanonicalizeShuffleInput(RootVT, RHS),
- DAG.getTargetConstant(PermMask, DL, MVT::i8));
+ CanonicalizeShuffleInput(RootVT, LHS),
+ CanonicalizeShuffleInput(RootVT, RHS),
+ DAG.getTargetConstant(PermMask, DL, MVT::i8));
}
}
}
@@ -41111,8 +41139,8 @@ static SDValue combineX86ShuffleChain(
}
if (matchUnaryPermuteShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
- AllowIntDomain, DAG, Subtarget, Shuffle, ShuffleVT,
- PermuteImm) &&
+ AllowIntDomain, DAG, Subtarget, Shuffle,
+ ShuffleVT, PermuteImm) &&
(!IsMaskedShuffle ||
(NumRootElts == ShuffleVT.getVectorNumElements()))) {
if (Depth == 0 && RootOpc == Shuffle)
@@ -41995,11 +42023,9 @@ static SDValue combineX86ShufflesConstants(MVT VT, ArrayRef<SDValue> Ops,
}
namespace llvm {
- namespace X86 {
- enum {
- MaxShuffleCombineDepth = 8
- };
- } // namespace X86
+namespace X86 {
+enum { MaxShuffleCombineDepth = 8 };
+} // namespace X86
} // namespace llvm
/// Fully generic combining of x86 shuffle instructions.
@@ -42403,7 +42429,8 @@ static SDValue combineX86ShufflesRecursively(
// The Op itself may be of different VT, so we need to scale the mask.
unsigned NumOpElts = Op.getValueType().getVectorNumElements();
- APInt OpScaledDemandedElts = APIntOps::ScaleBitMask(OpDemandedElts, NumOpElts);
+ APInt OpScaledDemandedElts =
+ APIntOps::ScaleBitMask(OpDemandedElts, NumOpElts);
// Can this operand be simplified any further, given it's demanded elements?
if (SDValue NewOp = TLI.SimplifyMultipleUseDemandedVectorElts(
@@ -43233,7 +43260,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
ISD::isNormalLoad(Src.getNode())) {
LoadSDNode *LN = cast<LoadSDNode>(Src);
SDVTList Tys = DAG.getVTList(VT, MVT::Other);
- SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
+ SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
SDValue BcastLd =
DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
LN->getMemoryVT(), LN->getMemOperand());
@@ -43265,7 +43292,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
// Unless its volatile or atomic.
if (LN->isSimple()) {
SDVTList Tys = DAG.getVTList(VT, MVT::Other);
- SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
+ SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
SDValue BcastLd = DAG.getMemIntrinsicNode(
X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
LN->getPointerInfo(), LN->getBaseAlign(),
@@ -43283,7 +43310,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
LoadSDNode *LN = cast<LoadSDNode>(Src.getOperand(0));
if (LN->getMemoryVT().getSizeInBits() == 16) {
SDVTList Tys = DAG.getVTList(VT, MVT::Other);
- SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
+ SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
SDValue BcastLd =
DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
LN->getMemoryVT(), LN->getMemOperand());
@@ -43310,7 +43337,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
SDVTList Tys = DAG.getVTList(VT, MVT::Other);
SDValue Ptr = DAG.getMemBasePlusOffset(
LN->getBasePtr(), TypeSize::getFixed(Offset), DL);
- SDValue Ops[] = { LN->getChain(), Ptr };
+ SDValue Ops[] = {LN->getChain(), Ptr};
SDValue BcastLd = DAG.getMemIntrinsicNode(
X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
LN->getPointerInfo().getWithOffset(Offset), LN->getBaseAlign(),
@@ -43328,7 +43355,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
MemSDNode *LN = cast<MemIntrinsicSDNode>(Src);
if (LN->getMemoryVT().getSizeInBits() == VT.getScalarSizeInBits()) {
SDVTList Tys = DAG.getVTList(VT, MVT::Other);
- SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
+ SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
SDValue BcastLd =
DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
LN->getMemoryVT(), LN->getMemOperand());
@@ -43889,13 +43916,13 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
if (Op1.getOpcode() == X86ISD::VBROADCAST_LOAD && Op1.hasOneUse()) {
auto *MemIntr = cast<MemIntrinsicSDNode>(Op1);
if (MemIntr->getMemoryVT().getScalarSizeInBits() == 32) {
- SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
- MemIntr->getBasePtr(),
- MemIntr->getMemOperand());
- SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0,
- DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT,
- Load),
- DAG.getTargetConstant(InsertPSMask & 0x3f, DL, MVT::i8));
+ SDValue Load =
+ DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
+ MemIntr->getBasePtr(), MemIntr->getMemOperand());
+ SDValue Insert = DAG.getNode(
+ X86ISD::INSERTPS, DL, VT, Op0,
+ DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Load),
+ DAG.getTargetConstant(InsertPSMask & 0x3f, DL, MVT::i8));
DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
return Insert;
}
@@ -44077,8 +44104,8 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
if (Mask[0] == Mask[1] && Mask[2] == Mask[3] &&
(V.getOpcode() == X86ISD::PSHUFLW ||
V.getOpcode() == X86ISD::PSHUFHW) &&
- V.getOpcode() != N.getOpcode() &&
- V.hasOneUse() && V.getOperand(0).hasOneUse()) {
+ V.getOpcode() != N.getOpcode() && V.hasOneUse() &&
+ V.getOperand(0).hasOneUse()) {
SDValue D = peekThroughOneUseBitcasts(V.getOperand(0));
if (D.getOpcode() == X86ISD::PSHUFD) {
SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
@@ -44152,11 +44179,11 @@ static bool isAddSubOrSubAddMask(ArrayRef<int> Mask, bool &Op0Even) {
/// operation. If true is returned then the operands of ADDSUB(SUBADD) operation
/// are written to the parameters \p Opnd0 and \p Opnd1.
///
-/// We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle nodes
-/// so it is easier to generically match. We also insert dummy vector shuffle
-/// nodes for the operands which explicitly discard the lanes which are unused
-/// by this operation to try to flow through the rest of the combiner
-/// the fact that they're unused.
+/// We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle
+/// nodes so it is easier to generically match. We also insert dummy vector
+/// shuffle nodes for the operands which explicitly discard the lanes which are
+/// unused by this operation to try to flow through the rest of the combiner the
+/// fact that they're unused.
static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
bool &IsSubAdd, bool &HasAllowContract) {
@@ -44191,13 +44218,15 @@ static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
// commute the FADD operands.
SDValue LHS, RHS;
if (V1.getOpcode() == ISD::FSUB) {
- LHS = V1->getOperand(0); RHS = V1->getOperand(1);
+ LHS = V1->getOperand(0);
+ RHS = V1->getOperand(1);
if ((V2->getOperand(0) != LHS || V2->getOperand(1) != RHS) &&
(V2->getOperand(0) != RHS || V2->getOperand(1) != LHS))
return false;
} else {
assert(V2.getOpcode() == ISD::FSUB && "Unexpected opcode");
- LHS = V2->getOperand(0); RHS = V2->getOperand(1);
+ LHS = V2->getOperand(0);
+ RHS = V2->getOperand(1);
if ((V1->getOperand(0) != LHS || V1->getOperand(1) != RHS) &&
(V1->getOperand(0) != RHS || V1->getOperand(1) != LHS))
return false;
@@ -44208,8 +44237,8 @@ static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
return false;
// It's a subadd if the vector in the even parity is an FADD.
- IsSubAdd = Op0Even ? V1->getOpcode() == ISD::FADD
- : V2->getOpcode() == ISD::FADD;
+ IsSubAdd =
+ Op0Even ? V1->getOpcode() == ISD::FADD : V2->getOpcode() == ISD::FADD;
HasAllowContract =
V1->getFlags().hasAllowContract() && V2->getFlags().hasAllowContract();
@@ -44498,7 +44527,8 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
// TODO: Multiply by zero.
- // If RHS/LHS elements are known zero then we don't need the LHS/RHS equivalent.
+ // If RHS/LHS elements are known zero then we don't need the LHS/RHS
+ // equivalent.
APInt DemandedLHSElts = DemandedSrcElts & ~RHSZero;
if (SimplifyDemandedVectorElts(LHS, DemandedLHSElts, LHSUndef, LHSZero, TLO,
Depth + 1))
@@ -45320,7 +45350,7 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
// For splats, unless we *only* demand the 0'th element,
// stop attempts at simplification here, we aren't going to improve things,
// this is better than any potential shuffle.
- if (!DemandedElts.isOne() && TLO.DAG.isSplatValue(Op, /*AllowUndefs*/false))
+ if (!DemandedElts.isOne() && TLO.DAG.isSplatValue(Op, /*AllowUndefs*/ false))
return false;
// Get target/faux shuffle mask.
@@ -45446,7 +45476,7 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
EVT VT = Op.getValueType();
unsigned BitWidth = OriginalDemandedBits.getBitWidth();
unsigned Opc = Op.getOpcode();
- switch(Opc) {
+ switch (Opc) {
case X86ISD::VTRUNC: {
KnownBits KnownOp;
SDValue Src = Op.getOperand(0);
@@ -45454,8 +45484,10 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
// Simplify the input, using demanded bit information.
APInt TruncMask = OriginalDemandedBits.zext(SrcVT.getScalarSizeInBits());
- APInt DemandedElts = OriginalDemandedElts.trunc(SrcVT.getVectorNumElements());
- if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, KnownOp, TLO, Depth + 1))
+ APInt DemandedElts =
+ OriginalDemandedElts.trunc(SrcVT.getVectorNumElements());
+ if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, KnownOp, TLO,
+ Depth + 1))
return true;
break;
}
@@ -45559,7 +45591,8 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
}
}
- // If we are only demanding sign bits then we can use the shift source directly.
+ // If we are only demanding sign bits then we can use the shift source
+ // directly.
unsigned NumSignBits =
TLO.DAG.ComputeNumSignBits(Op0, OriginalDemandedElts, Depth + 1);
unsigned UpperDemandedBits = BitWidth - OriginalDemandedBits.countr_zero();
@@ -45750,8 +45783,8 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
return true;
KnownBits KnownVec;
- if (SimplifyDemandedBits(Vec, DemandedVecBits, DemandedVecElts,
- KnownVec, TLO, Depth + 1))
+ if (SimplifyDemandedBits(Vec, DemandedVecBits, DemandedVecElts, KnownVec,
+ TLO, Depth + 1))
return true;
if (SDValue V = SimplifyMultipleUseDemandedBits(
@@ -46414,13 +46447,13 @@ static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
// Helper to flip between AND/OR/XOR opcodes and their X86ISD FP equivalents.
static unsigned getAltBitOpcode(unsigned Opcode) {
- switch(Opcode) {
- // clang-format off
+ switch (Opcode) {
+ // clang-format off
case ISD::AND: return X86ISD::FAND;
case ISD::OR: return X86ISD::FOR;
case ISD::XOR: return X86ISD::FXOR;
case X86ISD::ANDNP: return X86ISD::FANDN;
- // clang-format on
+ // clang-format on
}
llvm_unreachable("Unknown bitwise opcode");
}
@@ -46643,8 +46676,7 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
// Convert a vXi1 constant build vector to the same width scalar integer.
static SDValue combinevXi1ConstantToInteger(SDValue Op, SelectionDAG &DAG) {
EVT SrcVT = Op.getValueType();
- assert(SrcVT.getVectorElementType() == MVT::i1 &&
- "Expected a vXi1 vector");
+ assert(SrcVT.getVectorElementType() == MVT::i1 && "Expected a vXi1 vector");
assert(ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
"Expected a constant build vector");
@@ -46960,7 +46992,7 @@ static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
// and the vbroadcast_load are both integer or both fp. In some cases this
// will remove the bitcast entirely.
if (N0.getOpcode() == X86ISD::VBROADCAST_LOAD && N0.hasOneUse() &&
- VT.isFloatingPoint() != SrcVT.isFloatingPoint() && VT.isVector()) {
+ VT.isFloatingPoint() != SrcVT.isFloatingPoint() && VT.isVector()) {
auto *BCast = cast<MemIntrinsicSDNode>(N0);
unsigned SrcVTSize = SrcVT.getScalarSizeInBits();
unsigned MemSize = BCast->getMemoryVT().getScalarSizeInBits();
@@ -46973,7 +47005,7 @@ static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
LoadVT = MVT::getVectorVT(LoadVT, SrcVT.getVectorNumElements());
SDVTList Tys = DAG.getVTList(LoadVT, MVT::Other);
- SDValue Ops[] = { BCast->getChain(), BCast->getBasePtr() };
+ SDValue Ops[] = {BCast->getChain(), BCast->getBasePtr()};
SDValue ResNode =
DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, SDLoc(N), Tys, Ops,
MemVT, BCast->getMemOperand());
@@ -47023,7 +47055,7 @@ static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
bool LowUndef = true, AllUndefOrZero = true;
for (unsigned i = 1, e = SrcVT.getVectorNumElements(); i != e; ++i) {
SDValue Op = N0.getOperand(i);
- LowUndef &= Op.isUndef() || (i >= e/2);
+ LowUndef &= Op.isUndef() || (i >= e / 2);
AllUndefOrZero &= isNullConstantOrUndef(Op);
}
if (AllUndefOrZero) {
@@ -47138,12 +47170,12 @@ static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
// transferring the SSE operand to integer register and back.
unsigned FPOpcode;
switch (N0.getOpcode()) {
- // clang-format off
+ // clang-format off
case ISD::AND: FPOpcode = X86ISD::FAND; break;
case ISD::OR: FPOpcode = X86ISD::FOR; break;
case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
default: return SDValue();
- // clang-format on
+ // clang-format on
}
// Check if we have a bitcast from another integer type as well.
@@ -47244,7 +47276,7 @@ static SDValue createVPDPBUSD(SelectionDAG &DAG, SDValue LHS, SDValue RHS,
// Actually build the DotProduct, split as 256/512 bits for
// AVXVNNI/AVX512VNNI.
auto DpBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
- ArrayRef<SDValue> Ops) {
+ ArrayRef<SDValue> Ops) {
MVT VT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);
return DAG.getNode(X86ISD::VPDPBUSD, DL, VT, Ops);
};
@@ -47284,7 +47316,8 @@ static SDValue createPSADBW(SelectionDAG &DAG, SDValue N0, SDValue N1,
PSADBWBuilder);
}
-// Attempt to replace an all_of/any_of/parity style horizontal reduction with a MOVMSK.
+// Attempt to replace an all_of/any_of/parity style horizontal reduction with a
+// MOVMSK.
static SDValue combineVECREDUCE_LOGIC(SDNode *Reduce, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
// Bail without SSE2.
@@ -47559,9 +47592,9 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
if (Stages > 3) {
unsigned SadElems = SadVT.getVectorNumElements();
- for(unsigned i = Stages - 3; i > 0; --i) {
+ for (unsigned i = Stages - 3; i > 0; --i) {
SmallVector<int, 16> Mask(SadElems, -1);
- for(unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
+ for (unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
Mask[j] = MaskEnd + j;
SDValue Shuffle =
@@ -47877,10 +47910,10 @@ static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG,
SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
Vec.getOperand(0).getValueType().getScalarType(),
Vec.getOperand(0), Index);
- SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
- Vec.getOperand(1), Index);
- SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
- Vec.getOperand(2), Index);
+ SDValue Ext1 =
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Vec.getOperand(1), Index);
+ SDValue Ext2 =
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Vec.getOperand(2), Index);
return DAG.getNode(ISD::SELECT, DL, VT, Ext0, Ext1, Ext2);
}
@@ -48130,8 +48163,9 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
return DAG.getConstant(EltBits[Idx].zext(NumEltBits), dl, VT);
}
- // Convert extract_element(bitcast(<X x i1>) -> bitcast(extract_subvector()).
- // Improves lowering of bool masks on rust which splits them into byte array.
+ // Convert extract_element(bitcast(<X x i1>) ->
+ // bitcast(extract_subvector()). Improves lowering of bool masks on rust
+ // which splits them into byte array.
if (InputVector.getOpcode() == ISD::BITCAST && (NumEltBits % 8) == 0) {
SDValue Src = peekThroughBitcasts(InputVector);
if (Src.getValueType().getScalarType() == MVT::i1 &&
@@ -48533,8 +48567,7 @@ static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG,
TargetLowering::DAGCombinerInfo &DCI,
const X86Subtarget &Subtarget) {
SDValue Cond = N->getOperand(0);
- if ((N->getOpcode() != ISD::VSELECT &&
- N->getOpcode() != X86ISD::BLENDV) ||
+ if ((N->getOpcode() != ISD::VSELECT && N->getOpcode() != X86ISD::BLENDV) ||
ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
return SDValue();
@@ -48969,13 +49002,13 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
Cond1 == InnerSetCC.getOperand(1)) {
ISD::CondCode NewCC;
switch (CC == ISD::SETEQ ? InnerCC : CC) {
- // clang-format off
+ // clang-format off
case ISD::SETGT: NewCC = ISD::SETGE; break;
case ISD::SETLT: NewCC = ISD::SETLE; break;
case ISD::SETUGT: NewCC = ISD::SETUGE; break;
case ISD::SETULT: NewCC = ISD::SETULE; break;
default: NewCC = ISD::SETCC_INVALID; break;
- // clang-format on
+ // clang-format on
}
if (NewCC != ISD::SETCC_INVALID) {
Cond = DAG.getSetCC(DL, CondVT, Cond0, Cond1, NewCC);
@@ -49149,9 +49182,9 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
// 16-bit lacks a proper blendv.
// TODO: Can we merge this into the LowerVSETCC handling of uniform masks?
bool CanShiftBlend =
- TLI.isTypeLegal(VT) && ((Subtarget.hasAVX() && EltBitWidth == 32) ||
- (Subtarget.hasAVX2() && EltBitWidth == 64) ||
- (Subtarget.hasXOP()));
+ TLI.isTypeLegal(VT) &&
+ ((Subtarget.hasAVX() && EltBitWidth == 32) ||
+ (Subtarget.hasAVX2() && EltBitWidth == 64) || (Subtarget.hasXOP()));
if (CanShiftBlend && DAG.isKnownToBeAPowerOfTwo(Mask)) {
// Create a left-shift constant to get the mask bits over to the sign-bit.
// vsel ((X & C) == 0), LHS, RHS --> vsel ((shl X, C') < 0), RHS, LHS
@@ -49387,7 +49420,7 @@ static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
SDValue Op2 = Cmp.getOperand(1);
SDValue SetCC;
- const ConstantSDNode* C = nullptr;
+ const ConstantSDNode *C = nullptr;
bool needOppositeCond = (CC == X86::COND_E);
bool checkAgainstTrue = false; // Is it a comparison against 1?
@@ -49408,8 +49441,7 @@ static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
bool truncatedToBoolWithAnd = false;
// Skip (zext $x), (trunc $x), or (and $x, 1) node.
while (SetCC.getOpcode() == ISD::ZERO_EXTEND ||
- SetCC.getOpcode() == ISD::TRUNCATE ||
- SetCC.getOpcode() == ISD::AND) {
+ SetCC.getOpcode() == ISD::TRUNCATE || SetCC.getOpcode() == ISD::AND) {
if (SetCC.getOpcode() == ISD::AND) {
int OpIdx = -1;
if (isOneConstant(SetCC.getOperand(0)))
@@ -49452,13 +49484,13 @@ static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
if (!FVal) {
SDValue Op = SetCC.getOperand(0);
// Skip 'zext' or 'trunc' node.
- if (Op.getOpcode() == ISD::ZERO_EXTEND ||
- Op.getOpcode() == ISD::TRUNCATE)
+ if (Op.getOpcode() == ISD::ZERO_EXTEND || Op.getOpcode() == ISD::TRUNCATE)
Op = Op.getOperand(0);
// A special case for rdrand/rdseed, where 0 is set if false cond is
// found.
if ((Op.getOpcode() != X86ISD::RDRAND &&
- Op.getOpcode() != X86ISD::RDSEED) || Op.getResNo() != 0)
+ Op.getOpcode() != X86ISD::RDSEED) ||
+ Op.getResNo() != 0)
return SDValue();
}
// Quit if false value is not the constant 0 or 1.
@@ -49503,7 +49535,8 @@ static bool checkBoolTestAndOrSetCCCombine(SDValue Cond, X86::CondCode &CC0,
SDValue SetCC0, SetCC1;
switch (Cond->getOpcode()) {
- default: return false;
+ default:
+ return false;
case ISD::AND:
case X86ISD::AND:
isAnd = true;
@@ -49568,8 +49601,7 @@ static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) {
}
// If this is a check of the z flag of an add with 1, switch to the
// C flag.
- if (CarryCC == X86::COND_E &&
- CarryOp1.getOpcode() == X86ISD::ADD &&
+ if (CarryCC == X86::COND_E && CarryOp1.getOpcode() == X86ISD::ADD &&
isOneConstant(CarryOp1.getOperand(1)))
return CarryOp1;
} else if (FoundAndLSB) {
@@ -50104,12 +50136,11 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
// Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. This is efficient
// for any integer data type, including i8/i16.
- if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
+ if (FalseC->getAPIntValue() + 1 == TrueC->getAPIntValue()) {
Cond = getSETCC(CC, Cond, DL, DAG);
// Zero extend the condition if needed.
- Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
- FalseC->getValueType(0), Cond);
+ Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), Cond);
Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
SDValue(FalseC, 0));
return Cond;
@@ -50125,24 +50156,25 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
bool isFastMultiplier = false;
if (Diff.ult(10)) {
switch (Diff.getZExtValue()) {
- default: break;
- case 1: // result = add base, cond
- case 2: // result = lea base( , cond*2)
- case 3: // result = lea base(cond, cond*2)
- case 4: // result = lea base( , cond*4)
- case 5: // result = lea base(cond, cond*4)
- case 8: // result = lea base( , cond*8)
- case 9: // result = lea base(cond, cond*8)
+ default:
+ break;
+ case 1: // result = add base, cond
+ case 2: // result = lea base( , cond*2)
+ case 3: // result = lea base(cond, cond*2)
+ case 4: // result = lea base( , cond*4)
+ case 5: // result = lea base(cond, cond*4)
+ case 8: // result = lea base( , cond*8)
+ case 9: // result = lea base(cond, cond*8)
isFastMultiplier = true;
break;
}
}
if (isFastMultiplier) {
- Cond = getSETCC(CC, Cond, DL ,DAG);
+ Cond = getSETCC(CC, Cond, DL, DAG);
// Zero extend the condition if needed.
- Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
- Cond);
+ Cond =
+ DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), Cond);
// Scale the condition by the difference.
if (Diff != 1)
Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
@@ -50720,7 +50752,6 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
EVT VT = N->getValueType(0);
SDLoc DL(N);
-
if (!Subtarget.hasSSE2())
return SDValue();
@@ -50736,16 +50767,14 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
bool IsSigned = Opc == ISD::SDIV || Opc == ISD::SREM;
// Scalar functions that do not use XMM registers must not get this combine.
- if (!VT.isVector() &&
- (Subtarget.useSoftFloat() ||
- DAG.getMachineFunction().getFunction().hasFnAttribute(
- Attribute::NoImplicitFloat)))
+ if (!VT.isVector() && (Subtarget.useSoftFloat() ||
+ DAG.getMachineFunction().getFunction().hasFnAttribute(
+ Attribute::NoImplicitFloat)))
return SDValue();
// If the result is only read back as scalar extracts, scalarization computes
// just the demanded lanes.
- if (VT.isVector() &&
- all_of(N->users(), [](const SDNode *U) {
+ if (VT.isVector() && all_of(N->users(), [](const SDNode *U) {
return U->getOpcode() == ISD::EXTRACT_VECTOR_ELT;
}))
return SDValue();
@@ -50795,11 +50824,11 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
// Nothing will split an illegal FP type after type legalization and the
// strict SAE divide is 512-bit only. Scalar f32/f64 are always legal.
- bool FPVTUsable = !VT.isVector() ||
- (IsStrict
- ? FPVT.getSizeInBits() <= 512
- : DCI.isBeforeLegalize() ||
- DAG.getTargetLoweringInfo().isTypeLegal(FPVT));
+ bool FPVTUsable =
+ !VT.isVector() ||
+ (IsStrict ? FPVT.getSizeInBits() <= 512
+ : DCI.isBeforeLegalize() ||
+ DAG.getTargetLoweringInfo().isTypeLegal(FPVT));
// Halve the divide while the integer halves stay legal.
if (!FPVTUsable) {
@@ -51102,8 +51131,7 @@ static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG,
// fold (shl (and (setcc_c), c1), c2) -> (and setcc_c, (c1 << c2))
// since the result of setcc_c is all zero's or all ones.
- if (VT.isInteger() && !VT.isVector() &&
- N1C && N0.getOpcode() == ISD::AND &&
+ if (VT.isInteger() && !VT.isVector() && N1C && N0.getOpcode() == ISD::AND &&
N0.getOperand(1).getOpcode() == ISD::Constant) {
SDValue N00 = N0.getOperand(0);
APInt Mask = N0.getConstantOperandAPInt(1);
@@ -51187,7 +51215,7 @@ static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG,
if (SraConst.isNegative())
return SDValue();
- for (MVT SVT : { MVT::i8, MVT::i16, MVT::i32 }) {
+ for (MVT SVT : {MVT::i8, MVT::i16, MVT::i32}) {
unsigned ShiftSize = SVT.getSizeInBits();
// Only deal with (Size - ShlConst) being equal to 8, 16 or 32.
if (ShiftSize >= Size || ShlConst != Size - ShiftSize)
@@ -51550,8 +51578,8 @@ static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
// Try to combine a PACKUSWB/PACKSSWB implemented truncate with a regular
// truncate to create a larger truncate.
- if (Subtarget.hasAVX512() &&
- N0.getOpcode() == ISD::TRUNCATE && N1.isUndef() && VT == MVT::v16i8 &&
+ if (Subtarget.hasAVX512() && N0.getOpcode() == ISD::TRUNCATE &&
+ N1.isUndef() && VT == MVT::v16i8 &&
N0.getOperand(0).getValueType() == MVT::v8i32) {
if ((IsSigned && DAG.ComputeNumSignBits(N0) > 8) ||
(!IsSigned &&
@@ -51936,7 +51964,7 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
SDValue CMP00 = CMP0->getOperand(0);
SDValue CMP01 = CMP0->getOperand(1);
- EVT VT = CMP00.getValueType();
+ EVT VT = CMP00.getValueType();
if (VT == MVT::f32 || VT == MVT::f64 ||
(VT == MVT::f16 && Subtarget.hasFP16())) {
@@ -51962,8 +51990,10 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
}
if (!ExpectingFlags) {
- enum X86::CondCode cc0 = (enum X86::CondCode)N0.getConstantOperandVal(0);
- enum X86::CondCode cc1 = (enum X86::CondCode)N1.getConstantOperandVal(0);
+ enum X86::CondCode cc0 =
+ (enum X86::CondCode)N0.getConstantOperandVal(0);
+ enum X86::CondCode cc1 =
+ (enum X86::CondCode)N1.getConstantOperandVal(0);
if (cc1 == X86::COND_E || cc1 == X86::COND_NE) {
X86::CondCode tmp = cc0;
@@ -51971,7 +52001,7 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
cc1 = tmp;
}
- if ((cc0 == X86::COND_E && cc1 == X86::COND_NP) ||
+ if ((cc0 == X86::COND_E && cc1 == X86::COND_NP) ||
(cc0 == X86::COND_NE && cc1 == X86::COND_P)) {
// FIXME: need symbolic constants for these magic numbers.
// See X86ATTInstPrinter.cpp:printSSECC().
@@ -51981,7 +52011,8 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01,
DAG.getTargetConstant(x86cc, DL, MVT::i8));
// Need to fill with zeros to ensure the bitcast will produce zeroes
- // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
+ // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee
+ // that.
SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1,
DAG.getConstant(0, DL, MVT::v16i1),
FSetCC, DAG.getVectorIdxConstant(0, DL));
@@ -52013,8 +52044,8 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
SDValue OnesOrZeroesI = DAG.getBitcast(IntVT, OnesOrZeroesF);
SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
DAG.getConstant(1, DL, IntVT));
- SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
- ANDed);
+ SDValue OneBitOfTruth =
+ DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ANDed);
return OneBitOfTruth;
}
}
@@ -52209,7 +52240,8 @@ static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL,
assert(VT.isVector() && "Expected vector type");
assert((N.getOpcode() == ISD::ANY_EXTEND ||
N.getOpcode() == ISD::ZERO_EXTEND ||
- N.getOpcode() == ISD::SIGN_EXTEND) && "Invalid Node");
+ N.getOpcode() == ISD::SIGN_EXTEND) &&
+ "Invalid Node");
SDValue Narrow = N.getOperand(0);
EVT NarrowVT = Narrow.getValueType();
@@ -52219,26 +52251,27 @@ static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL,
if (!Op)
return SDValue();
switch (N.getOpcode()) {
- default: llvm_unreachable("Unexpected opcode");
+ default:
+ llvm_unreachable("Unexpected opcode");
case ISD::ANY_EXTEND:
return Op;
case ISD::ZERO_EXTEND:
return DAG.getZeroExtendInReg(Op, DL, NarrowVT);
case ISD::SIGN_EXTEND:
- return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,
- Op, DAG.getValueType(NarrowVT));
+ return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op,
+ DAG.getValueType(NarrowVT));
}
}
static unsigned convertIntLogicToFPLogicOpcode(unsigned Opcode) {
unsigned FPOpcode;
switch (Opcode) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected input node for FP logic conversion");
case ISD::AND: FPOpcode = X86ISD::FAND; break;
case ISD::OR: FPOpcode = X86ISD::FOR; break;
case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
- // clang-format on
+ // clang-format on
}
return FPOpcode;
}
@@ -52715,8 +52748,7 @@ static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG,
SmallVector<SDValue, 4> Ops(Src.getNumOperands(),
DAG.getConstant(0, dl, SubVecVT));
Ops[0] = SubVec;
- SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT,
- Ops);
+ SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT, Ops);
EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcVT.getSizeInBits());
return DAG.getZExtOrTrunc(DAG.getBitcast(IntVT, Concat), dl, VT);
}
@@ -53068,7 +53100,7 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
return R;
- if (SDValue R = combineAndNotIntoANDNP(N, dl ,DAG))
+ if (SDValue R = combineAndNotIntoANDNP(N, dl, DAG))
return R;
if (SDValue ShiftRight = combineAndMaskToShift(N, dl, DAG, Subtarget))
@@ -53914,7 +53946,8 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
if (NotCond) {
SDValue R = DAG.getZExtOrTrunc(NotCond, dl, VT);
- R = DAG.getNode(ISD::MUL, dl, VT, R, DAG.getConstant(Val + 1, dl, VT));
+ R = DAG.getNode(ISD::MUL, dl, VT, R,
+ DAG.getConstant(Val + 1, dl, VT));
R = DAG.getNode(ISD::SUB, dl, VT, R, DAG.getConstant(1, dl, VT));
return R;
}
@@ -54061,7 +54094,7 @@ static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
return SDValue();
switch (VT.getSimpleVT().SimpleTy) {
- // clang-format off
+ // clang-format off
default: return SDValue();
case MVT::v16i8:
case MVT::v8i16:
@@ -54191,8 +54224,8 @@ static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
// split across two registers. We can use a packusdw+perm to clamp to 0-65535
// and concatenate at the same time. Then we can use a final vpmovuswb to
// clip to 0-255.
- if (Subtarget.hasBWI() && !Subtarget.useAVX512Regs() &&
- InVT == MVT::v16i32 && VT == MVT::v16i8) {
+ if (Subtarget.hasBWI() && !Subtarget.useAVX512Regs() && InVT == MVT::v16i32 &&
+ VT == MVT::v16i8) {
if (SDValue USatVal = detectSSatPattern(In, VT, true)) {
// Emit a VPACKUSDW+VPERMQ followed by a VPMOVUSWB.
SDValue Mid = truncateVectorWithPACK(X86ISD::PACKUS, MVT::v16i16, USatVal,
@@ -54208,11 +54241,12 @@ static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
// FIXME: We could widen truncates to 512 to remove the VLX restriction.
// If the result type is 256-bits or larger and we have disable 512-bit
// registers, we should go ahead and use the pack instructions if possible.
- bool PreferAVX512 = ((Subtarget.hasAVX512() && InSVT == MVT::i32) ||
- (Subtarget.hasBWI() && InSVT == MVT::i16)) &&
- (InVT.getSizeInBits() > 128) &&
- (Subtarget.hasVLX() || InVT.getSizeInBits() > 256) &&
- !(!Subtarget.useAVX512Regs() && VT.getSizeInBits() >= 256);
+ bool PreferAVX512 =
+ ((Subtarget.hasAVX512() && InSVT == MVT::i32) ||
+ (Subtarget.hasBWI() && InSVT == MVT::i16)) &&
+ (InVT.getSizeInBits() > 128) &&
+ (Subtarget.hasVLX() || InVT.getSizeInBits() > 256) &&
+ !(!Subtarget.useAVX512Regs() && VT.getSizeInBits() >= 256);
if (!PreferAVX512 && VT.getVectorNumElements() > 1 &&
isPowerOf2_32(VT.getVectorNumElements()) &&
@@ -54225,8 +54259,8 @@ static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
SDValue Mid = truncateVectorWithPACK(X86ISD::PACKSS, MidVT, USatVal, DL,
DAG, Subtarget);
assert(Mid && "Failed to pack!");
- SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG,
- Subtarget);
+ SDValue V =
+ truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG, Subtarget);
assert(V && "Failed to pack!");
return V;
} else if (SVT == MVT::i8 || Subtarget.hasSSE41())
@@ -54593,10 +54627,9 @@ reduceMaskedLoadToScalarLoad(MaskedLoadSDNode *ML, SelectionDAG &DAG,
CastVT = VT.changeVectorElementType(*DAG.getContext(), EltVT);
}
- SDValue Load =
- DAG.getLoad(EltVT, DL, ML->getChain(), Addr,
- ML->getPointerInfo().getWithOffset(Offset),
- Alignment, ML->getMemOperand()->getFlags());
+ SDValue Load = DAG.getLoad(EltVT, DL, ML->getChain(), Addr,
+ ML->getPointerInfo().getWithOffset(Offset),
+ Alignment, ML->getMemOperand()->getFlags());
SDValue PassThru = DAG.getBitcast(CastVT, ML->getPassThru());
@@ -54627,8 +54660,8 @@ combineMaskedLoadConstantMask(MaskedLoadSDNode *ML, SelectionDAG &DAG,
if (LoadFirstElt && LoadLastElt) {
SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(),
ML->getMemOperand());
- SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd,
- ML->getPassThru());
+ SDValue Blend =
+ DAG.getSelect(DL, VT, ML->getMask(), VecLd, ML->getPassThru());
return DCI.CombineTo(ML, Blend, VecLd.getValue(1), true);
}
@@ -54650,8 +54683,8 @@ combineMaskedLoadConstantMask(MaskedLoadSDNode *ML, SelectionDAG &DAG,
VT, DL, ML->getChain(), ML->getBasePtr(), ML->getOffset(), ML->getMask(),
DAG.getUNDEF(VT), ML->getMemoryVT(), ML->getMemOperand(),
ML->getAddressingMode(), ML->getExtensionType());
- SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML,
- ML->getPassThru());
+ SDValue Blend =
+ DAG.getSelect(DL, VT, ML->getMask(), NewML, ML->getPassThru());
return DCI.CombineTo(ML, Blend, NewML.getValue(1), true);
}
@@ -54750,8 +54783,8 @@ static SDValue reduceMaskedStoreToScalarStore(MaskedStoreSDNode *MS,
// Store that element at the appropriate offset from the base pointer.
return DAG.getStore(MS->getChain(), DL, Extract, Addr,
- MS->getPointerInfo().getWithOffset(Offset),
- Alignment, MS->getMemOperand()->getFlags());
+ MS->getPointerInfo().getWithOffset(Offset), Alignment,
+ MS->getMemOperand()->getFlags());
}
static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
@@ -55107,15 +55140,16 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
// Turn vXi1 stores of constants into a scalar store.
if ((VT == MVT::v8i1 || VT == MVT::v16i1 || VT == MVT::v32i1 ||
- VT == MVT::v64i1) && VT == StVT && TLI.isTypeLegal(VT) &&
+ VT == MVT::v64i1) &&
+ VT == StVT && TLI.isTypeLegal(VT) &&
ISD::isBuildVectorOfConstantSDNodes(StoredVal.getNode())) {
// If its a v64i1 store without 64-bit support, we need two stores.
if (!DCI.isBeforeLegalize() && VT == MVT::v64i1 && !Subtarget.is64Bit()) {
- SDValue Lo = DAG.getBuildVector(MVT::v32i1, dl,
- StoredVal->ops().slice(0, 32));
+ SDValue Lo =
+ DAG.getBuildVector(MVT::v32i1, dl, StoredVal->ops().slice(0, 32));
Lo = combinevXi1ConstantToInteger(Lo, DAG);
- SDValue Hi = DAG.getBuildVector(MVT::v32i1, dl,
- StoredVal->ops().slice(32, 32));
+ SDValue Hi =
+ DAG.getBuildVector(MVT::v32i1, dl, StoredVal->ops().slice(32, 32));
Hi = combinevXi1ConstantToInteger(Hi, DAG);
SDValue Ptr0 = St->getBasePtr();
@@ -55265,14 +55299,14 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
if (TLI.isTruncStoreLegal(VT, StVT, St->getAlign(),
St->getAddressSpace())) {
if (SDValue Val = detectSSatPattern(St->getValue(), St->getMemoryVT()))
- return EmitTruncSStore(true /* Signed saturation */, St->getChain(),
- dl, Val, St->getBasePtr(),
- St->getMemoryVT(), St->getMemOperand(), DAG);
- if (SDValue Val = detectUSatPattern(St->getValue(), St->getMemoryVT(),
- DAG, dl))
+ return EmitTruncSStore(true /* Signed saturation */, St->getChain(), dl,
+ Val, St->getBasePtr(), St->getMemoryVT(),
+ St->getMemOperand(), DAG);
+ if (SDValue Val =
+ detectUSatPattern(St->getValue(), St->getMemoryVT(), DAG, dl))
return EmitTruncSStore(false /* Unsigned saturation */, St->getChain(),
- dl, Val, St->getBasePtr(),
- St->getMemoryVT(), St->getMemOperand(), DAG);
+ dl, Val, St->getBasePtr(), St->getMemoryVT(),
+ St->getMemOperand(), DAG);
}
return SDValue();
@@ -56110,8 +56144,7 @@ static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL,
// (i16 (ssat (add (mul (zext (even elts (i8 A))), (sext (even elts (i8 B)))),
// (mul (zext (odd elts (i8 A)), (sext (odd elts (i8 B))))))))
static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
- const X86Subtarget &Subtarget,
- const SDLoc &DL) {
+ const X86Subtarget &Subtarget, const SDLoc &DL) {
using namespace SDPatternMatch;
if (!VT.isVector() || !Subtarget.hasSSSE3())
return SDValue();
@@ -56185,8 +56218,8 @@ static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
std::swap(IdxN01, IdxN11);
}
// N0 indices be the even element. N1 indices must be the next odd element.
- if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 ||
- IdxN01 != 2 * i || IdxN11 != 2 * i + 1)
+ if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 || IdxN01 != 2 * i ||
+ IdxN11 != 2 * i + 1)
return SDValue();
SDValue N00In = N00Elt.getOperand(0);
SDValue N01In = N01Elt.getOperand(0);
@@ -56197,8 +56230,8 @@ static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
ZExtIn = N00In;
SExtIn = N01In;
}
- if (ZExtIn != N00In || SExtIn != N01In ||
- ZExtIn != N10In || SExtIn != N11In)
+ if (ZExtIn != N00In || SExtIn != N01In || ZExtIn != N10In ||
+ SExtIn != N11In)
return SDValue();
}
@@ -56218,14 +56251,13 @@ static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
// Shrink by adding truncate nodes and let DAGCombine fold with the
// sources.
EVT InVT = Ops[0].getValueType();
- assert(InVT.getScalarType() == MVT::i8 &&
- "Unexpected scalar element type");
+ assert(InVT.getScalarType() == MVT::i8 && "Unexpected scalar element type");
assert(InVT == Ops[1].getValueType() && "Operands' types mismatch");
EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
InVT.getVectorNumElements() / 2);
return DAG.getNode(X86ISD::VPMADDUBSW, DL, ResVT, Ops[0], Ops[1]);
};
- return SplitOpsAndApply(DAG, Subtarget, DL, VT, { ZExtIn, SExtIn },
+ return SplitOpsAndApply(DAG, Subtarget, DL, VT, {ZExtIn, SExtIn},
PMADDBuilder);
}
@@ -56473,7 +56505,7 @@ static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc,
bool NegRes) {
if (NegMul) {
switch (Opcode) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected opcode");
case ISD::FMA: Opcode = X86ISD::FNMADD; break;
case ISD::STRICT_FMA: Opcode = X86ISD::STRICT_FNMADD; break;
@@ -56487,13 +56519,13 @@ static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc,
case X86ISD::FNMSUB: Opcode = X86ISD::FMSUB; break;
case X86ISD::STRICT_FNMSUB: Opcode = X86ISD::STRICT_FMSUB; break;
case X86ISD::FNMSUB_RND: Opcode = X86ISD::FMSUB_RND; break;
- // clang-format on
+ // clang-format on
}
}
if (NegAcc) {
switch (Opcode) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected opcode");
case ISD::FMA: Opcode = X86ISD::FMSUB; break;
case ISD::STRICT_FMA: Opcode = X86ISD::STRICT_FMSUB; break;
@@ -56511,7 +56543,7 @@ static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc,
case X86ISD::FMADDSUB_RND: Opcode = X86ISD::FMSUBADD_RND; break;
case X86ISD::FMSUBADD: Opcode = X86ISD::FMADDSUB; break;
case X86ISD::FMSUBADD_RND: Opcode = X86ISD::FMADDSUB_RND; break;
- // clang-format on
+ // clang-format on
}
}
@@ -56528,7 +56560,7 @@ static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc,
case X86ISD::FNMADD_RND: Opcode = X86ISD::FMSUB_RND; break;
case X86ISD::FNMSUB: Opcode = ISD::FMA; break;
case X86ISD::FNMSUB_RND: Opcode = X86ISD::FMADD_RND; break;
- // clang-format on
+ // clang-format on
}
}
@@ -56672,13 +56704,13 @@ static SDValue combineX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
Op1 = DAG.getBitcast(IntVT, Op1);
unsigned IntOpcode;
switch (N->getOpcode()) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected FP logic op");
case X86ISD::FOR: IntOpcode = ISD::OR; break;
case X86ISD::FXOR: IntOpcode = ISD::XOR; break;
case X86ISD::FAND: IntOpcode = ISD::AND; break;
case X86ISD::FANDN: IntOpcode = X86ISD::ANDNP; break;
- // clang-format on
+ // clang-format on
}
SDValue IntOp = DAG.getNode(IntOpcode, dl, IntVT, Op0, Op1);
return DAG.getBitcast(VT, IntOp);
@@ -57118,17 +57150,18 @@ static SDValue combineFMinFMax(SDNode *N, SelectionDAG &DAG) {
// into FMINC and FMAXC, which are Commutative operations.
unsigned NewOp = 0;
switch (N->getOpcode()) {
- default: llvm_unreachable("unknown opcode");
- case ISD::PSEUDO_FMIN:
- NewOp = X86ISD::FMINC;
- break;
- case ISD::PSEUDO_FMAX:
- NewOp = X86ISD::FMAXC;
- break;
- }
+ default:
+ llvm_unreachable("unknown opcode");
+ case ISD::PSEUDO_FMIN:
+ NewOp = X86ISD::FMINC;
+ break;
+ case ISD::PSEUDO_FMAX:
+ NewOp = X86ISD::FMAXC;
+ break;
+ }
- return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0),
- N->getOperand(0), N->getOperand(1));
+ return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0), N->getOperand(0),
+ N->getOperand(1));
}
static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG,
@@ -57175,8 +57208,8 @@ static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG,
if (!VT.isVector() && DAG.getMachineFunction().getFunction().hasMinSize())
return SDValue();
- EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
- VT);
+ EVT SetCCType =
+ TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
// There are 4 possibilities involving NaN inputs, and these are the required
// outputs:
@@ -57226,8 +57259,8 @@ static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG,
MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);
if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
SDLoc dl(N);
- SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
- DAG.getBitcast(InVT, VZLoad));
+ SDValue Convert =
+ DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(InVT, VZLoad));
DCI.CombineTo(N, Convert);
DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
DCI.recursivelyDeleteUnusedNodes(LN);
@@ -57820,8 +57853,8 @@ static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
// Only combine legal element types.
EVT SVT = VT.getVectorElementType();
- if (SVT != MVT::i8 && SVT != MVT::i16 && SVT != MVT::i32 &&
- SVT != MVT::i64 && SVT != MVT::f32 && SVT != MVT::f64)
+ if (SVT != MVT::i8 && SVT != MVT::i16 && SVT != MVT::i32 && SVT != MVT::i64 &&
+ SVT != MVT::f32 && SVT != MVT::f64)
return SDValue();
// We don't have CMPP Instruction for vxf16
@@ -57861,16 +57894,15 @@ static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
SDLoc DL(N);
// (i32 (sext (i8 (x86isd::setcc_carry)))) -> (i32 (x86isd::setcc_carry))
- if (!DCI.isBeforeLegalizeOps() &&
- N0.getOpcode() == X86ISD::SETCC_CARRY) {
+ if (!DCI.isBeforeLegalizeOps() && N0.getOpcode() == X86ISD::SETCC_CARRY) {
SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0),
- N0->getOperand(1));
+ N0->getOperand(1));
bool ReplaceOtherUses = !N0.hasOneUse();
DCI.CombineTo(N, Setcc);
// Replace other uses with a truncate of the widened setcc_carry.
if (ReplaceOtherUses) {
- SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
- N0.getValueType(), Setcc);
+ SDValue Trunc =
+ DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), Setcc);
DCI.CombineTo(N0.getNode(), Trunc);
}
@@ -58163,13 +58195,13 @@ static SDValue combineZext(SDNode *N, SelectionDAG &DAG,
if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ANY_EXTEND &&
N0.getOpcode() == X86ISD::SETCC_CARRY) {
SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, N0->getOperand(0),
- N0->getOperand(1));
+ N0->getOperand(1));
bool ReplaceOtherUses = !N0.hasOneUse();
DCI.CombineTo(N, Setcc);
// Replace other uses with a truncate of the widened setcc_carry.
if (ReplaceOtherUses) {
- SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
- N0.getValueType(), Setcc);
+ SDValue Trunc =
+ DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), Setcc);
DCI.CombineTo(N0.getNode(), Trunc);
}
@@ -59010,8 +59042,8 @@ static SDValue combineVectorCompareAndMaskUnaryOp(SDNode *N,
SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
// The AND node needs bitcasts to/from an integer vector type around it.
SDValue MaskConst = DAG.getBitcast(IntVT, SourceConst);
- SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0),
- MaskConst);
+ SDValue NewAnd =
+ DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0), MaskConst);
SDValue Res = DAG.getBitcast(VT, NewAnd);
if (IsStrict)
return DAG.getMergeValues({Res, SourceConst.getValue(1)}, DL);
@@ -59197,8 +59229,8 @@ static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG,
// use CVTSI2P.
assert(InVT == MVT::v2i64 && "Unexpected VT!");
SDValue Cast = DAG.getBitcast(MVT::v4i32, Op0);
- SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast,
- { 0, 2, -1, -1 });
+ SDValue Shuf =
+ DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast, {0, 2, -1, -1});
if (IsStrict)
return DAG.getNode(X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other},
{N->getOperand(0), Shuf});
@@ -59394,7 +59426,7 @@ static bool needCarryOrOverflowFlag(SDValue Flags) {
}
switch (CC) {
- // clang-format off
+ // clang-format off
default: break;
case X86::COND_A: case X86::COND_AE:
case X86::COND_B: case X86::COND_BE:
@@ -59402,7 +59434,7 @@ static bool needCarryOrOverflowFlag(SDValue Flags) {
case X86::COND_G: case X86::COND_GE:
case X86::COND_L: case X86::COND_LE:
return true;
- // clang-format on
+ // clang-format on
}
}
@@ -59542,11 +59574,12 @@ static SDValue combineCMP(SDNode *N, SelectionDAG &DAG,
// After this the truncate and arithmetic op must have a single use.
if (!Trunc.hasOneUse() || !Op.hasOneUse())
- return SDValue();
+ return SDValue();
unsigned NewOpc;
switch (Op.getOpcode()) {
- default: return SDValue();
+ default:
+ return SDValue();
case ISD::AND:
// Skip and with constant. We have special handling for and with immediate
// during isel to generate test instructions.
@@ -59554,8 +59587,12 @@ static SDValue combineCMP(SDNode *N, SelectionDAG &DAG,
return SDValue();
NewOpc = X86ISD::AND;
break;
- case ISD::OR: NewOpc = X86ISD::OR; break;
- case ISD::XOR: NewOpc = X86ISD::XOR; break;
+ case ISD::OR:
+ NewOpc = X86ISD::OR;
+ break;
+ case ISD::XOR:
+ NewOpc = X86ISD::XOR;
+ break;
case ISD::ADD:
// If the carry or overflow flag is used, we can't truncate.
if (needCarryOrOverflowFlag(SDValue(N, 0)))
@@ -59783,9 +59820,8 @@ static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
-static SDValue matchPMADDWD(SelectionDAG &DAG, SDNode *N,
- const SDLoc &DL, EVT VT,
- const X86Subtarget &Subtarget) {
+static SDValue matchPMADDWD(SelectionDAG &DAG, SDNode *N, const SDLoc &DL,
+ EVT VT, const X86Subtarget &Subtarget) {
using namespace SDPatternMatch;
// Example of pattern we try to detect:
@@ -59930,9 +59966,8 @@ static SDValue matchPMADDWD(SelectionDAG &DAG, SDNode *N,
// Attempt to turn this pattern into PMADDWD.
// (add (mul (sext (build_vector)), (sext (build_vector))),
// (mul (sext (build_vector)), (sext (build_vector)))
-static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N,
- const SDLoc &DL, EVT VT,
- const X86Subtarget &Subtarget) {
+static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N, const SDLoc &DL,
+ EVT VT, const X86Subtarget &Subtarget) {
using namespace SDPatternMatch;
if (!Subtarget.hasSSE2())
@@ -60027,7 +60062,7 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N,
// If the output is narrower than an input, extract the low part of the input
// vector.
EVT OutVT16 = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
- VT.getVectorNumElements() * 2);
+ VT.getVectorNumElements() * 2);
if (OutVT16.bitsLT(In0.getValueType())) {
In0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In0,
DAG.getVectorIdxConstant(0, DL));
@@ -60036,8 +60071,7 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N,
In1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In1,
DAG.getVectorIdxConstant(0, DL));
}
- return SplitOpsAndApply(DAG, Subtarget, DL, VT, { In0, In1 },
- PMADDBuilder);
+ return SplitOpsAndApply(DAG, Subtarget, DL, VT, {In0, In1}, PMADDBuilder);
}
// ADD(VPMADDWD(X,Y),VPMADDWD(Z,W)) -> VPMADDWD(SHUFFLE(X,Z), SHUFFLE(Y,W))
@@ -61052,8 +61086,8 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
unsigned Imm1 = Ops[1].getConstantOperandVal(2);
// TODO: Handle zero'd subvectors.
if ((Imm0 & 0x88) == 0 && (Imm1 & 0x88) == 0) {
- int Mask[4] = {(int)(Imm0 & 0x03), (int)((Imm0 >> 4) & 0x3), (int)(Imm1 & 0x03),
- (int)((Imm1 >> 4) & 0x3)};
+ int Mask[4] = {(int)(Imm0 & 0x03), (int)((Imm0 >> 4) & 0x3),
+ (int)(Imm1 & 0x03), (int)((Imm1 >> 4) & 0x3)};
MVT ShuffleVT = VT.isFloatingPoint() ? MVT::v8f64 : MVT::v8i64;
SDValue LHS = concatSubVectors(Ops[0].getOperand(0),
Ops[0].getOperand(1), DAG, DL);
@@ -61325,8 +61359,7 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
break;
}
- ISD::CondCode ICC =
- Opcode == X86ISD::PCMPEQ ? ISD::SETEQ : ISD::SETGT;
+ ISD::CondCode ICC = Opcode == X86ISD::PCMPEQ ? ISD::SETEQ : ISD::SETGT;
ISD::CondCode FCC =
Opcode == X86ISD::PCMPEQ ? ISD::SETOEQ : ISD::SETOGT;
@@ -61840,7 +61873,8 @@ static SDValue combineCONCAT_VECTORS(SDNode *N, SelectionDAG &DAG,
APInt Constant = APInt::getZero(VT.getSizeInBits());
for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
auto *C = dyn_cast<ConstantSDNode>(peekThroughBitcasts(Ops[I]));
- if (!C) break;
+ if (!C)
+ break;
Constant.insertBits(C->getAPIntValue(), I * SubSizeInBits);
if (I == (E - 1)) {
EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
@@ -61927,9 +61961,9 @@ static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
ISD::isBuildVectorAllZeros(Ins.getOperand(0).getNode()) &&
Ins.getOperand(1).getValueSizeInBits().getFixedValue() <=
SubVecVT.getFixedSizeInBits())
- return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
- getZeroVector(OpVT, Subtarget, DAG, dl),
- Ins.getOperand(1), N->getOperand(2));
+ return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
+ getZeroVector(OpVT, Subtarget, DAG, dl),
+ Ins.getOperand(1), N->getOperand(2));
}
// See if were inserting into a zero vXi1 vector and the subvector was
@@ -62621,7 +62655,7 @@ static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
LHS.getOperand(0).getValueType() == MVT::v4i32) {
SDLoc dl(N);
LHS = DAG.getVectorShuffle(MVT::v4i32, dl, LHS.getOperand(0),
- LHS.getOperand(0), { 0, -1, 1, -1 });
+ LHS.getOperand(0), {0, -1, 1, -1});
LHS = DAG.getBitcast(MVT::v2i64, LHS);
return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
}
@@ -62631,7 +62665,7 @@ static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
RHS.getOperand(0).getValueType() == MVT::v4i32) {
SDLoc dl(N);
RHS = DAG.getVectorShuffle(MVT::v4i32, dl, RHS.getOperand(0),
- RHS.getOperand(0), { 0, -1, 1, -1 });
+ RHS.getOperand(0), {0, -1, 1, -1});
RHS = DAG.getBitcast(MVT::v2i64, RHS);
return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
}
@@ -63107,16 +63141,16 @@ static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG,
// Widen to at least 8 input elements.
if (NumElts < 8) {
unsigned NumConcats = 8 / NumElts;
- SDValue Fill = NumElts == 4 ? DAG.getUNDEF(IntVT)
- : DAG.getConstant(0, dl, IntVT);
+ SDValue Fill =
+ NumElts == 4 ? DAG.getUNDEF(IntVT) : DAG.getConstant(0, dl, IntVT);
SmallVector<SDValue, 4> Ops(NumConcats, Fill);
Ops[0] = Src;
Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, Ops);
}
// Destination is vXf32 with at least 4 elements.
- EVT CvtVT = EVT::getVectorVT(*DAG.getContext(), MVT::f32,
- std::max(4U, NumElts));
+ EVT CvtVT =
+ EVT::getVectorVT(*DAG.getContext(), MVT::f32, std::max(4U, NumElts));
SDValue Cvt, Chain;
if (IsStrict) {
Cvt = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {CvtVT, MVT::Other},
@@ -63438,7 +63472,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
switch (N->getOpcode()) {
- // clang-format off
+ // clang-format off
default: break;
case ISD::SCALAR_TO_VECTOR:
return combineSCALAR_TO_VECTOR(N, DAG, Subtarget);
@@ -63819,7 +63853,8 @@ bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
bool Commute = false;
switch (Op.getOpcode()) {
- default: return false;
+ default:
+ return false;
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
case ISD::ANY_EXTEND:
@@ -63859,8 +63894,7 @@ bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
((Commute && !isa<ConstantSDNode>(N1)) ||
(Op.getOpcode() != ISD::MUL && IsFoldableRMW(N0, Op))))
return false;
- if (IsFoldableAtomicRMW(N0, Op) ||
- (Commute && IsFoldableAtomicRMW(N1, Op)))
+ if (IsFoldableAtomicRMW(N0, Op) || (Commute && IsFoldableAtomicRMW(N1, Op)))
return false;
}
}
@@ -63947,8 +63981,7 @@ X86TargetLowering::getConstraintType(StringRef Constraint) const {
default:
break;
}
- }
- else if (Constraint.size() == 2) {
+ } else if (Constraint.size() == 2) {
switch (Constraint[0]) {
default:
break;
@@ -64137,8 +64170,7 @@ X86TargetLowering::getSingleConstraintMatchWeight(
/// Try to replace an X constraint, which matches anything, with another that
/// has more specific requirements based on the type of the corresponding
/// operand.
-const char *X86TargetLowering::
-LowerXConstraint(EVT ConstraintVT) const {
+const char *X86TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
// FP X constraints get lowered to SSE1/2 registers if available, otherwise
// 'f' like normal targets.
if (ConstraintVT.isFloatingPoint()) {
@@ -64184,7 +64216,8 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
SDValue Result;
char ConstraintLetter = Constraint[0];
switch (ConstraintLetter) {
- default: break;
+ default:
+ break;
case 'I':
if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
if (C->getZExtValue() <= 31) {
@@ -64259,8 +64292,8 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
DAG.getSignedTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64);
break;
}
- // FIXME gcc accepts some relocatable values here too, but only in certain
- // memory models; it's complicated.
+ // FIXME gcc accepts some relocatable values here too, but only in certain
+ // memory models; it's complicated.
}
return;
}
@@ -64303,8 +64336,8 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
if (auto *CST = dyn_cast<ConstantSDNode>(Op)) {
bool IsBool = CST->getConstantIntValue()->getBitWidth() == 1;
BooleanContent BCont = getBooleanContents(MVT::i64);
- ISD::NodeType ExtOpc = IsBool ? getExtendForContent(BCont)
- : ISD::SIGN_EXTEND;
+ ISD::NodeType ExtOpc =
+ IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;
SDLoc DL(Op);
Result =
ExtOpc == ISD::ZERO_EXTEND
@@ -64385,7 +64418,8 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
if (Constraint.size() == 1) {
// GCC Constraint Letters
switch (Constraint[0]) {
- default: break;
+ default:
+ break;
// 'A' means [ER]AX + [ER]DX.
case 'A':
if (Subtarget.is64Bit())
@@ -64413,7 +64447,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
return std::make_pair(0U, &X86::VK64RegClass);
}
break;
- case 'q': // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode.
+ case 'q': // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode.
if (Subtarget.is64Bit()) {
if (VT == MVT::i8 || VT == MVT::i1)
return std::make_pair(0U, useEGPRInlineAsm(Subtarget)
@@ -64435,7 +64469,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
}
[[fallthrough]];
// 32-bit fallthrough
- case 'Q': // Q_REGS
+ case 'Q': // Q_REGS
if (VT == MVT::i8 || VT == MVT::i1)
return std::make_pair(0U, &X86::GR8_ABCD_LRegClass);
if (VT == MVT::i16)
@@ -64446,8 +64480,8 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
if (VT != MVT::f80 && !VT.isVector() && VT != MVT::Other)
return std::make_pair(0U, &X86::GR64_ABCDRegClass);
break;
- case 'r': // GENERAL_REGS
- case 'l': // INDEX_REGS
+ case 'r': // GENERAL_REGS
+ case 'l': // INDEX_REGS
if (VT == MVT::i8 || VT == MVT::i1)
return std::make_pair(0U, useEGPRInlineAsm(Subtarget)
? &X86::GR8RegClass
@@ -64466,7 +64500,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
? &X86::GR64RegClass
: &X86::GR64_NOREX2RegClass);
break;
- case 'R': // LEGACY_REGS
+ case 'R': // LEGACY_REGS
if (VT == MVT::i8 || VT == MVT::i1)
return std::make_pair(0U, &X86::GR8_NOREXRegClass);
if (VT == MVT::i16)
@@ -64477,7 +64511,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
if (VT != MVT::f80 && !VT.isVector() && VT != MVT::Other)
return std::make_pair(0U, &X86::GR64_NOREXRegClass);
break;
- case 'f': // FP Stack registers.
+ case 'f': // FP Stack registers.
// If SSE is enabled for this VT, use f80 to ensure the isel moves the
// value to the correct fpstack register class.
if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT))
@@ -64487,16 +64521,19 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80)
return std::make_pair(0U, &X86::RFP80RegClass);
break;
- case 'y': // MMX_REGS if MMX allowed.
- if (!Subtarget.hasMMX()) break;
+ case 'y': // MMX_REGS if MMX allowed.
+ if (!Subtarget.hasMMX())
+ break;
return std::make_pair(0U, &X86::VR64RegClass);
case 'v':
- case 'x': // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed
- if (!Subtarget.hasSSE1()) break;
+ case 'x': // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed
+ if (!Subtarget.hasSSE1())
+ break;
bool VConstraint = (Constraint[0] == 'v');
switch (VT.SimpleTy) {
- default: break;
+ default:
+ break;
// Scalar SSE types.
case MVT::f16:
if (VConstraint && Subtarget.hasFP16())
@@ -64587,7 +64624,8 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
case MVT::v16f32:
case MVT::v16i32:
case MVT::v8i64:
- if (!Subtarget.hasAVX512()) break;
+ if (!Subtarget.hasAVX512())
+ break;
if (VConstraint)
return std::make_pair(0U, &X86::VR512RegClass);
return std::make_pair(0U, &X86::VR512_0_15RegClass);
@@ -64603,12 +64641,15 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
case '2':
return getRegForInlineAsmConstraint(TRI, "x", VT);
case 'm':
- if (!Subtarget.hasMMX()) break;
+ if (!Subtarget.hasMMX())
+ break;
return std::make_pair(0U, &X86::VR64RegClass);
case 'z':
- if (!Subtarget.hasSSE1()) break;
+ if (!Subtarget.hasSSE1())
+ break;
switch (VT.SimpleTy) {
- default: break;
+ default:
+ break;
// Scalar SSE types.
case MVT::f16:
if (!Subtarget.hasFP16())
@@ -64723,14 +64764,15 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
// Use the default implementation in TargetLowering to convert the register
// constraint into a member of a register class.
- std::pair<Register, const TargetRegisterClass*> Res;
+ std::pair<Register, const TargetRegisterClass *> Res;
Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
// Not found as a standard register?
if (!Res.second) {
// Only match x87 registers if the VT is one SelectionDAGBuilder can convert
// to/from f80.
- if (VT == MVT::Other || VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80) {
+ if (VT == MVT::Other || VT == MVT::f32 || VT == MVT::f64 ||
+ VT == MVT::f80) {
// Map st(0) -> st(7) -> ST0
if (Constraint.size() == 7 && Constraint[0] == '{' &&
tolower(Constraint[1]) == 's' && tolower(Constraint[2]) == 't' &&
@@ -64788,7 +64830,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
// turn into {ax},{dx}.
// MVT::Other is used to specify clobber names.
if (TRI->isTypeLegalForClass(*Res.second, VT) || VT == MVT::Other)
- return Res; // Correct type already, nothing to do.
+ return Res; // Correct type already, nothing to do.
// Get a matching integer of the correct size. i.e. "ax" with MVT::32 should
// return "eax". This should even work for things like getting 64bit integer
@@ -64800,7 +64842,8 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
// Therefore, use a helper method.
if (isGRClass(*Class)) {
unsigned Size = VT.getSizeInBits();
- if (Size == 1) Size = 8;
+ if (Size == 1)
+ Size = 8;
if (Size != 8 && Size != 16 && Size != 32 && Size != 64)
return std::make_pair(0, nullptr);
Register DestReg = getX86SubSuperRegister(Res.first, Size);
@@ -64808,9 +64851,11 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
bool is64Bit = Subtarget.is64Bit();
const TargetRegisterClass *RC =
Size == 8 ? (is64Bit ? &X86::GR8RegClass : &X86::GR8_NOREXRegClass)
- : Size == 16 ? (is64Bit ? &X86::GR16RegClass : &X86::GR16_NOREXRegClass)
- : Size == 32 ? (is64Bit ? &X86::GR32RegClass : &X86::GR32_NOREXRegClass)
- : /*Size == 64*/ (is64Bit ? &X86::GR64RegClass : nullptr);
+ : Size == 16
+ ? (is64Bit ? &X86::GR16RegClass : &X86::GR16_NOREXRegClass)
+ : Size == 32
+ ? (is64Bit ? &X86::GR32RegClass : &X86::GR32_NOREXRegClass)
+ : /*Size == 64*/ (is64Bit ? &X86::GR64RegClass : nullptr);
if (Size == 64 && !is64Bit) {
// Model GCC's behavior here and select a fixed pair of 32-bit
// registers.
@@ -65062,8 +65107,7 @@ X86TargetLowering::getStackProbeSymbolName(const MachineFunction &MF) const {
return getLibcallImplName(StackProbeImpl);
}
-unsigned
-X86TargetLowering::getStackProbeSize(const MachineFunction &MF) const {
+unsigned X86TargetLowering::getStackProbeSize(const MachineFunction &MF) const {
// The default stack probe size is 4096 if the function has no stackprobesize
// attribute.
return MF.getFunction().getFnAttributeAsParsedInteger("stack-probe-size",
>From 3d41fa72cfca77aecdb4e0033201d48b48437079 Mon Sep 17 00:00:00 2001
From: Ankit Kumar Tiwari <ankit.cybertron at gmail.com>
Date: Tue, 18 Aug 2026 22:43:41 +0530
Subject: [PATCH 5/5] Fix formatting
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 4038 +++++++++++------------
1 file changed, 1997 insertions(+), 2041 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b81e09bf5226d..1f10178fabd72 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -189,10 +189,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// We don't accept any truncstore of integer registers.
setTruncStoreAction(MVT::i64, MVT::i32, Expand);
setTruncStoreAction(MVT::i64, MVT::i16, Expand);
- setTruncStoreAction(MVT::i64, MVT::i8, Expand);
+ setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
setTruncStoreAction(MVT::i32, MVT::i16, Expand);
- setTruncStoreAction(MVT::i32, MVT::i8, Expand);
- setTruncStoreAction(MVT::i16, MVT::i8, Expand);
+ setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
+ setTruncStoreAction(MVT::i16, MVT::i8, Expand);
setTruncStoreAction(MVT::f64, MVT::f32, Expand);
@@ -204,106 +204,106 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// Integer absolute.
if (Subtarget.canUseCMOV()) {
- setOperationAction(ISD::ABS, MVT::i16, Custom);
- setOperationAction(ISD::ABS, MVT::i32, Custom);
+ setOperationAction(ISD::ABS , MVT::i16 , Custom);
+ setOperationAction(ISD::ABS , MVT::i32 , Custom);
if (Subtarget.is64Bit())
- setOperationAction(ISD::ABS, MVT::i64, Custom);
+ setOperationAction(ISD::ABS , MVT::i64 , Custom);
}
// Absolute difference.
for (auto Op : {ISD::ABDS, ISD::ABDU}) {
- setOperationAction(Op, MVT::i8, Custom);
- setOperationAction(Op, MVT::i16, Custom);
- setOperationAction(Op, MVT::i32, Custom);
+ setOperationAction(Op , MVT::i8 , Custom);
+ setOperationAction(Op , MVT::i16 , Custom);
+ setOperationAction(Op , MVT::i32 , Custom);
if (Subtarget.is64Bit())
- setOperationAction(Op, MVT::i64, Custom);
+ setOperationAction(Op , MVT::i64 , Custom);
}
// Signed saturation subtraction.
- setOperationAction(ISD::SSUBSAT, MVT::i8, Custom);
- setOperationAction(ISD::SSUBSAT, MVT::i16, Custom);
- setOperationAction(ISD::SSUBSAT, MVT::i32, Custom);
+ setOperationAction(ISD::SSUBSAT , MVT::i8 , Custom);
+ setOperationAction(ISD::SSUBSAT , MVT::i16 , Custom);
+ setOperationAction(ISD::SSUBSAT , MVT::i32 , Custom);
if (Subtarget.is64Bit())
- setOperationAction(ISD::SSUBSAT, MVT::i64, Custom);
+ setOperationAction(ISD::SSUBSAT , MVT::i64 , Custom);
// Funnel shifts.
for (auto ShiftOp : {ISD::FSHL, ISD::FSHR}) {
// For slow shld targets we only lower for code size.
LegalizeAction ShiftDoubleAction = Subtarget.isSHLDSlow() ? Custom : Legal;
- setOperationAction(ShiftOp, MVT::i8, Custom);
- setOperationAction(ShiftOp, MVT::i16, Custom);
- setOperationAction(ShiftOp, MVT::i32, ShiftDoubleAction);
+ setOperationAction(ShiftOp , MVT::i8 , Custom);
+ setOperationAction(ShiftOp , MVT::i16 , Custom);
+ setOperationAction(ShiftOp , MVT::i32 , ShiftDoubleAction);
if (Subtarget.is64Bit())
- setOperationAction(ShiftOp, MVT::i64, ShiftDoubleAction);
+ setOperationAction(ShiftOp , MVT::i64 , ShiftDoubleAction);
}
if (!Subtarget.useSoftFloat()) {
// Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
// operation.
- setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);
+ setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i8, Promote);
- setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
+ setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i16, Promote);
// We have an algorithm for SSE2, and we turn this into a 64-bit
// FILD or VCVTUSI2SS/SD for other targets.
- setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
// We have an algorithm for SSE2->double, and we turn this into a
// 64-bit FILD followed by conditional FADD for other targets.
- setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
// Promote i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
// this operation.
- setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);
+ setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i8, Promote);
// SSE has no i16 to fp conversion, only i32. We promote in the handler
// to allow f80 to use i16 and f64 to use i16 with sse1 only
- setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i16, Custom);
// f32 and f64 cases are Legal with SSE1/SSE2, f80 case is not
- setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
// In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
// are Legal, f80 is custom lowered.
- setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
// Promote i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
// this operation.
- setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote);
+ setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote);
// FIXME: This doesn't generate invalid exception when it should. PR44019.
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i8, Promote);
- setOperationAction(ISD::FP_TO_SINT, MVT::i16, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i8, Promote);
+ setOperationAction(ISD::FP_TO_SINT, MVT::i16, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i16, Custom);
- setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
// In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
// are Legal, f80 is custom lowered.
- setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
// Handle FP_TO_UINT by promoting the destination to a larger signed
// conversion.
- setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote);
+ setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote);
// FIXME: This doesn't generate invalid exception when it should. PR44019.
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i8, Promote);
- setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i8, Promote);
+ setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
// FIXME: This doesn't generate invalid exception when it should. PR44019.
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i16, Promote);
- setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
- setOperationAction(ISD::LRINT, MVT::f32, Custom);
- setOperationAction(ISD::LRINT, MVT::f64, Custom);
- setOperationAction(ISD::LLRINT, MVT::f32, Custom);
- setOperationAction(ISD::LLRINT, MVT::f64, Custom);
+ setOperationAction(ISD::LRINT, MVT::f32, Custom);
+ setOperationAction(ISD::LRINT, MVT::f64, Custom);
+ setOperationAction(ISD::LLRINT, MVT::f32, Custom);
+ setOperationAction(ISD::LLRINT, MVT::f64, Custom);
if (!Subtarget.is64Bit() && Subtarget.hasX87()) {
- setOperationAction(ISD::LRINT, MVT::i64, Custom);
+ setOperationAction(ISD::LRINT, MVT::i64, Custom);
setOperationAction(ISD::LLRINT, MVT::i64, Custom);
}
}
@@ -311,7 +311,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
if (Subtarget.hasSSE2()) {
// Custom lowering for saturating float to int conversions.
// We handle promotion to larger result types manually.
- for (MVT VT : {MVT::i8, MVT::i16, MVT::i32}) {
+ for (MVT VT : { MVT::i8, MVT::i16, MVT::i32 }) {
setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom);
setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
}
@@ -346,15 +346,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// TODO: when we have SSE, these could be more efficient, by using movd/movq.
if (!Subtarget.hasSSE2()) {
- setOperationAction(ISD::BITCAST, MVT::f32, Expand);
- setOperationAction(ISD::BITCAST, MVT::i32, Expand);
+ setOperationAction(ISD::BITCAST , MVT::f32 , Expand);
+ setOperationAction(ISD::BITCAST , MVT::i32 , Expand);
if (Subtarget.is64Bit()) {
- setOperationAction(ISD::BITCAST, MVT::f64, Expand);
+ setOperationAction(ISD::BITCAST , MVT::f64 , Expand);
// Without SSE, i64->f64 goes through memory.
- setOperationAction(ISD::BITCAST, MVT::i64, Expand);
+ setOperationAction(ISD::BITCAST , MVT::i64 , Expand);
}
} else if (!Subtarget.is64Bit())
- setOperationAction(ISD::BITCAST, MVT::i64, Custom);
+ setOperationAction(ISD::BITCAST , MVT::i64 , Custom);
// Scalar integer divide and remainder are lowered to use operations that
// produce two results, to match the available instructions. This exposes
@@ -366,7 +366,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// (low) operations are left as Legal, as there are single-result
// instructions for this in x86. Using the two-result multiply instructions
// when both high and low results are needed must be arranged by dagcombine.
- for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
+ for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
setOperationAction(ISD::MULHS, VT, Expand);
setOperationAction(ISD::MULHU, VT, Expand);
setOperationAction(ISD::SDIV, VT, Expand);
@@ -375,18 +375,18 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::UREM, VT, Expand);
}
- setOperationAction(ISD::BR_JT, MVT::Other, Expand);
- setOperationAction(ISD::BRCOND, MVT::Other, Custom);
- for (auto VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128, MVT::i8, MVT::i16,
- MVT::i32, MVT::i64}) {
- setOperationAction(ISD::BR_CC, VT, Expand);
+ setOperationAction(ISD::BR_JT , MVT::Other, Expand);
+ setOperationAction(ISD::BRCOND , MVT::Other, Custom);
+ for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128,
+ MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
+ setOperationAction(ISD::BR_CC, VT, Expand);
setOperationAction(ISD::SELECT_CC, VT, Expand);
}
if (Subtarget.is64Bit())
setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
- setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Legal);
- setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Legal);
- setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
+ setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal);
+ setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
+ setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand);
setOperationAction(ISD::FREM, MVT::f32, LibCall);
setOperationAction(ISD::FREM, MVT::f64, LibCall);
@@ -394,11 +394,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FREM, MVT::f128, LibCall);
if (!Subtarget.useSoftFloat() && Subtarget.hasX87()) {
- setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom);
- setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
- setOperationAction(ISD::GET_FPENV_MEM, MVT::Other, Custom);
- setOperationAction(ISD::SET_FPENV_MEM, MVT::Other, Custom);
- setOperationAction(ISD::RESET_FPENV, MVT::Other, Custom);
+ setOperationAction(ISD::GET_ROUNDING , MVT::i32 , Custom);
+ setOperationAction(ISD::SET_ROUNDING , MVT::Other, Custom);
+ setOperationAction(ISD::GET_FPENV_MEM , MVT::Other, Custom);
+ setOperationAction(ISD::SET_FPENV_MEM , MVT::Other, Custom);
+ setOperationAction(ISD::RESET_FPENV , MVT::Other, Custom);
}
// Promote the i8 variants and force them on up to i32 which has a shorter
@@ -474,10 +474,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// on the dest that popcntl hasn't had since Cannon Lake.
setOperationPromotedToType(ISD::CTPOP, MVT::i16, MVT::i32);
} else {
- setOperationAction(ISD::CTPOP, MVT::i8, Custom);
- setOperationAction(ISD::CTPOP, MVT::i16, Custom);
- setOperationAction(ISD::CTPOP, MVT::i32, Custom);
- setOperationAction(ISD::CTPOP, MVT::i64, Custom);
+ setOperationAction(ISD::CTPOP , MVT::i8 , Custom);
+ setOperationAction(ISD::CTPOP , MVT::i16 , Custom);
+ setOperationAction(ISD::CTPOP , MVT::i32 , Custom);
+ setOperationAction(ISD::CTPOP , MVT::i64 , Custom);
}
if (Subtarget.hasBMI2()) {
@@ -488,23 +488,23 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction({ISD::PEXT, ISD::PDEP}, MVT::i64, Legal);
}
- setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom);
+ setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
if (!Subtarget.hasMOVBE())
- setOperationAction(ISD::BSWAP, MVT::i16, Expand);
+ setOperationAction(ISD::BSWAP , MVT::i16 , Expand);
// X86 wants to expand cmov itself.
- for (auto VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) {
+ for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
setOperationAction(ISD::SELECT, VT, Custom);
setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
}
- for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
+ for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
if (VT == MVT::i64 && !Subtarget.is64Bit())
continue;
setOperationAction(ISD::SELECT, VT, Custom);
- setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::SETCC, VT, Custom);
}
setOperationAction(ISD::COND_LOOP, MVT::Other, Custom);
@@ -513,7 +513,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SELECT, MVT::x86mmx, Custom);
setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand);
- setOperationAction(ISD::EH_RETURN, MVT::Other, Custom);
+ setOperationAction(ISD::EH_RETURN , MVT::Other, Custom);
// NOTE: EH_SJLJ_SETJMP/_LONGJMP are not recommended, since
// LLVM/Clang supports zero-cost DWARF and SEH exception handling.
setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
@@ -521,19 +521,19 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
// Darwin ABI issue.
- for (auto VT : {MVT::i32, MVT::i64}) {
+ for (auto VT : { MVT::i32, MVT::i64 }) {
if (VT == MVT::i64 && !Subtarget.is64Bit())
continue;
- setOperationAction(ISD::ConstantPool, VT, Custom);
- setOperationAction(ISD::JumpTable, VT, Custom);
- setOperationAction(ISD::GlobalAddress, VT, Custom);
+ setOperationAction(ISD::ConstantPool , VT, Custom);
+ setOperationAction(ISD::JumpTable , VT, Custom);
+ setOperationAction(ISD::GlobalAddress , VT, Custom);
setOperationAction(ISD::GlobalTLSAddress, VT, Custom);
- setOperationAction(ISD::ExternalSymbol, VT, Custom);
- setOperationAction(ISD::BlockAddress, VT, Custom);
+ setOperationAction(ISD::ExternalSymbol , VT, Custom);
+ setOperationAction(ISD::BlockAddress , VT, Custom);
}
// 64-bit shl, sra, srl (iff 32-bit x86)
- for (auto VT : {MVT::i32, MVT::i64}) {
+ for (auto VT : { MVT::i32, MVT::i64 }) {
if (VT == MVT::i64 && !Subtarget.is64Bit())
continue;
setOperationAction(ISD::SHL_PARTS, VT, Custom);
@@ -542,12 +542,12 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
if (Subtarget.hasSSEPrefetch())
- setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
+ setOperationAction(ISD::PREFETCH , MVT::Other, Custom);
- setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
+ setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom);
// Expand certain atomics
- for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
+ for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom);
setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom);
@@ -591,14 +591,14 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
// VASTART needs to be custom lowered to use the VarArgsFrameIndex
- setOperationAction(ISD::VASTART, MVT::Other, Custom);
- setOperationAction(ISD::VAEND, MVT::Other, Expand);
+ setOperationAction(ISD::VASTART , MVT::Other, Custom);
+ setOperationAction(ISD::VAEND , MVT::Other, Expand);
bool Is64Bit = Subtarget.is64Bit();
- setOperationAction(ISD::VAARG, MVT::Other, Is64Bit ? Custom : Expand);
+ setOperationAction(ISD::VAARG, MVT::Other, Is64Bit ? Custom : Expand);
setOperationAction(ISD::VACOPY, MVT::Other, Is64Bit ? Custom : Expand);
- setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
- setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
+ setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
+ setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
@@ -608,7 +608,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
- auto setF16Action = [&](MVT VT, LegalizeAction Action) {
+ auto setF16Action = [&] (MVT VT, LegalizeAction Action) {
setOperationAction(ISD::FABS, VT, Action);
setOperationAction(ISD::FNEG, VT, Action);
setOperationAction(ISD::FCOPYSIGN, VT, Expand);
@@ -665,7 +665,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// non-optsize case.
setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
- for (auto VT : {MVT::f32, MVT::f64}) {
+ for (auto VT : { MVT::f32, MVT::f64 }) {
// Use ANDPD to simulate FABS.
setOperationAction(ISD::FABS, VT, Custom);
@@ -680,8 +680,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FSUB, VT, Custom);
// We don't support sin/cos/fmod
- setOperationAction(ISD::FSIN, VT, Expand);
- setOperationAction(ISD::FCOS, VT, Expand);
+ setOperationAction(ISD::FSIN , VT, Expand);
+ setOperationAction(ISD::FCOS , VT, Expand);
setOperationAction(ISD::FSINCOS, VT, Expand);
}
@@ -751,10 +751,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addRegisterClass(MVT::f64, &X86::RFP64RegClass);
// Use ANDPS to simulate FABS.
- setOperationAction(ISD::FABS, MVT::f32, Custom);
+ setOperationAction(ISD::FABS , MVT::f32, Custom);
// Use XORP to simulate FNEG.
- setOperationAction(ISD::FNEG, MVT::f32, Custom);
+ setOperationAction(ISD::FNEG , MVT::f32, Custom);
if (UseX87)
setOperationAction({ISD::UNDEF, ISD::POISON}, MVT::f64, Expand);
@@ -765,8 +765,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
// We don't support sin/cos/fmod
- setOperationAction(ISD::FSIN, MVT::f32, Expand);
- setOperationAction(ISD::FCOS, MVT::f32, Expand);
+ setOperationAction(ISD::FSIN , MVT::f32, Expand);
+ setOperationAction(ISD::FCOS , MVT::f32, Expand);
setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
if (UseX87) {
@@ -781,13 +781,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addRegisterClass(MVT::f64, &X86::RFP64RegClass);
addRegisterClass(MVT::f32, &X86::RFP32RegClass);
- for (auto VT : {MVT::f32, MVT::f64}) {
+ for (auto VT : { MVT::f32, MVT::f64 }) {
setOperationAction({ISD::UNDEF, ISD::POISON}, VT, Expand);
setOperationAction(ISD::FCOPYSIGN, VT, Expand);
// Always expand sin/cos functions even though x87 has an instruction.
- setOperationAction(ISD::FSIN, VT, Expand);
- setOperationAction(ISD::FCOS, VT, Expand);
+ setOperationAction(ISD::FSIN , VT, Expand);
+ setOperationAction(ISD::FCOS , VT, Expand);
setOperationAction(ISD::FSINCOS, VT, Expand);
}
}
@@ -799,7 +799,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addLegalFPImmediate(APFloat(+1.0f)); // FLD1
addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
- } else // SSE immediates.
+ } else // SSE immediates.
addLegalFPImmediate(APFloat(+0.0f)); // xorps
}
// Expand FP64 immediates into loads from the stack, save special cases.
@@ -809,7 +809,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addLegalFPImmediate(APFloat(+1.0)); // FLD1
addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
- } else // SSE immediates.
+ } else // SSE immediates.
addLegalFPImmediate(APFloat(+0.0)); // xorpd
}
// Support fp16 0 immediate.
@@ -817,18 +817,18 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addLegalFPImmediate(APFloat::getZero(APFloat::IEEEhalf()));
// Handle constrained floating-point operations of scalar.
- setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
// We don't support FMA.
setOperationAction(ISD::FMA, MVT::f64, Expand);
@@ -841,17 +841,17 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
{
APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended());
- addLegalFPImmediate(TmpFlt); // FLD0
+ addLegalFPImmediate(TmpFlt); // FLD0
TmpFlt.changeSign();
- addLegalFPImmediate(TmpFlt); // FLD0/FCHS
+ addLegalFPImmediate(TmpFlt); // FLD0/FCHS
bool ignored;
APFloat TmpFlt2(+1.0);
- TmpFlt2.convert(APFloat::x87DoubleExtended(),
- APFloat::rmNearestTiesToEven, &ignored);
- addLegalFPImmediate(TmpFlt2); // FLD1
+ TmpFlt2.convert(APFloat::x87DoubleExtended(), APFloat::rmNearestTiesToEven,
+ &ignored);
+ addLegalFPImmediate(TmpFlt2); // FLD1
TmpFlt2.changeSign();
- addLegalFPImmediate(TmpFlt2); // FLD1/FCHS
+ addLegalFPImmediate(TmpFlt2); // FLD1/FCHS
}
// Always expand sin/cos functions even though x87 has an instruction.
@@ -870,9 +870,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// clang-format on
setOperationAction(ISD::FFLOOR, MVT::f80, Expand);
- setOperationAction(ISD::FCEIL, MVT::f80, Expand);
+ setOperationAction(ISD::FCEIL, MVT::f80, Expand);
setOperationAction(ISD::FTRUNC, MVT::f80, Expand);
- setOperationAction(ISD::FRINT, MVT::f80, Expand);
+ setOperationAction(ISD::FRINT, MVT::f80, Expand);
setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand);
setOperationAction(ISD::FROUNDEVEN, MVT::f80, Expand);
setOperationAction(ISD::FMA, MVT::f80, Expand);
@@ -882,11 +882,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::LLRINT, MVT::f80, Custom);
// Handle constrained floating-point operations of scalar.
- setOperationAction(ISD::STRICT_FADD, MVT::f80, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::f80, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::f80, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::f80, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::f80, Legal);
+ setOperationAction(ISD::STRICT_FADD , MVT::f80, Legal);
+ setOperationAction(ISD::STRICT_FSUB , MVT::f80, Legal);
+ setOperationAction(ISD::STRICT_FMUL , MVT::f80, Legal);
+ setOperationAction(ISD::STRICT_FDIV , MVT::f80, Legal);
+ setOperationAction(ISD::STRICT_FSQRT , MVT::f80, Legal);
setOperationAction(ISD::FCANONICALIZE, MVT::f80, Expand);
if (isTypeLegal(MVT::f16)) {
setOperationAction(ISD::FP_EXTEND, MVT::f80, Custom);
@@ -906,16 +906,16 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addLegalFPImmediate(APFloat::getZero(APFloat::IEEEquad())); // xorps
- setOperationAction(ISD::FADD, MVT::f128, LibCall);
+ setOperationAction(ISD::FADD, MVT::f128, LibCall);
setOperationAction(ISD::STRICT_FADD, MVT::f128, LibCall);
- setOperationAction(ISD::FSUB, MVT::f128, LibCall);
+ setOperationAction(ISD::FSUB, MVT::f128, LibCall);
setOperationAction(ISD::STRICT_FSUB, MVT::f128, LibCall);
- setOperationAction(ISD::FDIV, MVT::f128, LibCall);
+ setOperationAction(ISD::FDIV, MVT::f128, LibCall);
setOperationAction(ISD::STRICT_FDIV, MVT::f128, LibCall);
- setOperationAction(ISD::FMUL, MVT::f128, LibCall);
+ setOperationAction(ISD::FMUL, MVT::f128, LibCall);
setOperationAction(ISD::STRICT_FMUL, MVT::f128, LibCall);
- setOperationAction(ISD::FMA, MVT::f128, LibCall);
- setOperationAction(ISD::STRICT_FMA, MVT::f128, LibCall);
+ setOperationAction(ISD::FMA, MVT::f128, LibCall);
+ setOperationAction(ISD::STRICT_FMA, MVT::f128, LibCall);
setOperationAction(ISD::FABS, MVT::f128, Custom);
setOperationAction(ISD::FNEG, MVT::f128, Custom);
@@ -931,10 +931,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::STRICT_FTAN, MVT::f128, LibCall);
// clang-format on
// No STRICT_FSINCOS
- setOperationAction(ISD::FSQRT, MVT::f128, LibCall);
+ setOperationAction(ISD::FSQRT, MVT::f128, LibCall);
setOperationAction(ISD::STRICT_FSQRT, MVT::f128, LibCall);
- setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
+ setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Custom);
// We need to custom handle any FP_ROUND with an f128 input, but
// LegalizeDAG uses the result type to know when to run a custom handler.
@@ -964,10 +964,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
// Always use a library call for pow.
- setOperationAction(ISD::FPOW, MVT::f32, Expand);
- setOperationAction(ISD::FPOW, MVT::f64, Expand);
- setOperationAction(ISD::FPOW, MVT::f80, Expand);
- setOperationAction(ISD::FPOW, MVT::f128, Expand);
+ setOperationAction(ISD::FPOW , MVT::f32 , Expand);
+ setOperationAction(ISD::FPOW , MVT::f64 , Expand);
+ setOperationAction(ISD::FPOW , MVT::f80 , Expand);
+ setOperationAction(ISD::FPOW , MVT::f128 , Expand);
setOperationAction(ISD::FLOG, MVT::f80, Expand);
setOperationAction(ISD::FLOG2, MVT::f80, Expand);
@@ -979,8 +979,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);
// Some FP actions are always expanded for vector types.
- for (auto VT : {MVT::v8f16, MVT::v16f16, MVT::v32f16, MVT::v4f32, MVT::v8f32,
- MVT::v16f32, MVT::v2f64, MVT::v4f64, MVT::v8f64}) {
+ for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16,
+ MVT::v4f32, MVT::v8f32, MVT::v16f32,
+ MVT::v2f64, MVT::v4f64, MVT::v8f64 }) {
// clang-format off
setOperationAction(ISD::FSIN, VT, Expand);
setOperationAction(ISD::FSINCOS, VT, Expand);
@@ -1006,11 +1007,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::UDIV, VT, Expand);
setOperationAction(ISD::SREM, VT, Expand);
setOperationAction(ISD::UREM, VT, Expand);
- setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Expand);
+ setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);
setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
- setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Expand);
- setOperationAction(ISD::INSERT_SUBVECTOR, VT, Expand);
- setOperationAction(ISD::FMA, VT, Expand);
+ setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);
+ setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);
+ setOperationAction(ISD::FMA, VT, Expand);
setOperationAction(ISD::FFLOOR, VT, Expand);
setOperationAction(ISD::FCEIL, VT, Expand);
setOperationAction(ISD::FTRUNC, VT, Expand);
@@ -1034,7 +1035,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FP_TO_SINT, VT, Expand);
setOperationAction(ISD::UINT_TO_FP, VT, Expand);
setOperationAction(ISD::SINT_TO_FP, VT, Expand);
- setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
+ setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);
setOperationAction(ISD::TRUNCATE, VT, Expand);
setOperationAction(ISD::SIGN_EXTEND, VT, Expand);
setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
@@ -1085,25 +1086,25 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
SetFPMinMaxAction(MVT::f32);
- setOperationAction(ISD::FNEG, MVT::v4f32, Custom);
- setOperationAction(ISD::FABS, MVT::v4f32, Custom);
- setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Custom);
- setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
- setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
+ setOperationAction(ISD::FNEG, MVT::v4f32, Custom);
+ setOperationAction(ISD::FABS, MVT::v4f32, Custom);
+ setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Custom);
+ setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
+ setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
- setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
+ setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
setOperationAction(ISD::FCANONICALIZE, MVT::v4f32, Expand);
- setOperationAction(ISD::LOAD, MVT::v2f32, Custom);
- setOperationAction(ISD::STORE, MVT::v2f32, Custom);
+ setOperationAction(ISD::LOAD, MVT::v2f32, Custom);
+ setOperationAction(ISD::STORE, MVT::v2f32, Custom);
setOperationAction(ISD::FCANONICALIZE, MVT::v2f32, Expand);
- setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal);
}
if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
@@ -1123,34 +1124,34 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass
: &X86::VR128RegClass);
- for (auto VT : {MVT::f64, MVT::v4f32, MVT::v2f64})
+ for (auto VT : { MVT::f64, MVT::v4f32, MVT::v2f64 })
SetFPMinMaxAction(VT);
- setOperationAction(ISD::MUL, MVT::v2i8, Custom);
- setOperationAction(ISD::MUL, MVT::v4i8, Custom);
- setOperationAction(ISD::MUL, MVT::v8i8, Custom);
-
- setOperationAction(ISD::MUL, MVT::v16i8, Custom);
- setOperationAction(ISD::MUL, MVT::v4i32, Custom);
- setOperationAction(ISD::MUL, MVT::v2i64, Custom);
- setOperationAction(ISD::MULHU, MVT::v4i32, Custom);
- setOperationAction(ISD::MULHS, MVT::v4i32, Custom);
- setOperationAction(ISD::MULHU, MVT::v16i8, Custom);
- setOperationAction(ISD::MULHS, MVT::v16i8, Custom);
- setOperationAction(ISD::MULHU, MVT::v8i16, Legal);
- setOperationAction(ISD::MULHS, MVT::v8i16, Legal);
- setOperationAction(ISD::MUL, MVT::v8i16, Legal);
- setOperationAction(ISD::AVGCEILU, MVT::v16i8, Legal);
- setOperationAction(ISD::AVGCEILU, MVT::v8i16, Legal);
-
- setOperationAction(ISD::SMULO, MVT::v16i8, Custom);
- setOperationAction(ISD::UMULO, MVT::v16i8, Custom);
- setOperationAction(ISD::UMULO, MVT::v2i32, Custom);
-
- setOperationAction(ISD::FNEG, MVT::v2f64, Custom);
+ setOperationAction(ISD::MUL, MVT::v2i8, Custom);
+ setOperationAction(ISD::MUL, MVT::v4i8, Custom);
+ setOperationAction(ISD::MUL, MVT::v8i8, Custom);
+
+ setOperationAction(ISD::MUL, MVT::v16i8, Custom);
+ setOperationAction(ISD::MUL, MVT::v4i32, Custom);
+ setOperationAction(ISD::MUL, MVT::v2i64, Custom);
+ setOperationAction(ISD::MULHU, MVT::v4i32, Custom);
+ setOperationAction(ISD::MULHS, MVT::v4i32, Custom);
+ setOperationAction(ISD::MULHU, MVT::v16i8, Custom);
+ setOperationAction(ISD::MULHS, MVT::v16i8, Custom);
+ setOperationAction(ISD::MULHU, MVT::v8i16, Legal);
+ setOperationAction(ISD::MULHS, MVT::v8i16, Legal);
+ setOperationAction(ISD::MUL, MVT::v8i16, Legal);
+ setOperationAction(ISD::AVGCEILU, MVT::v16i8, Legal);
+ setOperationAction(ISD::AVGCEILU, MVT::v8i16, Legal);
+
+ setOperationAction(ISD::SMULO, MVT::v16i8, Custom);
+ setOperationAction(ISD::UMULO, MVT::v16i8, Custom);
+ setOperationAction(ISD::UMULO, MVT::v2i32, Custom);
+
+ setOperationAction(ISD::FNEG, MVT::v2f64, Custom);
setOperationAction(ISD::FCANONICALIZE, MVT::v2f64, Expand);
- setOperationAction(ISD::FABS, MVT::v2f64, Custom);
- setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Custom);
+ setOperationAction(ISD::FABS, MVT::v2f64, Custom);
+ setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Custom);
setOperationAction(ISD::LRINT, MVT::v4f32, Custom);
setOperationAction(ISD::LRINT, MVT::v2i32, Custom);
@@ -1170,7 +1171,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::CLMUL, MVT::i8, Custom);
}
- for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
+ for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom);
setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom);
setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom);
@@ -1184,30 +1185,30 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
- setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
}
- setOperationAction(ISD::UADDSAT, MVT::v16i8, Legal);
- setOperationAction(ISD::SADDSAT, MVT::v16i8, Legal);
- setOperationAction(ISD::USUBSAT, MVT::v16i8, Legal);
- setOperationAction(ISD::SSUBSAT, MVT::v16i8, Legal);
- setOperationAction(ISD::UADDSAT, MVT::v8i16, Legal);
- setOperationAction(ISD::SADDSAT, MVT::v8i16, Legal);
- setOperationAction(ISD::USUBSAT, MVT::v8i16, Legal);
- setOperationAction(ISD::SSUBSAT, MVT::v8i16, Legal);
- setOperationAction(ISD::USUBSAT, MVT::v4i32, Custom);
- setOperationAction(ISD::USUBSAT, MVT::v2i64, Custom);
+ setOperationAction(ISD::UADDSAT, MVT::v16i8, Legal);
+ setOperationAction(ISD::SADDSAT, MVT::v16i8, Legal);
+ setOperationAction(ISD::USUBSAT, MVT::v16i8, Legal);
+ setOperationAction(ISD::SSUBSAT, MVT::v16i8, Legal);
+ setOperationAction(ISD::UADDSAT, MVT::v8i16, Legal);
+ setOperationAction(ISD::SADDSAT, MVT::v8i16, Legal);
+ setOperationAction(ISD::USUBSAT, MVT::v8i16, Legal);
+ setOperationAction(ISD::SSUBSAT, MVT::v8i16, Legal);
+ setOperationAction(ISD::USUBSAT, MVT::v4i32, Custom);
+ setOperationAction(ISD::USUBSAT, MVT::v2i64, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
- for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
+ for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::CTPOP, VT, Custom);
setOperationAction(ISD::ABS, VT, Custom);
@@ -1220,30 +1221,30 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setCondCodeAction(ISD::SETLE, VT, Custom);
}
- setOperationAction(ISD::SETCC, MVT::v2f64, Custom);
- setOperationAction(ISD::SETCC, MVT::v4f32, Custom);
- setOperationAction(ISD::STRICT_FSETCC, MVT::v2f64, Custom);
- setOperationAction(ISD::STRICT_FSETCC, MVT::v4f32, Custom);
+ setOperationAction(ISD::SETCC, MVT::v2f64, Custom);
+ setOperationAction(ISD::SETCC, MVT::v4f32, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, MVT::v2f64, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, MVT::v4f32, Custom);
setOperationAction(ISD::STRICT_FSETCCS, MVT::v2f64, Custom);
setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f32, Custom);
- for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32}) {
- setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
- setOperationAction(ISD::VSELECT, VT, Custom);
+ for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
+ setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::VSELECT, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
}
- for (auto VT : {MVT::v8f16, MVT::v2f64, MVT::v2i64}) {
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
- setOperationAction(ISD::VSELECT, VT, Custom);
+ for (auto VT : { MVT::v8f16, MVT::v2f64, MVT::v2i64 }) {
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::VSELECT, VT, Custom);
if (VT == MVT::v2i64 && !Subtarget.is64Bit())
continue;
- setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
}
setF16Action(MVT::v8f16, Expand);
@@ -1256,67 +1257,67 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Custom);
// Custom lower v2i64 and v2f64 selects.
- setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
- setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
- setOperationAction(ISD::SELECT, MVT::v4i32, Custom);
- setOperationAction(ISD::SELECT, MVT::v8i16, Custom);
- setOperationAction(ISD::SELECT, MVT::v8f16, Custom);
- setOperationAction(ISD::SELECT, MVT::v16i8, Custom);
-
- setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Custom);
- setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v2i32, Custom);
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Custom);
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i32, Custom);
+ setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
+ setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
+ setOperationAction(ISD::SELECT, MVT::v4i32, Custom);
+ setOperationAction(ISD::SELECT, MVT::v8i16, Custom);
+ setOperationAction(ISD::SELECT, MVT::v8f16, Custom);
+ setOperationAction(ISD::SELECT, MVT::v16i8, Custom);
+
+ setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v2i32, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i32, Custom);
// Custom legalize these to avoid over promotion or custom promotion.
for (auto VT : {MVT::v2i8, MVT::v4i8, MVT::v8i8, MVT::v2i16, MVT::v4i16}) {
- setOperationAction(ISD::FP_TO_SINT, VT, Custom);
- setOperationAction(ISD::FP_TO_UINT, VT, Custom);
+ setOperationAction(ISD::FP_TO_SINT, VT, Custom);
+ setOperationAction(ISD::FP_TO_UINT, VT, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
}
- setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Custom);
- setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i32, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i32, Custom);
- setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
- setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i32, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
+ setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i32, Custom);
- setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
- setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
+ setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Custom);
// Fast v2f32 UINT_TO_FP( v2i32 ) custom conversion.
- setOperationAction(ISD::SINT_TO_FP, MVT::v2f32, Custom);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f32, Custom);
- setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom);
- setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f32, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v2f32, Custom);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f32, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom);
+ setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f32, Custom);
- setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f32, Custom);
- setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f32, Custom);
+ setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f32, Custom);
+ setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f32, Custom);
// We want to legalize this to an f64 load rather than an i64 load on
// 64-bit targets and two 32-bit loads on a 32-bit target. Similar for
// store.
- setOperationAction(ISD::LOAD, MVT::v2i32, Custom);
- setOperationAction(ISD::LOAD, MVT::v4i16, Custom);
- setOperationAction(ISD::LOAD, MVT::v8i8, Custom);
- setOperationAction(ISD::STORE, MVT::v2i32, Custom);
- setOperationAction(ISD::STORE, MVT::v4i16, Custom);
- setOperationAction(ISD::STORE, MVT::v8i8, Custom);
+ setOperationAction(ISD::LOAD, MVT::v2i32, Custom);
+ setOperationAction(ISD::LOAD, MVT::v4i16, Custom);
+ setOperationAction(ISD::LOAD, MVT::v8i8, Custom);
+ setOperationAction(ISD::STORE, MVT::v2i32, Custom);
+ setOperationAction(ISD::STORE, MVT::v4i16, Custom);
+ setOperationAction(ISD::STORE, MVT::v8i8, Custom);
// Add 32-bit vector stores to help vectorization opportunities.
- setOperationAction(ISD::STORE, MVT::v2i16, Custom);
- setOperationAction(ISD::STORE, MVT::v4i8, Custom);
+ setOperationAction(ISD::STORE, MVT::v2i16, Custom);
+ setOperationAction(ISD::STORE, MVT::v4i8, Custom);
- setOperationAction(ISD::BITCAST, MVT::v2i32, Custom);
- setOperationAction(ISD::BITCAST, MVT::v4i16, Custom);
- setOperationAction(ISD::BITCAST, MVT::v8i8, Custom);
+ setOperationAction(ISD::BITCAST, MVT::v2i32, Custom);
+ setOperationAction(ISD::BITCAST, MVT::v4i16, Custom);
+ setOperationAction(ISD::BITCAST, MVT::v8i8, Custom);
if (!Subtarget.hasAVX512())
setOperationAction(ISD::BITCAST, MVT::v16i1, Custom);
@@ -1326,42 +1327,41 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v2i32, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v2i64, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v4i64, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v8i64, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v16i32, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v16i64, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v2i32, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v2i64, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v4i64, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v8i64, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v16i32, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v16i64, Custom);
// In the customized shift lowering, the legal v4i32/v2i64 cases
// in AVX2 will be recognized.
- for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
- setOperationAction(ISD::SRL, VT, Custom);
- setOperationAction(ISD::SHL, VT, Custom);
- setOperationAction(ISD::SRA, VT, Custom);
- if (VT == MVT::v2i64)
- continue;
- setOperationAction(ISD::ROTL, VT, Custom);
- setOperationAction(ISD::ROTR, VT, Custom);
- setOperationAction(ISD::FSHL, VT, Custom);
- setOperationAction(ISD::FSHR, VT, Custom);
+ for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
+ setOperationAction(ISD::SRL, VT, Custom);
+ setOperationAction(ISD::SHL, VT, Custom);
+ setOperationAction(ISD::SRA, VT, Custom);
+ if (VT == MVT::v2i64) continue;
+ setOperationAction(ISD::ROTL, VT, Custom);
+ setOperationAction(ISD::ROTR, VT, Custom);
+ setOperationAction(ISD::FSHL, VT, Custom);
+ setOperationAction(ISD::FSHR, VT, Custom);
}
- setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal);
- setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal);
}
if (!Subtarget.useSoftFloat() && Subtarget.hasGFNI()) {
@@ -1379,21 +1379,21 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
if (!Subtarget.useSoftFloat() && Subtarget.hasSSSE3()) {
- setOperationAction(ISD::ABS, MVT::v16i8, Legal);
- setOperationAction(ISD::ABS, MVT::v8i16, Legal);
- setOperationAction(ISD::ABS, MVT::v4i32, Legal);
+ setOperationAction(ISD::ABS, MVT::v16i8, Legal);
+ setOperationAction(ISD::ABS, MVT::v8i16, Legal);
+ setOperationAction(ISD::ABS, MVT::v4i32, Legal);
for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
- setOperationAction(ISD::BITREVERSE, VT, Custom);
- setOperationAction(ISD::CTLZ, VT, Custom);
+ setOperationAction(ISD::BITREVERSE, VT, Custom);
+ setOperationAction(ISD::CTLZ, VT, Custom);
}
- setOperationAction(ISD::BITREVERSE, MVT::i128, Custom);
+ setOperationAction(ISD::BITREVERSE, MVT::i128, Custom);
// These might be better off as horizontal vector ops.
- setOperationAction(ISD::ADD, MVT::i16, Custom);
- setOperationAction(ISD::ADD, MVT::i32, Custom);
- setOperationAction(ISD::SUB, MVT::i16, Custom);
- setOperationAction(ISD::SUB, MVT::i32, Custom);
+ setOperationAction(ISD::ADD, MVT::i16, Custom);
+ setOperationAction(ISD::ADD, MVT::i32, Custom);
+ setOperationAction(ISD::SUB, MVT::i16, Custom);
+ setOperationAction(ISD::SUB, MVT::i32, Custom);
}
if (Subtarget.hasNDD()) {
// Enable custom lowering for scalar USUBSAT to optimize usub.sat(X,1)
@@ -1405,55 +1405,55 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
- setOperationAction(ISD::FFLOOR, RoundedTy, Legal);
- setOperationAction(ISD::STRICT_FFLOOR, RoundedTy, Legal);
- setOperationAction(ISD::FCEIL, RoundedTy, Legal);
- setOperationAction(ISD::STRICT_FCEIL, RoundedTy, Legal);
- setOperationAction(ISD::FTRUNC, RoundedTy, Legal);
- setOperationAction(ISD::STRICT_FTRUNC, RoundedTy, Legal);
- setOperationAction(ISD::FRINT, RoundedTy, Legal);
- setOperationAction(ISD::STRICT_FRINT, RoundedTy, Legal);
- setOperationAction(ISD::FNEARBYINT, RoundedTy, Legal);
- setOperationAction(ISD::STRICT_FNEARBYINT, RoundedTy, Legal);
- setOperationAction(ISD::FROUNDEVEN, RoundedTy, Legal);
- setOperationAction(ISD::STRICT_FROUNDEVEN, RoundedTy, Legal);
-
- setOperationAction(ISD::FROUND, RoundedTy, Custom);
- }
-
- setOperationAction(ISD::SMAX, MVT::v16i8, Legal);
- setOperationAction(ISD::SMAX, MVT::v4i32, Legal);
- setOperationAction(ISD::UMAX, MVT::v8i16, Legal);
- setOperationAction(ISD::UMAX, MVT::v4i32, Legal);
- setOperationAction(ISD::SMIN, MVT::v16i8, Legal);
- setOperationAction(ISD::SMIN, MVT::v4i32, Legal);
- setOperationAction(ISD::UMIN, MVT::v8i16, Legal);
- setOperationAction(ISD::UMIN, MVT::v4i32, Legal);
-
- setOperationAction(ISD::UADDSAT, MVT::v4i32, Custom);
- setOperationAction(ISD::SADDSAT, MVT::v2i64, Custom);
- setOperationAction(ISD::SSUBSAT, MVT::v2i64, Custom);
+ setOperationAction(ISD::FFLOOR, RoundedTy, Legal);
+ setOperationAction(ISD::STRICT_FFLOOR, RoundedTy, Legal);
+ setOperationAction(ISD::FCEIL, RoundedTy, Legal);
+ setOperationAction(ISD::STRICT_FCEIL, RoundedTy, Legal);
+ setOperationAction(ISD::FTRUNC, RoundedTy, Legal);
+ setOperationAction(ISD::STRICT_FTRUNC, RoundedTy, Legal);
+ setOperationAction(ISD::FRINT, RoundedTy, Legal);
+ setOperationAction(ISD::STRICT_FRINT, RoundedTy, Legal);
+ setOperationAction(ISD::FNEARBYINT, RoundedTy, Legal);
+ setOperationAction(ISD::STRICT_FNEARBYINT, RoundedTy, Legal);
+ setOperationAction(ISD::FROUNDEVEN, RoundedTy, Legal);
+ setOperationAction(ISD::STRICT_FROUNDEVEN, RoundedTy, Legal);
+
+ setOperationAction(ISD::FROUND, RoundedTy, Custom);
+ }
+
+ setOperationAction(ISD::SMAX, MVT::v16i8, Legal);
+ setOperationAction(ISD::SMAX, MVT::v4i32, Legal);
+ setOperationAction(ISD::UMAX, MVT::v8i16, Legal);
+ setOperationAction(ISD::UMAX, MVT::v4i32, Legal);
+ setOperationAction(ISD::SMIN, MVT::v16i8, Legal);
+ setOperationAction(ISD::SMIN, MVT::v4i32, Legal);
+ setOperationAction(ISD::UMIN, MVT::v8i16, Legal);
+ setOperationAction(ISD::UMIN, MVT::v4i32, Legal);
+
+ setOperationAction(ISD::UADDSAT, MVT::v4i32, Custom);
+ setOperationAction(ISD::SADDSAT, MVT::v2i64, Custom);
+ setOperationAction(ISD::SSUBSAT, MVT::v2i64, Custom);
// FIXME: Do we need to handle scalar-to-vector here?
- setOperationAction(ISD::MUL, MVT::v4i32, Legal);
- setOperationAction(ISD::SMULO, MVT::v2i32, Custom);
+ setOperationAction(ISD::MUL, MVT::v4i32, Legal);
+ setOperationAction(ISD::SMULO, MVT::v2i32, Custom);
// We directly match byte blends in the backend as they match the VSELECT
// condition form.
- setOperationAction(ISD::VSELECT, MVT::v16i8, Legal);
+ setOperationAction(ISD::VSELECT, MVT::v16i8, Legal);
// SSE41 brings specific instructions for doing vector sign extend even in
// cases where we don't have SRA.
- for (auto VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
+ for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal);
setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal);
}
// SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
- for (auto LoadExtOp : {ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
- setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8, Legal);
- setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8, Legal);
- setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8, Legal);
+ for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
+ setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8, Legal);
setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal);
setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal);
setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i32, Legal);
@@ -1462,69 +1462,69 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
if (Subtarget.is64Bit() && !Subtarget.hasAVX512()) {
// We need to scalarize v4i64->v432 uint_to_fp using cvtsi2ss, but we can
// do the pre and post work in the vector domain.
- setOperationAction(ISD::UINT_TO_FP, MVT::v4i64, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v4i64, Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i64, Custom);
// We need to mark SINT_TO_FP as Custom even though we want to expand it
// so that DAG combine doesn't try to turn it into uint_to_fp.
- setOperationAction(ISD::SINT_TO_FP, MVT::v4i64, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v4i64, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i64, Custom);
}
}
if (!Subtarget.useSoftFloat() && Subtarget.hasSSE42()) {
- setOperationAction(ISD::UADDSAT, MVT::v2i64, Custom);
+ setOperationAction(ISD::UADDSAT, MVT::v2i64, Custom);
}
if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
- for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v32i8,
- MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
+ for (MVT VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
+ MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
setOperationAction(ISD::ROTL, VT, VT.is128BitVector() ? Legal : Custom);
setOperationAction(ISD::ROTR, VT, Custom);
}
// XOP can efficiently perform BITREVERSE with VPPERM.
- for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64})
+ for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 })
setOperationAction(ISD::BITREVERSE, VT, Custom);
}
if (!Subtarget.useSoftFloat() && Subtarget.hasAVX()) {
bool HasInt256 = Subtarget.hasInt256();
- addRegisterClass(MVT::v32i8, Subtarget.hasVLX() ? &X86::VR256XRegClass
- : &X86::VR256RegClass);
+ addRegisterClass(MVT::v32i8, Subtarget.hasVLX() ? &X86::VR256XRegClass
+ : &X86::VR256RegClass);
addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass
: &X86::VR256RegClass);
addRegisterClass(MVT::v16f16, Subtarget.hasVLX() ? &X86::VR256XRegClass
: &X86::VR256RegClass);
- addRegisterClass(MVT::v8i32, Subtarget.hasVLX() ? &X86::VR256XRegClass
- : &X86::VR256RegClass);
- addRegisterClass(MVT::v8f32, Subtarget.hasVLX() ? &X86::VR256XRegClass
- : &X86::VR256RegClass);
- addRegisterClass(MVT::v4i64, Subtarget.hasVLX() ? &X86::VR256XRegClass
- : &X86::VR256RegClass);
- addRegisterClass(MVT::v4f64, Subtarget.hasVLX() ? &X86::VR256XRegClass
- : &X86::VR256RegClass);
-
- for (auto VT : {MVT::v8f32, MVT::v4f64}) {
- setOperationAction(ISD::FFLOOR, VT, Legal);
- setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
- setOperationAction(ISD::FCEIL, VT, Legal);
- setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
- setOperationAction(ISD::FTRUNC, VT, Legal);
- setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
- setOperationAction(ISD::FRINT, VT, Legal);
- setOperationAction(ISD::STRICT_FRINT, VT, Legal);
- setOperationAction(ISD::FNEARBYINT, VT, Legal);
+ addRegisterClass(MVT::v8i32, Subtarget.hasVLX() ? &X86::VR256XRegClass
+ : &X86::VR256RegClass);
+ addRegisterClass(MVT::v8f32, Subtarget.hasVLX() ? &X86::VR256XRegClass
+ : &X86::VR256RegClass);
+ addRegisterClass(MVT::v4i64, Subtarget.hasVLX() ? &X86::VR256XRegClass
+ : &X86::VR256RegClass);
+ addRegisterClass(MVT::v4f64, Subtarget.hasVLX() ? &X86::VR256XRegClass
+ : &X86::VR256RegClass);
+
+ for (auto VT : { MVT::v8f32, MVT::v4f64 }) {
+ setOperationAction(ISD::FFLOOR, VT, Legal);
+ setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
+ setOperationAction(ISD::FCEIL, VT, Legal);
+ setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
+ setOperationAction(ISD::FTRUNC, VT, Legal);
+ setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
+ setOperationAction(ISD::FRINT, VT, Legal);
+ setOperationAction(ISD::STRICT_FRINT, VT, Legal);
+ setOperationAction(ISD::FNEARBYINT, VT, Legal);
setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
- setOperationAction(ISD::FROUNDEVEN, VT, Legal);
+ setOperationAction(ISD::FROUNDEVEN, VT, Legal);
setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
- setOperationAction(ISD::FROUND, VT, Custom);
+ setOperationAction(ISD::FROUND, VT, Custom);
- setOperationAction(ISD::FNEG, VT, Custom);
- setOperationAction(ISD::FABS, VT, Custom);
- setOperationAction(ISD::FCOPYSIGN, VT, Custom);
- setOperationAction(ISD::FCANONICALIZE, VT, Expand);
+ setOperationAction(ISD::FNEG, VT, Custom);
+ setOperationAction(ISD::FABS, VT, Custom);
+ setOperationAction(ISD::FCOPYSIGN, VT, Custom);
+ setOperationAction(ISD::FCANONICALIZE, VT, Expand);
SetFPMinMaxAction(VT);
}
@@ -1539,90 +1539,89 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted
// even though v8i16 is a legal type.
- setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i16, MVT::v8i32);
- setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i16, MVT::v8i32);
+ setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i16, MVT::v8i32);
+ setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i16, MVT::v8i32);
setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i16, MVT::v8i32);
setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i16, MVT::v8i32);
- setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Custom);
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i32, Custom);
-
- setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Custom);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i32, Custom);
- setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Expand);
- setOperationAction(ISD::FP_ROUND, MVT::v8f16, Expand);
- setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Custom);
-
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f32, Legal);
- setOperationAction(ISD::STRICT_FADD, MVT::v8f32, Legal);
- setOperationAction(ISD::STRICT_FADD, MVT::v4f64, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::v8f32, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::v4f64, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::v8f32, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::v4f64, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::v8f32, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::v4f64, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::v8f32, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::v4f64, Legal);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i32, Custom);
+
+ setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Custom);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i32, Custom);
+ setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Expand);
+ setOperationAction(ISD::FP_ROUND, MVT::v8f16, Expand);
+ setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Custom);
+
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f32, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::v8f32, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::v4f64, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::v8f32, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::v4f64, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::v8f32, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::v4f64, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::v8f32, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::v4f64, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::v8f32, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::v4f64, Legal);
if (!Subtarget.hasAVX512())
setOperationAction(ISD::BITCAST, MVT::v32i1, Custom);
// In the customized shift lowering, the legal v8i32/v4i64 cases
// in AVX2 will be recognized.
- for (auto VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
- setOperationAction(ISD::SRL, VT, Custom);
- setOperationAction(ISD::SHL, VT, Custom);
- setOperationAction(ISD::SRA, VT, Custom);
- setOperationAction(ISD::ABDS, VT, Custom);
- setOperationAction(ISD::ABDU, VT, Custom);
- setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
- setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
- setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
- setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
- setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
- setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
- setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
- setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
- if (VT == MVT::v4i64)
- continue;
- setOperationAction(ISD::ROTL, VT, Custom);
- setOperationAction(ISD::ROTR, VT, Custom);
- setOperationAction(ISD::FSHL, VT, Custom);
- setOperationAction(ISD::FSHR, VT, Custom);
+ for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
+ setOperationAction(ISD::SRL, VT, Custom);
+ setOperationAction(ISD::SHL, VT, Custom);
+ setOperationAction(ISD::SRA, VT, Custom);
+ setOperationAction(ISD::ABDS, VT, Custom);
+ setOperationAction(ISD::ABDU, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
+ if (VT == MVT::v4i64) continue;
+ setOperationAction(ISD::ROTL, VT, Custom);
+ setOperationAction(ISD::ROTR, VT, Custom);
+ setOperationAction(ISD::FSHL, VT, Custom);
+ setOperationAction(ISD::FSHR, VT, Custom);
}
// These types need custom splitting if their input is a 128-bit vector.
- setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
- setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
- setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
- setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
-
- setOperationAction(ISD::SELECT, MVT::v4f64, Custom);
- setOperationAction(ISD::SELECT, MVT::v4i64, Custom);
- setOperationAction(ISD::SELECT, MVT::v8i32, Custom);
- setOperationAction(ISD::SELECT, MVT::v16i16, Custom);
- setOperationAction(ISD::SELECT, MVT::v16f16, Custom);
- setOperationAction(ISD::SELECT, MVT::v32i8, Custom);
- setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
-
- for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
- setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
- setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
- setOperationAction(ISD::ANY_EXTEND, VT, Custom);
- }
-
- setOperationAction(ISD::TRUNCATE, MVT::v32i8, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v32i16, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v32i32, Custom);
- setOperationAction(ISD::TRUNCATE, MVT::v32i64, Custom);
-
- for (auto VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
- setOperationAction(ISD::SETCC, VT, Custom);
- setOperationAction(ISD::CTPOP, VT, Custom);
- setOperationAction(ISD::CTLZ, VT, Custom);
- setOperationAction(ISD::BITREVERSE, VT, Custom);
+ setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
+ setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
+
+ setOperationAction(ISD::SELECT, MVT::v4f64, Custom);
+ setOperationAction(ISD::SELECT, MVT::v4i64, Custom);
+ setOperationAction(ISD::SELECT, MVT::v8i32, Custom);
+ setOperationAction(ISD::SELECT, MVT::v16i16, Custom);
+ setOperationAction(ISD::SELECT, MVT::v16f16, Custom);
+ setOperationAction(ISD::SELECT, MVT::v32i8, Custom);
+ setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
+
+ for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
+ setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
+ setOperationAction(ISD::ANY_EXTEND, VT, Custom);
+ }
+
+ setOperationAction(ISD::TRUNCATE, MVT::v32i8, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v32i16, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v32i32, Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v32i64, Custom);
+
+ for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
+ setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::CTPOP, VT, Custom);
+ setOperationAction(ISD::CTLZ, VT, Custom);
+ setOperationAction(ISD::BITREVERSE, VT, Custom);
// The condition codes aren't legal in SSE/AVX and under AVX512 we use
// setcc all the way to isel and prefer SETGT in some isel patterns.
@@ -1630,64 +1629,64 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setCondCodeAction(ISD::SETLE, VT, Custom);
}
- setOperationAction(ISD::SETCC, MVT::v4f64, Custom);
- setOperationAction(ISD::SETCC, MVT::v8f32, Custom);
- setOperationAction(ISD::STRICT_FSETCC, MVT::v4f64, Custom);
- setOperationAction(ISD::STRICT_FSETCC, MVT::v8f32, Custom);
+ setOperationAction(ISD::SETCC, MVT::v4f64, Custom);
+ setOperationAction(ISD::SETCC, MVT::v8f32, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, MVT::v4f64, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, MVT::v8f32, Custom);
setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f64, Custom);
setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f32, Custom);
if (Subtarget.hasAnyFMA()) {
- for (auto VT : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32, MVT::v2f64,
- MVT::v4f64}) {
+ for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32,
+ MVT::v2f64, MVT::v4f64 }) {
setOperationAction(ISD::FMA, VT, Legal);
setOperationAction(ISD::STRICT_FMA, VT, Legal);
}
}
- for (auto VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
+ for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom);
setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom);
}
- setOperationAction(ISD::MUL, MVT::v4i64, Custom);
- setOperationAction(ISD::MUL, MVT::v8i32, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::MUL, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::MUL, MVT::v32i8, Custom);
-
- setOperationAction(ISD::MULHU, MVT::v8i32, Custom);
- setOperationAction(ISD::MULHS, MVT::v8i32, Custom);
- setOperationAction(ISD::MULHU, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::MULHS, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::MULHU, MVT::v32i8, Custom);
- setOperationAction(ISD::MULHS, MVT::v32i8, Custom);
- setOperationAction(ISD::AVGCEILU, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::AVGCEILU, MVT::v32i8, HasInt256 ? Legal : Custom);
-
- setOperationAction(ISD::SMULO, MVT::v32i8, Custom);
- setOperationAction(ISD::UMULO, MVT::v32i8, Custom);
-
- setOperationAction(ISD::ABS, MVT::v4i64, Custom);
- setOperationAction(ISD::SMAX, MVT::v4i64, Custom);
- setOperationAction(ISD::UMAX, MVT::v4i64, Custom);
- setOperationAction(ISD::SMIN, MVT::v4i64, Custom);
- setOperationAction(ISD::UMIN, MVT::v4i64, Custom);
-
- setOperationAction(ISD::UADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::SADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::USUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::SSUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::UADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::SADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::USUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::SSUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
- setOperationAction(ISD::UADDSAT, MVT::v8i32, Custom);
- setOperationAction(ISD::USUBSAT, MVT::v8i32, Custom);
- setOperationAction(ISD::UADDSAT, MVT::v4i64, Custom);
- setOperationAction(ISD::USUBSAT, MVT::v4i64, Custom);
-
- for (auto VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32}) {
- setOperationAction(ISD::ABS, VT, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::MUL, MVT::v4i64, Custom);
+ setOperationAction(ISD::MUL, MVT::v8i32, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::MUL, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::MUL, MVT::v32i8, Custom);
+
+ setOperationAction(ISD::MULHU, MVT::v8i32, Custom);
+ setOperationAction(ISD::MULHS, MVT::v8i32, Custom);
+ setOperationAction(ISD::MULHU, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::MULHS, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::MULHU, MVT::v32i8, Custom);
+ setOperationAction(ISD::MULHS, MVT::v32i8, Custom);
+ setOperationAction(ISD::AVGCEILU, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::AVGCEILU, MVT::v32i8, HasInt256 ? Legal : Custom);
+
+ setOperationAction(ISD::SMULO, MVT::v32i8, Custom);
+ setOperationAction(ISD::UMULO, MVT::v32i8, Custom);
+
+ setOperationAction(ISD::ABS, MVT::v4i64, Custom);
+ setOperationAction(ISD::SMAX, MVT::v4i64, Custom);
+ setOperationAction(ISD::UMAX, MVT::v4i64, Custom);
+ setOperationAction(ISD::SMIN, MVT::v4i64, Custom);
+ setOperationAction(ISD::UMIN, MVT::v4i64, Custom);
+
+ setOperationAction(ISD::UADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::SADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::USUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::SSUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::UADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::SADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::USUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::SSUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
+ setOperationAction(ISD::UADDSAT, MVT::v8i32, Custom);
+ setOperationAction(ISD::USUBSAT, MVT::v8i32, Custom);
+ setOperationAction(ISD::UADDSAT, MVT::v4i64, Custom);
+ setOperationAction(ISD::USUBSAT, MVT::v4i64, Custom);
+
+ for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) {
+ setOperationAction(ISD::ABS, VT, HasInt256 ? Legal : Custom);
setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom);
setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom);
setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom);
@@ -1706,41 +1705,41 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i32, Custom);
// AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
- for (auto LoadExtOp : {ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
+ for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
setLoadExtAction(LoadExtOp, MVT::v16i16, MVT::v16i8, Legal);
- setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i8, Legal);
- setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i8, Legal);
- setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i16, Legal);
- setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i16, Legal);
- setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i32, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i8, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i16, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i16, Legal);
+ setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i32, Legal);
}
}
- for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, MVT::v4f32,
- MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
- setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
+ for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
+ MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) {
+ setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
setOperationAction(ISD::MSTORE, VT, Legal);
}
// Extract subvector is special because the value type
// (result) is 128-bit but the source is 256-bit wide.
- for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v8f16,
- MVT::v4f32, MVT::v2f64}) {
+ for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
+ MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
}
// Custom lower several nodes for 256-bit types.
- for (MVT VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v16f16,
- MVT::v8f32, MVT::v4f64}) {
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
- setOperationAction(ISD::VSELECT, VT, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
+ for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
+ MVT::v16f16, MVT::v8f32, MVT::v4f64 }) {
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::VSELECT, VT, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
- setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
- setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
- setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
- setOperationAction(ISD::STORE, VT, Custom);
+ setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
+ setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
+ setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
+ setOperationAction(ISD::STORE, VT, Custom);
}
setF16Action(MVT::v16f16, Expand);
setOperationAction(ISD::FNEG, MVT::v16f16, Custom);
@@ -1771,9 +1770,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::MGATHER, MVT::v2f32, Custom);
setOperationAction(ISD::MGATHER, MVT::v2i32, Custom);
- for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
- MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64})
- setOperationAction(ISD::MGATHER, VT, Custom);
+ for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
+ MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
+ setOperationAction(ISD::MGATHER, VT, Custom);
}
if (Subtarget.hasGFNI()) {
@@ -1784,13 +1783,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
if (!Subtarget.useSoftFloat() && !Subtarget.hasFP16() &&
Subtarget.hasF16C()) {
- for (MVT VT : {MVT::f16, MVT::v2f16, MVT::v4f16, MVT::v8f16}) {
- setOperationAction(ISD::FP_ROUND, VT, Custom);
- setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom);
+ for (MVT VT : { MVT::f16, MVT::v2f16, MVT::v4f16, MVT::v8f16 }) {
+ setOperationAction(ISD::FP_ROUND, VT, Custom);
+ setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom);
}
- for (MVT VT : {MVT::f32, MVT::v2f32, MVT::v4f32, MVT::v8f32}) {
- setOperationAction(ISD::FP_EXTEND, VT, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, VT, Custom);
+ for (MVT VT : { MVT::f32, MVT::v2f32, MVT::v4f32, MVT::v8f32 }) {
+ setOperationAction(ISD::FP_EXTEND, VT, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, VT, Custom);
}
for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
setOperationPromotedToType(Opc, MVT::v8f16, MVT::v8f32);
@@ -1804,28 +1803,28 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// available with AVX512. 512-bit vectors are in a separate block controlled
// by useAVX512Regs.
if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
- addRegisterClass(MVT::v1i1, &X86::VK1RegClass);
- addRegisterClass(MVT::v2i1, &X86::VK2RegClass);
- addRegisterClass(MVT::v4i1, &X86::VK4RegClass);
- addRegisterClass(MVT::v8i1, &X86::VK8RegClass);
- addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
+ addRegisterClass(MVT::v1i1, &X86::VK1RegClass);
+ addRegisterClass(MVT::v2i1, &X86::VK2RegClass);
+ addRegisterClass(MVT::v4i1, &X86::VK4RegClass);
+ addRegisterClass(MVT::v8i1, &X86::VK8RegClass);
+ addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
- setOperationAction(ISD::SELECT, MVT::v1i1, Custom);
+ setOperationAction(ISD::SELECT, MVT::v1i1, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom);
- setOperationAction(ISD::BUILD_VECTOR, MVT::v1i1, Custom);
-
- setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i1, MVT::v8i32);
- setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i1, MVT::v8i32);
- setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v4i1, MVT::v4i32);
- setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v4i1, MVT::v4i32);
- setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i1, MVT::v8i32);
- setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i1, MVT::v8i32);
- setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v4i1, MVT::v4i32);
- setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v4i1, MVT::v4i32);
- setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Custom);
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i1, Custom);
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i1, Custom);
+ setOperationAction(ISD::BUILD_VECTOR, MVT::v1i1, Custom);
+
+ setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i1, MVT::v8i32);
+ setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i1, MVT::v8i32);
+ setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v4i1, MVT::v4i32);
+ setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v4i1, MVT::v4i32);
+ setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i1, MVT::v8i32);
+ setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i1, MVT::v8i32);
+ setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v4i1, MVT::v4i32);
+ setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v4i1, MVT::v4i32);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i1, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i1, Custom);
setOperationAction(ISD::FCANONICALIZE, MVT::v8f16, Expand);
setOperationAction(ISD::FCANONICALIZE, MVT::v16f16, Expand);
setOperationAction(ISD::FCANONICALIZE, MVT::v32f16, Expand);
@@ -1844,29 +1843,29 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
// Extends of v16i1/v8i1/v4i1/v2i1 to 128-bit vectors.
- for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
+ for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
- setOperationAction(ISD::ANY_EXTEND, VT, Custom);
+ setOperationAction(ISD::ANY_EXTEND, VT, Custom);
}
- for (auto VT : {MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1})
- setOperationAction(ISD::VSELECT, VT, Expand);
+ for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 })
+ setOperationAction(ISD::VSELECT, VT, Expand);
- for (auto VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1}) {
- setOperationAction(ISD::SETCC, VT, Custom);
- setOperationAction(ISD::SELECT, VT, Custom);
- setOperationAction(ISD::TRUNCATE, VT, Custom);
+ for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {
+ setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::TRUNCATE, VT, Custom);
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
- setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
}
- for (auto VT : {MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1})
+ for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 })
setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
}
if (Subtarget.hasDQI() && Subtarget.hasVLX()) {
@@ -1884,27 +1883,27 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
- addRegisterClass(MVT::v8i64, &X86::VR512RegClass);
- addRegisterClass(MVT::v8f64, &X86::VR512RegClass);
+ addRegisterClass(MVT::v8i64, &X86::VR512RegClass);
+ addRegisterClass(MVT::v8f64, &X86::VR512RegClass);
addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
addRegisterClass(MVT::v32f16, &X86::VR512RegClass);
- addRegisterClass(MVT::v64i8, &X86::VR512RegClass);
+ addRegisterClass(MVT::v64i8, &X86::VR512RegClass);
for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD}) {
- setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8, Legal);
+ setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8, Legal);
setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i16, Legal);
- setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i8, Legal);
- setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i16, Legal);
- setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i32, Legal);
+ setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i8, Legal);
+ setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i16, Legal);
+ setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i32, Legal);
if (HasBWI)
setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal);
}
- for (MVT VT : {MVT::v16f32, MVT::v8f64}) {
+ for (MVT VT : { MVT::v16f32, MVT::v8f64 }) {
SetFPMinMaxAction(VT);
- setOperationAction(ISD::FNEG, VT, Custom);
- setOperationAction(ISD::FABS, VT, Custom);
- setOperationAction(ISD::FMA, VT, Legal);
+ setOperationAction(ISD::FNEG, VT, Custom);
+ setOperationAction(ISD::FABS, VT, Custom);
+ setOperationAction(ISD::FMA, VT, Legal);
setOperationAction(ISD::STRICT_FMA, VT, Legal);
setOperationAction(ISD::FCOPYSIGN, VT, Custom);
setOperationAction(ISD::FCANONICALIZE, VT, Expand);
@@ -1932,93 +1931,93 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SELECT, MVT::i512, Custom);
setOperationAction(ISD::BITREVERSE, MVT::i512, Custom);
- for (MVT VT : {MVT::v16i1, MVT::v16i8}) {
- setOperationPromotedToType(ISD::FP_TO_SINT, VT, MVT::v16i32);
- setOperationPromotedToType(ISD::FP_TO_UINT, VT, MVT::v16i32);
+ for (MVT VT : { MVT::v16i1, MVT::v16i8 }) {
+ setOperationPromotedToType(ISD::FP_TO_SINT , VT, MVT::v16i32);
+ setOperationPromotedToType(ISD::FP_TO_UINT , VT, MVT::v16i32);
setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, VT, MVT::v16i32);
setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, VT, MVT::v16i32);
}
- for (MVT VT : {MVT::v16i16, MVT::v16i32}) {
- setOperationAction(ISD::FP_TO_SINT, VT, Custom);
- setOperationAction(ISD::FP_TO_UINT, VT, Custom);
+ for (MVT VT : { MVT::v16i16, MVT::v16i32 }) {
+ setOperationAction(ISD::FP_TO_SINT, VT, Custom);
+ setOperationAction(ISD::FP_TO_UINT, VT, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
}
- setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Custom);
- setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i32, Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i32, Custom);
- setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Custom);
-
- setOperationAction(ISD::STRICT_FADD, MVT::v16f32, Legal);
- setOperationAction(ISD::STRICT_FADD, MVT::v8f64, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::v16f32, Legal);
- setOperationAction(ISD::STRICT_FSUB, MVT::v8f64, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::v16f32, Legal);
- setOperationAction(ISD::STRICT_FMUL, MVT::v8f64, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::v16f32, Legal);
- setOperationAction(ISD::STRICT_FDIV, MVT::v8f64, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::v16f32, Legal);
- setOperationAction(ISD::STRICT_FSQRT, MVT::v8f64, Legal);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f32, Legal);
-
- setTruncStoreAction(MVT::v8i64, MVT::v8i8, Legal);
- setTruncStoreAction(MVT::v8i64, MVT::v8i16, Legal);
- setTruncStoreAction(MVT::v8i64, MVT::v8i32, Legal);
- setTruncStoreAction(MVT::v16i32, MVT::v16i8, Legal);
- setTruncStoreAction(MVT::v16i32, MVT::v16i16, Legal);
+ setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Custom);
+
+ setOperationAction(ISD::STRICT_FADD, MVT::v16f32, Legal);
+ setOperationAction(ISD::STRICT_FADD, MVT::v8f64, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::v16f32, Legal);
+ setOperationAction(ISD::STRICT_FSUB, MVT::v8f64, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::v16f32, Legal);
+ setOperationAction(ISD::STRICT_FMUL, MVT::v8f64, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::v16f32, Legal);
+ setOperationAction(ISD::STRICT_FDIV, MVT::v8f64, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::v16f32, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, MVT::v8f64, Legal);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f32, Legal);
+
+ setTruncStoreAction(MVT::v8i64, MVT::v8i8, Legal);
+ setTruncStoreAction(MVT::v8i64, MVT::v8i16, Legal);
+ setTruncStoreAction(MVT::v8i64, MVT::v8i32, Legal);
+ setTruncStoreAction(MVT::v16i32, MVT::v16i8, Legal);
+ setTruncStoreAction(MVT::v16i32, MVT::v16i16, Legal);
if (HasBWI)
- setTruncStoreAction(MVT::v32i16, MVT::v32i8, Legal);
+ setTruncStoreAction(MVT::v32i16, MVT::v32i8, Legal);
// With 512-bit vectors and no VLX, we prefer to widen MLOAD/MSTORE
// to 512-bit rather than use the AVX2 instructions so that we can use
// k-masks.
if (!Subtarget.hasVLX()) {
for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
- MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
- setOperationAction(ISD::MLOAD, VT, Custom);
+ MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
+ setOperationAction(ISD::MLOAD, VT, Custom);
setOperationAction(ISD::MSTORE, VT, Custom);
}
}
- setOperationAction(ISD::TRUNCATE, MVT::v8i32, Legal);
- setOperationAction(ISD::TRUNCATE, MVT::v16i16, Legal);
- setOperationAction(ISD::TRUNCATE, MVT::v32i8, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::TRUNCATE, MVT::v8i32, Legal);
+ setOperationAction(ISD::TRUNCATE, MVT::v16i16, Legal);
+ setOperationAction(ISD::TRUNCATE, MVT::v32i8, HasBWI ? Legal : Custom);
setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom);
setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
- setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
- setOperationAction(ISD::ANY_EXTEND, MVT::v32i16, Custom);
- setOperationAction(ISD::ANY_EXTEND, MVT::v16i32, Custom);
- setOperationAction(ISD::ANY_EXTEND, MVT::v8i64, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
+ setOperationAction(ISD::ANY_EXTEND, MVT::v32i16, Custom);
+ setOperationAction(ISD::ANY_EXTEND, MVT::v16i32, Custom);
+ setOperationAction(ISD::ANY_EXTEND, MVT::v8i64, Custom);
setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom);
setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
- setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
+ setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
if (HasBWI) {
// Extends from v64i1 masks to 512-bit vectors.
- setOperationAction(ISD::SIGN_EXTEND, MVT::v64i8, Custom);
- setOperationAction(ISD::ZERO_EXTEND, MVT::v64i8, Custom);
- setOperationAction(ISD::ANY_EXTEND, MVT::v64i8, Custom);
- }
-
- for (auto VT : {MVT::v16f32, MVT::v8f64}) {
- setOperationAction(ISD::FFLOOR, VT, Legal);
- setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
- setOperationAction(ISD::FCEIL, VT, Legal);
- setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
- setOperationAction(ISD::FTRUNC, VT, Legal);
- setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
- setOperationAction(ISD::FRINT, VT, Legal);
- setOperationAction(ISD::STRICT_FRINT, VT, Legal);
- setOperationAction(ISD::FNEARBYINT, VT, Legal);
+ setOperationAction(ISD::SIGN_EXTEND, MVT::v64i8, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, MVT::v64i8, Custom);
+ setOperationAction(ISD::ANY_EXTEND, MVT::v64i8, Custom);
+ }
+
+ for (auto VT : { MVT::v16f32, MVT::v8f64 }) {
+ setOperationAction(ISD::FFLOOR, VT, Legal);
+ setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
+ setOperationAction(ISD::FCEIL, VT, Legal);
+ setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
+ setOperationAction(ISD::FTRUNC, VT, Legal);
+ setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
+ setOperationAction(ISD::FRINT, VT, Legal);
+ setOperationAction(ISD::STRICT_FRINT, VT, Legal);
+ setOperationAction(ISD::FNEARBYINT, VT, Legal);
setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
- setOperationAction(ISD::FROUNDEVEN, VT, Legal);
+ setOperationAction(ISD::FROUNDEVEN, VT, Legal);
setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
- setOperationAction(ISD::FROUND, VT, Custom);
+ setOperationAction(ISD::FROUND, VT, Custom);
}
for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
@@ -2028,13 +2027,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::ADD, MVT::v32i16, HasBWI ? Legal : Custom);
setOperationAction(ISD::SUB, MVT::v32i16, HasBWI ? Legal : Custom);
- setOperationAction(ISD::ADD, MVT::v64i8, HasBWI ? Legal : Custom);
- setOperationAction(ISD::SUB, MVT::v64i8, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::ADD, MVT::v64i8, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::SUB, MVT::v64i8, HasBWI ? Legal : Custom);
- setOperationAction(ISD::MUL, MVT::v8i64, Custom);
+ setOperationAction(ISD::MUL, MVT::v8i64, Custom);
setOperationAction(ISD::MUL, MVT::v16i32, Legal);
setOperationAction(ISD::MUL, MVT::v32i16, HasBWI ? Legal : Custom);
- setOperationAction(ISD::MUL, MVT::v64i8, Custom);
+ setOperationAction(ISD::MUL, MVT::v64i8, Custom);
setOperationAction(ISD::MULHU, MVT::v8i64, Custom);
setOperationAction(ISD::UMUL_LOHI, MVT::v8i64, Custom);
@@ -2043,32 +2042,32 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::MULHS, MVT::v16i32, Custom);
setOperationAction(ISD::MULHS, MVT::v32i16, HasBWI ? Legal : Custom);
setOperationAction(ISD::MULHU, MVT::v32i16, HasBWI ? Legal : Custom);
- setOperationAction(ISD::MULHS, MVT::v64i8, Custom);
- setOperationAction(ISD::MULHU, MVT::v64i8, Custom);
+ setOperationAction(ISD::MULHS, MVT::v64i8, Custom);
+ setOperationAction(ISD::MULHU, MVT::v64i8, Custom);
setOperationAction(ISD::AVGCEILU, MVT::v32i16, HasBWI ? Legal : Custom);
- setOperationAction(ISD::AVGCEILU, MVT::v64i8, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::AVGCEILU, MVT::v64i8, HasBWI ? Legal : Custom);
setOperationAction(ISD::SMULO, MVT::v64i8, Custom);
setOperationAction(ISD::UMULO, MVT::v64i8, Custom);
- for (auto VT : {MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
- setOperationAction(ISD::SRL, VT, Custom);
- setOperationAction(ISD::SHL, VT, Custom);
- setOperationAction(ISD::SRA, VT, Custom);
- setOperationAction(ISD::ROTL, VT, Custom);
- setOperationAction(ISD::ROTR, VT, Custom);
- setOperationAction(ISD::SETCC, VT, Custom);
- setOperationAction(ISD::ABDS, VT, Custom);
- setOperationAction(ISD::ABDU, VT, Custom);
- setOperationAction(ISD::BITREVERSE, VT, Custom);
- setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
- setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
- setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
- setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
- setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
- setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
- setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
- setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
+ for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64 }) {
+ setOperationAction(ISD::SRL, VT, Custom);
+ setOperationAction(ISD::SHL, VT, Custom);
+ setOperationAction(ISD::SRA, VT, Custom);
+ setOperationAction(ISD::ROTL, VT, Custom);
+ setOperationAction(ISD::ROTR, VT, Custom);
+ setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::ABDS, VT, Custom);
+ setOperationAction(ISD::ABDU, VT, Custom);
+ setOperationAction(ISD::BITREVERSE, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
// The condition codes aren't legal in SSE/AVX and under AVX512 we use
// setcc all the way to isel and prefer SETGT in some isel patterns.
@@ -2076,54 +2075,53 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setCondCodeAction(ISD::SETLE, VT, Custom);
}
- setOperationAction(ISD::SETCC, MVT::v8f64, Custom);
- setOperationAction(ISD::SETCC, MVT::v16f32, Custom);
- setOperationAction(ISD::STRICT_FSETCC, MVT::v8f64, Custom);
- setOperationAction(ISD::STRICT_FSETCC, MVT::v16f32, Custom);
+ setOperationAction(ISD::SETCC, MVT::v8f64, Custom);
+ setOperationAction(ISD::SETCC, MVT::v16f32, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, MVT::v8f64, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, MVT::v16f32, Custom);
setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f64, Custom);
setOperationAction(ISD::STRICT_FSETCCS, MVT::v16f32, Custom);
- for (auto VT : {MVT::v16i32, MVT::v8i64}) {
- setOperationAction(ISD::SMAX, VT, Legal);
- setOperationAction(ISD::UMAX, VT, Legal);
- setOperationAction(ISD::SMIN, VT, Legal);
- setOperationAction(ISD::UMIN, VT, Legal);
- setOperationAction(ISD::ROTL, VT, Legal);
- setOperationAction(ISD::ROTR, VT, Legal);
- setOperationAction(ISD::ABS, VT, Legal);
- setOperationAction(ISD::CTPOP, VT, Custom);
- }
-
- for (auto VT : {MVT::v64i8, MVT::v32i16}) {
- setOperationAction(ISD::ABS, VT, HasBWI ? Legal : Custom);
- setOperationAction(ISD::CTPOP, VT,
- Subtarget.hasBITALG() ? Legal : Custom);
- setOperationAction(ISD::CTLZ, VT, Custom);
- setOperationAction(ISD::SMAX, VT, HasBWI ? Legal : Custom);
- setOperationAction(ISD::UMAX, VT, HasBWI ? Legal : Custom);
- setOperationAction(ISD::SMIN, VT, HasBWI ? Legal : Custom);
- setOperationAction(ISD::UMIN, VT, HasBWI ? Legal : Custom);
+ for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
+ setOperationAction(ISD::SMAX, VT, Legal);
+ setOperationAction(ISD::UMAX, VT, Legal);
+ setOperationAction(ISD::SMIN, VT, Legal);
+ setOperationAction(ISD::UMIN, VT, Legal);
+ setOperationAction(ISD::ROTL, VT, Legal);
+ setOperationAction(ISD::ROTR, VT, Legal);
+ setOperationAction(ISD::ABS, VT, Legal);
+ setOperationAction(ISD::CTPOP, VT, Custom);
+ }
+
+ for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
+ setOperationAction(ISD::ABS, VT, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::CTPOP, VT, Subtarget.hasBITALG() ? Legal : Custom);
+ setOperationAction(ISD::CTLZ, VT, Custom);
+ setOperationAction(ISD::SMAX, VT, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::UMAX, VT, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::SMIN, VT, HasBWI ? Legal : Custom);
+ setOperationAction(ISD::UMIN, VT, HasBWI ? Legal : Custom);
setOperationAction(ISD::UADDSAT, VT, HasBWI ? Legal : Custom);
setOperationAction(ISD::SADDSAT, VT, HasBWI ? Legal : Custom);
setOperationAction(ISD::USUBSAT, VT, HasBWI ? Legal : Custom);
setOperationAction(ISD::SSUBSAT, VT, HasBWI ? Legal : Custom);
}
- setOperationAction(ISD::FSHL, MVT::v64i8, Custom);
- setOperationAction(ISD::FSHR, MVT::v64i8, Custom);
- setOperationAction(ISD::FSHL, MVT::v32i16, Custom);
- setOperationAction(ISD::FSHR, MVT::v32i16, Custom);
- setOperationAction(ISD::FSHL, MVT::v16i32, Custom);
- setOperationAction(ISD::FSHR, MVT::v16i32, Custom);
+ setOperationAction(ISD::FSHL, MVT::v64i8, Custom);
+ setOperationAction(ISD::FSHR, MVT::v64i8, Custom);
+ setOperationAction(ISD::FSHL, MVT::v32i16, Custom);
+ setOperationAction(ISD::FSHR, MVT::v32i16, Custom);
+ setOperationAction(ISD::FSHL, MVT::v16i32, Custom);
+ setOperationAction(ISD::FSHR, MVT::v16i32, Custom);
if (Subtarget.hasDQI() || Subtarget.hasFP16())
for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT})
- setOperationAction(Opc, MVT::v8i64, Custom);
+ setOperationAction(Opc, MVT::v8i64, Custom);
if (Subtarget.hasDQI()) {
- setOperationAction(ISD::MUL, MVT::v8i64, Legal);
+ setOperationAction(ISD::MUL, MVT::v8i64, Legal);
// MULHS needs vpmullq (AVX512DQ) for its low multiply to be a win.
setOperationAction(ISD::MULHS, MVT::v8i64, Custom);
@@ -2131,13 +2129,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
if (Subtarget.hasCDI()) {
// NonVLX sub-targets extend 128/256 vectors to use the 512 version.
- for (auto VT : {MVT::v16i32, MVT::v8i64}) {
- setOperationAction(ISD::CTLZ, VT, Legal);
+ for (auto VT : { MVT::v16i32, MVT::v8i64} ) {
+ setOperationAction(ISD::CTLZ, VT, Legal);
}
} // Subtarget.hasCDI()
if (Subtarget.hasVPOPCNTDQ()) {
- for (auto VT : {MVT::v16i32, MVT::v8i64})
+ for (auto VT : { MVT::v16i32, MVT::v8i64 })
setOperationAction(ISD::CTPOP, VT, Legal);
setOperationAction(ISD::CTPOP, MVT::i512, Custom);
}
@@ -2145,21 +2143,21 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// Extract subvector is special because the value type
// (result) is 256-bit but the source is 512-bit wide.
// 128-bit was made Legal under AVX1.
- for (auto VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
- MVT::v16f16, MVT::v8f32, MVT::v4f64})
+ for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
+ MVT::v16f16, MVT::v8f32, MVT::v4f64 })
setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
- for (auto VT : {MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64,
- MVT::v32f16, MVT::v16f32, MVT::v8f64}) {
- setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
- setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
- setOperationAction(ISD::SELECT, VT, Custom);
- setOperationAction(ISD::VSELECT, VT, Custom);
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64,
+ MVT::v32f16, MVT::v16f32, MVT::v8f64 }) {
+ setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
+ setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
+ setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::VSELECT, VT, Custom);
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
- setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
}
setF16Action(MVT::v32f16, Expand);
setOperationAction(ISD::FP_ROUND, MVT::v16f16, Custom);
@@ -2170,20 +2168,20 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationPromotedToType(Opc, MVT::v32f16, MVT::v32f32);
setOperationAction(ISD::SETCC, MVT::v32f16, Custom);
- for (auto VT : {MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64}) {
- setOperationAction(ISD::MLOAD, VT, Legal);
- setOperationAction(ISD::MSTORE, VT, Legal);
- setOperationAction(ISD::MGATHER, VT, Custom);
- setOperationAction(ISD::MSCATTER, VT, Custom);
+ for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) {
+ setOperationAction(ISD::MLOAD, VT, Legal);
+ setOperationAction(ISD::MSTORE, VT, Legal);
+ setOperationAction(ISD::MGATHER, VT, Custom);
+ setOperationAction(ISD::MSCATTER, VT, Custom);
}
if (HasBWI) {
- for (auto VT : {MVT::v64i8, MVT::v32i16}) {
- setOperationAction(ISD::MLOAD, VT, Legal);
- setOperationAction(ISD::MSTORE, VT, Legal);
+ for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
+ setOperationAction(ISD::MLOAD, VT, Legal);
+ setOperationAction(ISD::MSTORE, VT, Legal);
}
} else {
setOperationAction(ISD::STORE, MVT::v32i16, Custom);
- setOperationAction(ISD::STORE, MVT::v64i8, Custom);
+ setOperationAction(ISD::STORE, MVT::v64i8, Custom);
}
if (Subtarget.hasVBMI2()) {
@@ -2213,7 +2211,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::CTLZ, MVT::v64i8, Custom);
setOperationAction(ISD::CTTZ, MVT::v64i8, Custom);
}
- } // useAVX512Regs
+ }// useAVX512Regs
if (!Subtarget.useSoftFloat() && Subtarget.hasVBMI2()) {
for (auto VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v16i16, MVT::v8i32,
@@ -2238,9 +2236,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// These operations are handled on non-VLX by artificially widening in
// isel patterns.
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i32, Custom);
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Custom);
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i32, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i32, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i32, Custom);
if (Subtarget.hasDQI()) {
// Fast v2f32 SINT_TO_FP( v2i64 ) custom conversion.
@@ -2249,31 +2247,31 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
isOperationCustom(ISD::STRICT_UINT_TO_FP, MVT::v2f32) &&
"Unexpected operation action!");
// v2i64 FP_TO_S/UINT(v2f32) custom conversion.
- setOperationAction(ISD::FP_TO_SINT, MVT::v2f32, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v2f32, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v2f32, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v2f32, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f32, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f32, Custom);
}
- for (auto VT : {MVT::v2i64, MVT::v4i64}) {
+ for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
setOperationAction(ISD::SMAX, VT, Legal);
setOperationAction(ISD::UMAX, VT, Legal);
setOperationAction(ISD::SMIN, VT, Legal);
setOperationAction(ISD::UMIN, VT, Legal);
- setOperationAction(ISD::ABS, VT, Legal);
+ setOperationAction(ISD::ABS, VT, Legal);
}
- for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64}) {
- setOperationAction(ISD::ROTL, VT, Legal);
- setOperationAction(ISD::ROTR, VT, Legal);
+ for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
+ setOperationAction(ISD::ROTL, VT, Legal);
+ setOperationAction(ISD::ROTR, VT, Legal);
}
// Custom legalize 2x32 to get a little better code.
setOperationAction(ISD::MSCATTER, MVT::v2f32, Custom);
setOperationAction(ISD::MSCATTER, MVT::v2i32, Custom);
- for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, MVT::v4f32,
- MVT::v8f32, MVT::v2f64, MVT::v4f64})
+ for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
+ MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
setOperationAction(ISD::MSCATTER, VT, Custom);
if (Subtarget.hasDQI()) {
@@ -2300,8 +2298,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::CTLZ_ZERO_POISON, VT, Custom);
setOperationAction(ISD::CTTZ_ZERO_POISON, VT, Custom);
}
- for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64}) {
- setOperationAction(ISD::CTLZ, VT, Legal);
+ for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
+ setOperationAction(ISD::CTLZ, VT, Legal);
}
} // Subtarget.hasCDI()
@@ -2344,33 +2342,33 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// This block control legalization of v32i1/v64i1 which are available with
// AVX512BW..
if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
- addRegisterClass(MVT::v32i1, &X86::VK32RegClass);
- addRegisterClass(MVT::v64i1, &X86::VK64RegClass);
+ addRegisterClass(MVT::v32i1, &X86::VK32RegClass);
+ addRegisterClass(MVT::v64i1, &X86::VK64RegClass);
- for (auto VT : {MVT::v32i1, MVT::v64i1}) {
- setOperationAction(ISD::VSELECT, VT, Expand);
- setOperationAction(ISD::TRUNCATE, VT, Custom);
- setOperationAction(ISD::SETCC, VT, Custom);
+ for (auto VT : { MVT::v32i1, MVT::v64i1 }) {
+ setOperationAction(ISD::VSELECT, VT, Expand);
+ setOperationAction(ISD::TRUNCATE, VT, Custom);
+ setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
- setOperationAction(ISD::SELECT, VT, Custom);
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
- setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
- setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
+ setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
+ setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
}
- for (auto VT : {MVT::v16i1, MVT::v32i1})
+ for (auto VT : { MVT::v16i1, MVT::v32i1 })
setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
// Extends from v32i1 masks to 256-bit vectors.
- setOperationAction(ISD::SIGN_EXTEND, MVT::v32i8, Custom);
- setOperationAction(ISD::ZERO_EXTEND, MVT::v32i8, Custom);
- setOperationAction(ISD::ANY_EXTEND, MVT::v32i8, Custom);
+ setOperationAction(ISD::SIGN_EXTEND, MVT::v32i8, Custom);
+ setOperationAction(ISD::ZERO_EXTEND, MVT::v32i8, Custom);
+ setOperationAction(ISD::ANY_EXTEND, MVT::v32i8, Custom);
for (auto VT : {MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16,
MVT::v16f16, MVT::v8f16}) {
- setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
+ setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom);
}
@@ -2379,7 +2377,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// TODO: Custom widen in lowering on non-VLX and drop the isel patterns?
if (Subtarget.hasBITALG()) {
- for (auto VT : {MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16})
+ for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 })
setOperationAction(ISD::CTPOP, VT, Legal);
}
@@ -2395,110 +2393,110 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
if (!Subtarget.useSoftFloat() && Subtarget.hasFP16()) {
- auto setGroup = [&](MVT VT) {
- setOperationAction(ISD::FADD, VT, Legal);
- setOperationAction(ISD::STRICT_FADD, VT, Legal);
- setOperationAction(ISD::FSUB, VT, Legal);
- setOperationAction(ISD::STRICT_FSUB, VT, Legal);
- setOperationAction(ISD::FMUL, VT, Legal);
- setOperationAction(ISD::STRICT_FMUL, VT, Legal);
- setOperationAction(ISD::FDIV, VT, Legal);
- setOperationAction(ISD::STRICT_FDIV, VT, Legal);
- setOperationAction(ISD::FSQRT, VT, Legal);
- setOperationAction(ISD::STRICT_FSQRT, VT, Legal);
-
- setOperationAction(ISD::FFLOOR, VT, Legal);
- setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
- setOperationAction(ISD::FCEIL, VT, Legal);
- setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
- setOperationAction(ISD::FTRUNC, VT, Legal);
- setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
- setOperationAction(ISD::FRINT, VT, Legal);
- setOperationAction(ISD::STRICT_FRINT, VT, Legal);
- setOperationAction(ISD::FNEARBYINT, VT, Legal);
- setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
+ auto setGroup = [&] (MVT VT) {
+ setOperationAction(ISD::FADD, VT, Legal);
+ setOperationAction(ISD::STRICT_FADD, VT, Legal);
+ setOperationAction(ISD::FSUB, VT, Legal);
+ setOperationAction(ISD::STRICT_FSUB, VT, Legal);
+ setOperationAction(ISD::FMUL, VT, Legal);
+ setOperationAction(ISD::STRICT_FMUL, VT, Legal);
+ setOperationAction(ISD::FDIV, VT, Legal);
+ setOperationAction(ISD::STRICT_FDIV, VT, Legal);
+ setOperationAction(ISD::FSQRT, VT, Legal);
+ setOperationAction(ISD::STRICT_FSQRT, VT, Legal);
+
+ setOperationAction(ISD::FFLOOR, VT, Legal);
+ setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
+ setOperationAction(ISD::FCEIL, VT, Legal);
+ setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
+ setOperationAction(ISD::FTRUNC, VT, Legal);
+ setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
+ setOperationAction(ISD::FRINT, VT, Legal);
+ setOperationAction(ISD::STRICT_FRINT, VT, Legal);
+ setOperationAction(ISD::FNEARBYINT, VT, Legal);
+ setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
setOperationAction(ISD::FROUNDEVEN, VT, Legal);
setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
- setOperationAction(ISD::FROUND, VT, Custom);
+ setOperationAction(ISD::FROUND, VT, Custom);
- setOperationAction(ISD::LOAD, VT, Legal);
- setOperationAction(ISD::STORE, VT, Legal);
+ setOperationAction(ISD::LOAD, VT, Legal);
+ setOperationAction(ISD::STORE, VT, Legal);
- setOperationAction(ISD::FMA, VT, Legal);
- setOperationAction(ISD::STRICT_FMA, VT, Legal);
- setOperationAction(ISD::VSELECT, VT, Legal);
- setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
- setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::FMA, VT, Legal);
+ setOperationAction(ISD::STRICT_FMA, VT, Legal);
+ setOperationAction(ISD::VSELECT, VT, Legal);
+ setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+ setOperationAction(ISD::SELECT, VT, Custom);
- setOperationAction(ISD::FNEG, VT, Custom);
- setOperationAction(ISD::FABS, VT, Custom);
- setOperationAction(ISD::FCOPYSIGN, VT, Custom);
+ setOperationAction(ISD::FNEG, VT, Custom);
+ setOperationAction(ISD::FABS, VT, Custom);
+ setOperationAction(ISD::FCOPYSIGN, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
- setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+ setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
- setOperationAction(ISD::SETCC, VT, Custom);
- setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
- setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
+ setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
+ setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
};
// AVX512_FP16 scalar operations
setGroup(MVT::f16);
SetFPMinMaxAction(MVT::f16);
- setOperationAction(ISD::FREM, MVT::f16, Promote);
- setOperationAction(ISD::STRICT_FREM, MVT::f16, Promote);
- setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
- setOperationAction(ISD::BR_CC, MVT::f16, Expand);
- setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);
- setOperationAction(ISD::FROUNDEVEN, MVT::f16, Legal);
- setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Legal);
- setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
- setOperationAction(ISD::FP_EXTEND, MVT::f32, Legal);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
- setOperationAction(ISD::LRINT, MVT::f16, Legal);
- setOperationAction(ISD::LLRINT, MVT::f16, Legal);
+ setOperationAction(ISD::FREM, MVT::f16, Promote);
+ setOperationAction(ISD::STRICT_FREM, MVT::f16, Promote);
+ setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
+ setOperationAction(ISD::BR_CC, MVT::f16, Expand);
+ setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);
+ setOperationAction(ISD::FROUNDEVEN, MVT::f16, Legal);
+ setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Legal);
+ setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
+ setOperationAction(ISD::FP_EXTEND, MVT::f32, Legal);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
+ setOperationAction(ISD::LRINT, MVT::f16, Legal);
+ setOperationAction(ISD::LLRINT, MVT::f16, Legal);
setCondCodeAction(ISD::SETOEQ, MVT::f16, Expand);
setCondCodeAction(ISD::SETUNE, MVT::f16, Expand);
if (Subtarget.useAVX512Regs()) {
setGroup(MVT::v32f16);
- setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v32f16, Custom);
- setOperationAction(ISD::SINT_TO_FP, MVT::v32i16, Legal);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v32i16, Legal);
- setOperationAction(ISD::UINT_TO_FP, MVT::v32i16, Legal);
- setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v32i16, Legal);
- setOperationAction(ISD::FP_ROUND, MVT::v16f16, Legal);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Legal);
- setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Legal);
- setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Legal);
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32f16, Custom);
-
- setOperationAction(ISD::FP_TO_SINT, MVT::v32i16, Custom);
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v32i16, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v32i16, Custom);
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v32i16, Custom);
- setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i8, MVT::v32i16);
+ setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v32f16, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v32i16, Legal);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v32i16, Legal);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v32i16, Legal);
+ setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v32i16, Legal);
+ setOperationAction(ISD::FP_ROUND, MVT::v16f16, Legal);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Legal);
+ setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Legal);
+ setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Legal);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32f16, Custom);
+
+ setOperationAction(ISD::FP_TO_SINT, MVT::v32i16, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v32i16, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v32i16, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v32i16, Custom);
+ setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i8, MVT::v32i16);
setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i8,
MVT::v32i16);
- setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i8, MVT::v32i16);
+ setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i8, MVT::v32i16);
setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i8,
MVT::v32i16);
- setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i1, MVT::v32i16);
+ setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i1, MVT::v32i16);
setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i1,
MVT::v32i16);
- setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i1, MVT::v32i16);
+ setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i1, MVT::v32i16);
setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i1,
MVT::v32i16);
- setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v16f16, Legal);
- setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32f16, Legal);
- setOperationAction(ISD::CONCAT_VECTORS, MVT::v32f16, Custom);
+ setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v16f16, Legal);
+ setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32f16, Legal);
+ setOperationAction(ISD::CONCAT_VECTORS, MVT::v32f16, Custom);
- setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Legal);
+ setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Legal);
setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Legal);
SetFPMinMaxAction(MVT::v32f16);
@@ -2506,40 +2504,40 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::LLRINT, MVT::v8f16, Legal);
}
- setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom);
- setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i16, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Custom);
- setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i16, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i16, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Custom);
+ setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i16, Custom);
if (Subtarget.hasVLX()) {
setGroup(MVT::v8f16);
setGroup(MVT::v16f16);
- setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8f16, Legal);
- setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16f16, Custom);
- setOperationAction(ISD::SINT_TO_FP, MVT::v16i16, Legal);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i16, Legal);
- setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Legal);
- setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i16, Legal);
- setOperationAction(ISD::UINT_TO_FP, MVT::v16i16, Legal);
- setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i16, Legal);
- setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Legal);
- setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i16, Legal);
-
- setOperationAction(ISD::FP_ROUND, MVT::v8f16, Legal);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f16, Legal);
- setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f32, Legal);
- setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Legal);
+ setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8f16, Legal);
+ setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16f16, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v16i16, Legal);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i16, Legal);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Legal);
+ setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i16, Legal);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v16i16, Legal);
+ setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i16, Legal);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Legal);
+ setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i16, Legal);
+
+ setOperationAction(ISD::FP_ROUND, MVT::v8f16, Legal);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f16, Legal);
+ setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f32, Legal);
+ setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Legal);
// INSERT_VECTOR_ELT v8f16 extended to VECTOR_SHUFFLE
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8f16, Custom);
- setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16f16, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8f16, Custom);
+ setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16f16, Custom);
- setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f16, Legal);
- setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v16f16, Legal);
- setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f16, Custom);
+ setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f16, Legal);
+ setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v16f16, Legal);
+ setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f16, Custom);
setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Legal);
setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Legal);
@@ -2547,7 +2545,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Legal);
// Need to custom widen these to prevent scalarization.
- setOperationAction(ISD::LOAD, MVT::v4f16, Custom);
+ setOperationAction(ISD::LOAD, MVT::v4f16, Custom);
setOperationAction(ISD::STORE, MVT::v4f16, Custom);
SetFPMinMaxAction(MVT::v8f16);
@@ -2637,52 +2635,52 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) {
- setTruncStoreAction(MVT::v4i64, MVT::v4i8, Legal);
+ setTruncStoreAction(MVT::v4i64, MVT::v4i8, Legal);
setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal);
setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal);
- setTruncStoreAction(MVT::v8i32, MVT::v8i8, Legal);
+ setTruncStoreAction(MVT::v8i32, MVT::v8i8, Legal);
setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal);
- setTruncStoreAction(MVT::v2i64, MVT::v2i8, Legal);
+ setTruncStoreAction(MVT::v2i64, MVT::v2i8, Legal);
setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal);
setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal);
- setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal);
+ setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal);
setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
if (Subtarget.hasBWI()) {
- setTruncStoreAction(MVT::v16i16, MVT::v16i8, Legal);
- setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
+ setTruncStoreAction(MVT::v16i16, MVT::v16i8, Legal);
+ setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
}
if (Subtarget.hasFP16()) {
// vcvttph2[u]dq v4f16 -> v4i32/64, v2f16 -> v2i32/64
- setOperationAction(ISD::FP_TO_SINT, MVT::v2f16, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v2f16, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f16, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v2f16, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v2f16, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f16, Custom);
- setOperationAction(ISD::FP_TO_SINT, MVT::v4f16, Custom);
+ setOperationAction(ISD::FP_TO_SINT, MVT::v4f16, Custom);
setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4f16, Custom);
- setOperationAction(ISD::FP_TO_UINT, MVT::v4f16, Custom);
+ setOperationAction(ISD::FP_TO_UINT, MVT::v4f16, Custom);
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4f16, Custom);
// vcvt[u]dq2ph v4i32/64 -> v4f16, v2i32/64 -> v2f16
- setOperationAction(ISD::SINT_TO_FP, MVT::v2f16, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v2f16, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f16, Custom);
- setOperationAction(ISD::UINT_TO_FP, MVT::v2f16, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v2f16, Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f16, Custom);
- setOperationAction(ISD::SINT_TO_FP, MVT::v4f16, Custom);
+ setOperationAction(ISD::SINT_TO_FP, MVT::v4f16, Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4f16, Custom);
- setOperationAction(ISD::UINT_TO_FP, MVT::v4f16, Custom);
+ setOperationAction(ISD::UINT_TO_FP, MVT::v4f16, Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4f16, Custom);
// vcvtps2phx v4f32 -> v4f16, v2f32 -> v2f16
- setOperationAction(ISD::FP_ROUND, MVT::v2f16, Custom);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f16, Custom);
- setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
- setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f16, Custom);
+ setOperationAction(ISD::FP_ROUND, MVT::v2f16, Custom);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f16, Custom);
+ setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
+ setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f16, Custom);
// vcvtph2psx v4f16 -> v4f32, v2f16 -> v2f32
- setOperationAction(ISD::FP_EXTEND, MVT::v2f16, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f16, Custom);
- setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Custom);
- setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f16, Custom);
+ setOperationAction(ISD::FP_EXTEND, MVT::v2f16, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f16, Custom);
+ setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Custom);
+ setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f16, Custom);
}
}
@@ -2704,7 +2702,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// FIXME: We really should do custom legalization for addition and
// subtraction on x86-32 once PR3203 is fixed. We really can't do much better
// than generic legalization for 64-bit multiplication-with-overflow, though.
- for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
+ for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
if (VT == MVT::i64 && !Subtarget.is64Bit())
continue;
// Add/Sub/Mul with overflow operations are custom lowered.
@@ -3053,9 +3051,8 @@ static bool isLogicOp(unsigned Opcode) {
}
static bool isTargetShuffle(unsigned Opcode) {
- switch (Opcode) {
- default:
- return false;
+ switch(Opcode) {
+ default: return false;
case X86ISD::BLENDI:
case X86ISD::PSHUFB:
case X86ISD::PSHUFD:
@@ -3098,8 +3095,7 @@ static bool isTargetShuffle(unsigned Opcode) {
static bool isTargetShuffleVariableMask(unsigned Opcode) {
switch (Opcode) {
- default:
- return false;
+ default: return false;
// Target Shuffles.
case X86ISD::PSHUFB:
case X86ISD::VPERMILPV:
@@ -3125,8 +3121,9 @@ SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
if (ReturnAddrIndex == 0) {
// Set up a frame object for the return address.
unsigned SlotSize = RegInfo->getSlotSize();
- ReturnAddrIndex = MF.getFrameInfo().CreateFixedObject(
- SlotSize, -(int64_t)SlotSize, false);
+ ReturnAddrIndex = MF.getFrameInfo().CreateFixedObject(SlotSize,
+ -(int64_t)SlotSize,
+ false);
FuncInfo->setRAIndex(ReturnAddrIndex);
}
@@ -3184,7 +3181,7 @@ static bool isX86CCSigned(X86::CondCode X86CC) {
static X86::CondCode TranslateIntegerX86CC(ISD::CondCode SetCCOpcode) {
switch (SetCCOpcode) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Invalid integer condition!");
case ISD::SETEQ: return X86::COND_E;
case ISD::SETGT: return X86::COND_G;
@@ -3196,7 +3193,7 @@ static X86::CondCode TranslateIntegerX86CC(ISD::CondCode SetCCOpcode) {
case ISD::SETUGT: return X86::COND_A;
case ISD::SETULE: return X86::COND_BE;
case ISD::SETUGE: return X86::COND_AE;
- // clang-format on
+ // clang-format on
}
}
@@ -3234,14 +3231,14 @@ static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
// First determine if it is required or is profitable to flip the operands.
// If LHS is a foldable load, but RHS is not, flip the condition.
- if (ISD::isNON_EXTLoad(LHS.getNode()) && !ISD::isNON_EXTLoad(RHS.getNode())) {
+ if (ISD::isNON_EXTLoad(LHS.getNode()) &&
+ !ISD::isNON_EXTLoad(RHS.getNode())) {
SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
std::swap(LHS, RHS);
}
switch (SetCCOpcode) {
- default:
- break;
+ default: break;
case ISD::SETOLT:
case ISD::SETOLE:
case ISD::SETUGT:
@@ -3257,7 +3254,7 @@ static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
// 1 | 0 | 0 | X == Y
// 1 | 1 | 1 | unordered
switch (SetCCOpcode) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Condcode should be pre-legalized away");
case ISD::SETUEQ:
case ISD::SETEQ: return X86::COND_E;
@@ -3279,7 +3276,7 @@ static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
case ISD::SETO: return X86::COND_NP;
case ISD::SETOEQ:
case ISD::SETUNE: return X86::COND_INVALID;
- // clang-format on
+ // clang-format on
}
}
@@ -3314,7 +3311,7 @@ void X86TargetLowering::getTgtMemIntrinsic(
Info.flags = MachineMemOperand::MONone;
Info.offset = 0;
- const IntrinsicData *IntrData = getIntrinsicWithChain(Intrinsic);
+ const IntrinsicData* IntrData = getIntrinsicWithChain(Intrinsic);
if (!IntrData) {
switch (Intrinsic) {
case Intrinsic::x86_aesenc128kl:
@@ -3414,7 +3411,7 @@ void X86TargetLowering::getTgtMemIntrinsic(
case TRUNCATE_TO_MEM_VI32: {
Info.opc = ISD::INTRINSIC_VOID;
Info.ptrVal = I.getArgOperand(0);
- MVT VT = MVT::getVT(I.getArgOperand(1)->getType());
+ MVT VT = MVT::getVT(I.getArgOperand(1)->getType());
MVT ScalarVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
if (IntrData->Type == TRUNCATE_TO_MEM_VI8)
ScalarVT = MVT::i8;
@@ -3435,8 +3432,8 @@ void X86TargetLowering::getTgtMemIntrinsic(
Info.ptrVal = nullptr;
MVT DataVT = MVT::getVT(I.getType());
MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
- unsigned NumElts =
- std::min(DataVT.getVectorNumElements(), IndexVT.getVectorNumElements());
+ unsigned NumElts = std::min(DataVT.getVectorNumElements(),
+ IndexVT.getVectorNumElements());
Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
Info.align = Align(1);
Info.flags |= MachineMemOperand::MOLoad;
@@ -3448,8 +3445,8 @@ void X86TargetLowering::getTgtMemIntrinsic(
Info.ptrVal = nullptr;
MVT DataVT = MVT::getVT(I.getArgOperand(3)->getType());
MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
- unsigned NumElts =
- std::min(DataVT.getVectorNumElements(), IndexVT.getVectorNumElements());
+ unsigned NumElts = std::min(DataVT.getVectorNumElements(),
+ IndexVT.getVectorNumElements());
Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
Info.align = Align(1);
Info.flags |= MachineMemOperand::MOStore;
@@ -3699,9 +3696,9 @@ bool X86TargetLowering::isScalarFPTypeInSSEReg(EVT VT) const {
(VT == MVT::f32 && Subtarget.hasSSE1()) || VT == MVT::f16;
}
-bool X86TargetLowering::isLoadBitCastBeneficial(
- EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG,
- const MachineMemOperand &MMO) const {
+bool X86TargetLowering::isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
+ const SelectionDAG &DAG,
+ const MachineMemOperand &MMO) const {
if (!Subtarget.hasAVX512() && !LoadVT.isVector() && BitcastVT.isVector() &&
BitcastVT.getVectorElementType() == MVT::i1)
return false;
@@ -3743,7 +3740,9 @@ bool X86TargetLowering::canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
return true;
}
-bool X86TargetLowering::isCtlzFast() const { return Subtarget.hasFastLZCNT(); }
+bool X86TargetLowering::isCtlzFast() const {
+ return Subtarget.hasFastLZCNT();
+}
bool X86TargetLowering::isMaskAndCmp0FoldingBeneficial(
const Instruction &AndI) const {
@@ -4168,7 +4167,8 @@ static bool canWidenShuffleElements(ArrayRef<int> Mask,
return true;
}
-static bool canWidenShuffleElements(ArrayRef<int> Mask, const APInt &Zeroable,
+static bool canWidenShuffleElements(ArrayRef<int> Mask,
+ const APInt &Zeroable,
bool V2IsZero,
SmallVectorImpl<int> &WidenedMask) {
// Create an alternative mask with info about zeroable elements.
@@ -4252,7 +4252,7 @@ bool X86::isZeroNode(SDValue Elt) {
static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
const SDLoc &dl, bool IsMask = false) {
- SmallVector<SDValue, 32> Ops;
+ SmallVector<SDValue, 32> Ops;
bool Split = false;
MVT ConstVecVT = VT;
@@ -4266,12 +4266,12 @@ static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
MVT EltVT = ConstVecVT.getVectorElementType();
for (unsigned i = 0; i < NumElts; ++i) {
bool IsUndef = Values[i] < 0 && IsMask;
- SDValue OpNode =
- IsUndef ? DAG.getUNDEF(EltVT) : DAG.getConstant(Values[i], dl, EltVT);
+ SDValue OpNode = IsUndef ? DAG.getUNDEF(EltVT) :
+ DAG.getConstant(Values[i], dl, EltVT);
Ops.push_back(OpNode);
if (Split)
- Ops.push_back(IsUndef ? DAG.getUNDEF(EltVT)
- : DAG.getConstant(0, dl, EltVT));
+ Ops.push_back(IsUndef ? DAG.getUNDEF(EltVT) :
+ DAG.getConstant(0, dl, EltVT));
}
SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
if (Split)
@@ -4279,8 +4279,8 @@ static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
return ConstsNode;
}
-static SDValue getConstVector(ArrayRef<APInt> Bits, const APInt &Undefs, MVT VT,
- SelectionDAG &DAG, const SDLoc &dl) {
+static SDValue getConstVector(ArrayRef<APInt> Bits, const APInt &Undefs,
+ MVT VT, SelectionDAG &DAG, const SDLoc &dl) {
assert(Bits.size() == Undefs.getBitWidth() &&
"Unequal constant and undef arrays");
SmallVector<SDValue, 32> Ops;
@@ -4315,8 +4315,8 @@ static SDValue getConstVector(ArrayRef<APInt> Bits, const APInt &Undefs, MVT VT,
return DAG.getBitcast(VT, ConstsNode);
}
-static SDValue getConstVector(ArrayRef<APInt> Bits, MVT VT, SelectionDAG &DAG,
- const SDLoc &dl) {
+static SDValue getConstVector(ArrayRef<APInt> Bits, MVT VT,
+ SelectionDAG &DAG, const SDLoc &dl) {
APInt Undefs = APInt::getZero(Bits.size());
return getConstVector(Bits, Undefs, VT, DAG, dl);
}
@@ -4853,7 +4853,8 @@ static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
if (IdxVal == 0 && ISD::isBuildVectorAllZeros(Vec.getNode())) {
// May need to promote to a legal type.
Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
- DAG.getConstant(0, dl, WideOpVT), SubVec, Idx);
+ DAG.getConstant(0, dl, WideOpVT),
+ SubVec, Idx);
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
}
@@ -4868,18 +4869,20 @@ static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
if (IdxVal == 0) {
// Zero lower bits of the Vec
SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8);
- Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, ZeroIdx);
+ Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec,
+ ZeroIdx);
Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
// Merge them together, SubVec should be zero extended.
SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
- DAG.getConstant(0, dl, WideOpVT), SubVec, ZeroIdx);
+ DAG.getConstant(0, dl, WideOpVT),
+ SubVec, ZeroIdx);
Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
}
- SubVec =
- DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, SubVec, ZeroIdx);
+ SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
+ Undef, SubVec, ZeroIdx);
if (Vec.isUndef()) {
assert(IdxVal != 0 && "Unexpected index");
@@ -4917,11 +4920,12 @@ static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
// isel to optimize when bits are known zero.
Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVecVT, Vec, ZeroIdx);
Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
- DAG.getConstant(0, dl, WideOpVT), Vec, ZeroIdx);
+ DAG.getConstant(0, dl, WideOpVT),
+ Vec, ZeroIdx);
} else {
// Otherwise use explicit shifts to zero the bits.
- Vec =
- DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, ZeroIdx);
+ Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
+ Undef, Vec, ZeroIdx);
NumElems = WideOpVT.getVectorNumElements();
SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8);
Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
@@ -4974,9 +4978,9 @@ static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
// Isolate the bits after the last inserted bit.
unsigned HighShift = IdxVal + SubVecNumElems;
SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
- DAG.getTargetConstant(HighShift, dl, MVT::i8));
+ DAG.getTargetConstant(HighShift, dl, MVT::i8));
High = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, High,
- DAG.getTargetConstant(HighShift, dl, MVT::i8));
+ DAG.getTargetConstant(HighShift, dl, MVT::i8));
// Now OR all 3 pieces together.
Vec = DAG.getNode(ISD::OR, dl, WideOpVT, Low, High);
@@ -5225,8 +5229,8 @@ static SDValue getBitSelect(const SDLoc &DL, MVT VT, SDValue LHS, SDValue RHS,
return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
}
-void llvm::createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask, bool Lo,
- bool Unary) {
+void llvm::createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask,
+ bool Lo, bool Unary) {
assert(VT.getScalarType().isSimple() && (VT.getSizeInBits() % 128) == 0 &&
"Illegal vector type to unpack");
assert(Mask.empty() && "Expected an empty shuffle mask vector");
@@ -5363,12 +5367,13 @@ static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget,
/// This produces a shuffle where the low element of V2 is swizzled into the
/// zero/undef vector, landing at element Idx.
/// This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
-static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx, bool IsZero,
+static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx,
+ bool IsZero,
const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
MVT VT = V2.getSimpleValueType();
- SDValue V1 =
- IsZero ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
+ SDValue V1 = IsZero
+ ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
int NumElems = VT.getVectorNumElements();
SmallVector<int, 16> MaskVec(NumElems);
for (int i = 0; i != NumElems; ++i)
@@ -6333,9 +6338,9 @@ static bool getTargetShuffleMask(SDValue N, bool AllowSentinelZero,
/// zero. Many x86 shuffles can zero lanes cheaply and we often want to handle
/// as many lanes with this technique as possible to simplify the remaining
/// shuffle.
-static void computeZeroableShuffleElements(ArrayRef<int> Mask, SDValue V1,
- SDValue V2, APInt &KnownUndef,
- APInt &KnownZero) {
+static void computeZeroableShuffleElements(ArrayRef<int> Mask,
+ SDValue V1, SDValue V2,
+ APInt &KnownUndef, APInt &KnownZero) {
int Size = Mask.size();
KnownUndef = KnownZero = APInt::getZero(Size);
@@ -6521,7 +6526,7 @@ static bool getTargetShuffleAndZeroables(SDValue N, SmallVectorImpl<int> &Mask,
static void resolveTargetShuffleFromZeroables(SmallVectorImpl<int> &Mask,
const APInt &KnownUndef,
const APInt &KnownZero,
- bool ResolveKnownZeros = true) {
+ bool ResolveKnownZeros= true) {
unsigned NumElts = Mask.size();
assert(KnownUndef.getBitWidth() == NumElts &&
KnownZero.getBitWidth() == NumElts && "Shuffle mask size mismatch");
@@ -7553,8 +7558,8 @@ static SDValue LowerBuildVectorv4x32(SDValue Op, const SDLoc &DL,
MVT EltVT = VT.getVectorElementType();
// Create a new build vector with the first 2 elements followed by undef
// padding, bitcast to v2f64, duplicate, and bitcast back.
- SDValue Ops[4] = {Op.getOperand(0), Op.getOperand(1), DAG.getUNDEF(EltVT),
- DAG.getUNDEF(EltVT)};
+ SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
+ DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));
SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV);
return DAG.getBitcast(VT, Dup);
@@ -7602,7 +7607,7 @@ static SDValue LowerBuildVectorv4x32(SDValue Op, const SDLoc &DL,
for (EltIdx = 0; EltIdx < 4; ++EltIdx) {
if (Zeroable[EltIdx]) {
// The zero vector will be on the right hand side.
- Mask[EltIdx] = EltIdx + 4;
+ Mask[EltIdx] = EltIdx+4;
continue;
}
@@ -7832,7 +7837,7 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
APInt ZeroMask = APInt::getZero(NumElems);
APInt UndefMask = APInt::getZero(NumElems);
- SmallVector<LoadSDNode *, 8> Loads(NumElems, nullptr);
+ SmallVector<LoadSDNode*, 8> Loads(NumElems, nullptr);
SmallVector<int64_t, 8> ByteOffsets(NumElems, 0);
// For each element in the initializer, see if we've found a load, zero or an
@@ -7939,7 +7944,8 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, LoadSDNode *LDBase) {
auto MMOFlags = LDBase->getMemOperand()->getFlags();
- assert(LDBase->isSimple() && "Cannot merge volatile or atomic loads.");
+ assert(LDBase->isSimple() &&
+ "Cannot merge volatile or atomic loads.");
SDValue NewLd =
DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
LDBase->getPointerInfo(), LDBase->getBaseAlign(), MMOFlags);
@@ -8027,7 +8033,7 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
VecVT = MVT::v4f32;
if (TLI.isTypeLegal(VecVT)) {
SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
- SDValue Ops[] = {LDBase->getChain(), LDBase->getBasePtr()};
+ SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
SDValue ResNode = DAG.getMemIntrinsicNode(
X86ISD::VZEXT_LOAD, DL, Tys, Ops, VecSVT, LDBase->getPointerInfo(),
LDBase->getBaseAlign(), MachineMemOperand::MOLoad);
@@ -8835,8 +8841,7 @@ static SDValue LowerBUILD_VECTORvXi1(SDValue Op, const SDLoc &dl,
/// Returns true if is possible to fold MUL and an idiom that has already been
/// recognized as ADDSUB/SUBADD(\p Opnd0, \p Opnd1) into
/// FMADDSUB/FMSUBADD(x, y, \p Opnd1). If (and only if) true is returned, the
-/// operands of FMADDSUB/FMSUBADD are written to parameters \p Opnd0, \p Opnd1,
-/// \p Opnd2.
+/// operands of FMADDSUB/FMSUBADD are written to parameters \p Opnd0, \p Opnd1, \p Opnd2.
///
/// Prior to calling this function it should be known that there is some
/// SDNode that potentially can be replaced with an X86ISD::ADDSUB operation
@@ -9425,8 +9430,8 @@ LowerBUILD_VECTORAsVariablePermute(SDValue V, const SDLoc &DL,
return createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);
}
-SDValue X86TargetLowering::LowerBUILD_VECTOR(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue
+X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
SDLoc dl(Op);
MVT VT = Op.getSimpleValueType();
@@ -9648,13 +9653,14 @@ SDValue X86TargetLowering::LowerBUILD_VECTOR(SDValue Op,
}
// Is it a vector logical left shift?
- if (NumElems == 2 && Idx == 1 && X86::isZeroNode(Op.getOperand(0)) &&
+ if (NumElems == 2 && Idx == 1 &&
+ X86::isZeroNode(Op.getOperand(0)) &&
!X86::isZeroNode(Op.getOperand(1))) {
unsigned NumBits = VT.getSizeInBits();
- return getVShift(
- true, VT,
- DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(1)),
- NumBits / 2, DAG, *this, dl);
+ return getVShift(true, VT,
+ DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
+ VT, Op.getOperand(1)),
+ NumBits/2, DAG, *this, dl);
}
if (IsAllConstants) // Otherwise, it's better to do a constpool load.
@@ -9667,8 +9673,7 @@ SDValue X86TargetLowering::LowerBUILD_VECTOR(SDValue Op,
// place.
if (EVTBits == 32) {
Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
- return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget,
- DAG);
+ return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget, DAG);
}
}
@@ -9707,8 +9712,8 @@ SDValue X86TargetLowering::LowerBUILD_VECTOR(SDValue Op,
// build_vector and broadcast it.
// TODO: We could probably generalize this more.
if (Subtarget.hasAVX2() && EVTBits == 32 && Values.size() == 2) {
- SDValue Ops[4] = {Op.getOperand(0), Op.getOperand(1), DAG.getUNDEF(EltVT),
- DAG.getUNDEF(EltVT)};
+ SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
+ DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
auto CanSplat = [](SDValue Op, unsigned NumElems, ArrayRef<SDValue> Ops) {
// Make sure all the even/odd operands match.
for (unsigned i = 2; i != NumElems; ++i)
@@ -9724,8 +9729,8 @@ SDValue X86TargetLowering::LowerBUILD_VECTOR(SDValue Op,
DAG.getBuildVector(NarrowVT, dl, Ops));
// Broadcast from v2i64/v2f64 and cast to final VT.
MVT BcastVT = MVT::getVectorVT(WideEltVT, NumElems / 2);
- return DAG.getBitcast(
- VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT, NewBV));
+ return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT,
+ NewBV));
}
}
@@ -9738,7 +9743,7 @@ SDValue X86TargetLowering::LowerBUILD_VECTOR(SDValue Op,
SDValue Lower =
DAG.getBuildVector(HVT, dl, Op->ops().slice(0, NumElems / 2));
SDValue Upper = DAG.getBuildVector(
- HVT, dl, Op->ops().slice(NumElems / 2, NumElems / 2));
+ HVT, dl, Op->ops().slice(NumElems / 2, NumElems /2));
// Recreate the wider vector with the lower and upper part.
return concatSubVectors(Lower, Upper, DAG, dl);
@@ -9749,8 +9754,8 @@ SDValue X86TargetLowering::LowerBUILD_VECTOR(SDValue Op,
if (NumNonZero == 1) {
// One half is zero or undef.
unsigned Idx = NonZeroMask.countr_zero();
- SDValue V2 =
- DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(Idx));
+ SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
+ Op.getOperand(Idx));
return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG);
}
return SDValue();
@@ -9785,28 +9790,30 @@ SDValue X86TargetLowering::LowerBUILD_VECTOR(SDValue Op,
for (unsigned i = 0; i < 2; ++i) {
switch (NonZeroMask.extractBitsAsZExtValue(2, i * 2)) {
- default:
- llvm_unreachable("Unexpected NonZero count");
- case 0:
- Ops[i] = Ops[i * 2]; // Must be a zero vector.
- break;
- case 1:
- Ops[i] = getMOVL(DAG, dl, VT, Ops[i * 2 + 1], Ops[i * 2]);
- break;
- case 2:
- Ops[i] = getMOVL(DAG, dl, VT, Ops[i * 2], Ops[i * 2 + 1]);
- break;
- case 3:
- Ops[i] = getUnpackl(DAG, dl, VT, Ops[i * 2], Ops[i * 2 + 1]);
- break;
+ default: llvm_unreachable("Unexpected NonZero count");
+ case 0:
+ Ops[i] = Ops[i*2]; // Must be a zero vector.
+ break;
+ case 1:
+ Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2+1], Ops[i*2]);
+ break;
+ case 2:
+ Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
+ break;
+ case 3:
+ Ops[i] = getUnpackl(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
+ break;
}
}
bool Reverse1 = NonZeroMask.extractBitsAsZExtValue(2, 0) == 2;
bool Reverse2 = NonZeroMask.extractBitsAsZExtValue(2, 2) == 2;
- int MaskVec[] = {Reverse1 ? 1 : 0, Reverse1 ? 0 : 1,
- static_cast<int>(Reverse2 ? NumElems + 1 : NumElems),
- static_cast<int>(Reverse2 ? NumElems : NumElems + 1)};
+ int MaskVec[] = {
+ Reverse1 ? 1 : 0,
+ Reverse1 ? 0 : 1,
+ static_cast<int>(Reverse2 ? NumElems+1 : NumElems),
+ static_cast<int>(Reverse2 ? NumElems : NumElems+1)
+ };
return DAG.getVectorShuffle(VT, dl, Ops[0], Ops[1], MaskVec);
}
@@ -9825,8 +9832,7 @@ SDValue X86TargetLowering::LowerBUILD_VECTOR(SDValue Op,
Result = DAG.getUNDEF(VT);
for (unsigned i = 1; i < NumElems; ++i) {
- if (Op.getOperand(i).isUndef())
- continue;
+ if (Op.getOperand(i).isUndef()) continue;
Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result,
Op.getOperand(i), DAG.getVectorIdxConstant(i, dl));
}
@@ -9851,14 +9857,14 @@ SDValue X86TargetLowering::LowerBUILD_VECTOR(SDValue Op,
for (unsigned Scale = 1; Scale < NumElems; Scale *= 2) {
// Generate scaled UNPCKL shuffle mask.
SmallVector<int, 16> Mask;
- for (unsigned i = 0; i != Scale; ++i)
+ for(unsigned i = 0; i != Scale; ++i)
Mask.push_back(i);
for (unsigned i = 0; i != Scale; ++i)
- Mask.push_back(NumElems + i);
+ Mask.push_back(NumElems+i);
Mask.append(NumElems - Mask.size(), SM_SentinelUndef);
for (unsigned i = 0, e = NumElems / (2 * Scale); i != e; ++i)
- Ops[i] = DAG.getVectorShuffle(VT, dl, Ops[2 * i], Ops[(2 * i) + 1], Mask);
+ Ops[i] = DAG.getVectorShuffle(VT, dl, Ops[2*i], Ops[(2*i)+1], Mask);
}
return Ops[0];
}
@@ -9884,14 +9890,15 @@ static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, const SDLoc &dl,
if (SubVec.isUndef())
continue;
if (ISD::isFreezeUndef(SubVec.getNode())) {
- // If the freeze(undef) has multiple uses then we must fold to zero.
- if (SubVec.hasOneUse()) {
- ++NumFreezeUndef;
- } else {
- ++NumZero;
- Undefs.insert(SubVec);
- }
- } else if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
+ // If the freeze(undef) has multiple uses then we must fold to zero.
+ if (SubVec.hasOneUse()) {
+ ++NumFreezeUndef;
+ } else {
+ ++NumZero;
+ Undefs.insert(SubVec);
+ }
+ }
+ else if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
++NumZero;
else {
assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
@@ -9905,9 +9912,9 @@ static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, const SDLoc &dl,
MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
ArrayRef<SDUse> Ops = Op->ops();
SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
- Ops.slice(0, NumOperands / 2));
+ Ops.slice(0, NumOperands/2));
SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
- Ops.slice(NumOperands / 2));
+ Ops.slice(NumOperands/2));
return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
}
@@ -9940,7 +9947,7 @@ static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, const SDLoc &dl,
// TODO: Merge this with LowerAVXCONCAT_VECTORS?
static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op, const SDLoc &dl,
const X86Subtarget &Subtarget,
- SelectionDAG &DAG) {
+ SelectionDAG & DAG) {
MVT ResVT = Op.getSimpleValueType();
unsigned NumOperands = Op.getNumOperands();
assert(NumOperands > 1 && isPowerOf2_32(NumOperands) &&
@@ -10011,7 +10018,8 @@ static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op, const SDLoc &dl,
DAG.getVectorIdxConstant(NumElems / 2, dl));
}
-static SDValue LowerCONCAT_VECTORS(SDValue Op, const X86Subtarget &Subtarget,
+static SDValue LowerCONCAT_VECTORS(SDValue Op,
+ const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
SDLoc DL(Op);
MVT VT = Op.getSimpleValueType();
@@ -10233,8 +10241,8 @@ static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,
// Ok, handle the in-lane shuffles by detecting if and when they repeat.
// Adjust second vector indices to start at LaneSize instead of Size.
- int LocalM =
- Mask[i] < Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + LaneSize;
+ int LocalM = Mask[i] < Size ? Mask[i] % LaneSize
+ : Mask[i] % LaneSize + LaneSize;
if (RepeatedMask[i % LaneSize] < 0)
// This is the first non-undef entry in this slot of a 128-bit lane.
RepeatedMask[i % LaneSize] = LocalM;
@@ -10252,7 +10260,8 @@ is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,
return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);
}
-static bool is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask) {
+static bool
+is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask) {
SmallVector<int, 32> RepeatedMask;
return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);
}
@@ -10551,8 +10560,8 @@ static SDValue getSHUFPDImmForMask(ArrayRef<int> Mask, const SDLoc &DL,
//
// The function looks for a sub-mask that the nonzero elements are in
// increasing order. If such sub-mask exist. The function returns true.
-static bool isNonZeroElementsInOrder(const APInt &Zeroable, ArrayRef<int> Mask,
- const EVT &VectorType,
+static bool isNonZeroElementsInOrder(const APInt &Zeroable,
+ ArrayRef<int> Mask, const EVT &VectorType,
bool &IsZeroSideLeft) {
int NextElement = -1;
// Check if the Mask's nonzero elements are in increasing order.
@@ -11339,7 +11348,7 @@ static bool matchShuffleAsBlend(MVT VT, SDValue V1, SDValue V2,
if (M == SM_SentinelUndef)
continue;
if (M == Elt || (0 <= M && M < NumElts &&
- IsElementEquivalent(NumElts, V1, V1, M, Elt))) {
+ IsElementEquivalent(NumElts, V1, V1, M, Elt))) {
Mask[Elt] = Elt;
LaneV1InUse = true;
continue;
@@ -11472,7 +11481,8 @@ static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
// If we have VPTERNLOG, we can use that as a bit blend.
if (Subtarget.hasVLX())
- if (SDValue BitBlend = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
+ if (SDValue BitBlend =
+ lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
return BitBlend;
// Scale the blend by the number of bytes per element.
@@ -11780,11 +11790,9 @@ static SDValue lowerShuffleAsPermuteAndUnpack(const SDLoc &DL, MVT VT,
/// Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then
/// permuting the elements of the result in place.
-static SDValue lowerShuffleAsByteRotateAndPermute(const SDLoc &DL, MVT VT,
- SDValue V1, SDValue V2,
- ArrayRef<int> Mask,
- const X86Subtarget &Subtarget,
- SelectionDAG &DAG) {
+static SDValue lowerShuffleAsByteRotateAndPermute(
+ const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
+ const X86Subtarget &Subtarget, SelectionDAG &DAG) {
if ((VT.is128BitVector() && !Subtarget.hasSSSE3()) ||
(VT.is256BitVector() && !Subtarget.hasAVX2()) ||
(VT.is512BitVector() && !Subtarget.hasBWI()))
@@ -11985,9 +11993,9 @@ static SDValue lowerShuffleAsDecomposedShuffleMerge(
// If either input vector provides only a single element which is repeated
// multiple times, unpacking from both input vectors would generate worse
// code. e.g. for
- // t5: v16i8 = vector_shuffle<16,0,16,1,16,2,16,3,16,4,16,5,16,6,16,7> t2,
- // t4 it is better to process t4 first to create a vector of t4[0], then
- // unpack that vector with t2.
+ // t5: v16i8 = vector_shuffle<16,0,16,1,16,2,16,3,16,4,16,5,16,6,16,7> t2, t4
+ // it is better to process t4 first to create a vector of t4[0], then unpack
+ // that vector with t2.
if (!V1Zero && !V2Zero && !isSingleElementRepeatedMask(V1Mask) &&
!isSingleElementRepeatedMask(V2Mask))
if (SDValue UnpackPerm =
@@ -11999,8 +12007,8 @@ static SDValue lowerShuffleAsDecomposedShuffleMerge(
return RotatePerm;
// Unpack/rotate failed - try again with variable blends.
- if (SDValue BlendPerm =
- lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask, DAG))
+ if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
+ DAG))
return BlendPerm;
if (EltSizeInBits >= 32)
@@ -12115,7 +12123,7 @@ static int matchShuffleAsElementRotate(SDValue &V1, SDValue &V2,
SDValue Lo, Hi;
for (int i = 0; i < NumElts; ++i) {
int M = Mask[i];
- assert((M == SM_SentinelUndef || (0 <= M && M < (2 * NumElts))) &&
+ assert((M == SM_SentinelUndef || (0 <= M && M < (2*NumElts))) &&
"Unexpected mask index.");
if (M < 0)
continue;
@@ -12237,7 +12245,8 @@ static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
"Rotate-based lowering only supports 128-bit lowering!");
assert(Mask.size() <= 16 &&
"Can shuffle at most 16 bytes in a 128-bit vector!");
- assert(ByteVT == MVT::v16i8 && "SSE2 rotate lowering only needed for v16i8!");
+ assert(ByteVT == MVT::v16i8 &&
+ "SSE2 rotate lowering only needed for v16i8!");
// Default SSE2 implementation
int LoByteShift = 16 - ByteRotation;
@@ -12870,7 +12879,8 @@ static SDValue lowerShuffleAsSpecificExtension(const SDLoc &DL, MVT VT,
/// are both incredibly common and often quite performance sensitive.
static SDValue lowerShuffleAsZeroOrAnyExtend(
const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
- const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
+ const APInt &Zeroable, const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
int Bits = VT.getSizeInBits();
int NumLanes = Bits / 128;
int NumElements = VT.getVectorNumElements();
@@ -13003,8 +13013,7 @@ static SDValue getScalarValueForVectorElement(SDValue V, int Idx,
// If the bitcasts shift the element size, we can't extract an equivalent
// element from it.
MVT NewVT = V.getSimpleValueType();
- if (!NewVT.isVector() ||
- NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
+ if (!NewVT.isVector() || NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
return SDValue();
if (V.getOpcode() == ISD::BUILD_VECTOR ||
@@ -13028,14 +13037,14 @@ static bool isShuffleFoldableLoad(SDValue V) {
ISD::isNON_EXTLoad(peekThroughOneUseBitcasts(V).getNode());
}
-template <typename T>
+template<typename T>
static bool isSoftF16(T VT, const X86Subtarget &Subtarget) {
T EltVT = VT.getScalarType();
return (EltVT == MVT::bf16 && !Subtarget.hasAVX10_2()) ||
(EltVT == MVT::f16 && !Subtarget.hasFP16());
}
-template <typename T>
+template<typename T>
static bool isBF16orSoftF16(T VT, const X86Subtarget &Subtarget) {
T EltVT = VT.getScalarType();
return EltVT == MVT::bf16 || (EltVT == MVT::f16 && !Subtarget.hasFP16());
@@ -13047,7 +13056,8 @@ static bool isBF16orSoftF16(T VT, const X86Subtarget &Subtarget) {
/// across all subtarget feature sets.
static SDValue lowerShuffleAsElementInsertion(
const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
- const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
+ const APInt &Zeroable, const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
MVT ExtVT = VT;
MVT EltVT = VT.getVectorElementType();
unsigned NumElts = VT.getVectorNumElements();
@@ -13080,8 +13090,8 @@ static SDValue lowerShuffleAsElementInsertion(
// all the smarts here sunk into that routine. However, the current
// lowering of BUILD_VECTOR makes that nearly impossible until the old
// vector shuffle lowering is dead.
- SDValue V2S =
- getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(), DAG);
+ SDValue V2S = getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(),
+ DAG);
if (V2S && DAG.getTargetLoweringInfo().isTypeLegal(V2S.getValueType())) {
// We need to zext the scalar if it is smaller than an i32.
V2S = DAG.getBitcast(EltVT, V2S);
@@ -13284,8 +13294,8 @@ static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
// Check that both sources are extracts of the same source vector.
if (N0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
N1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
- N0.getOperand(0) != N1.getOperand(0) || !N0.hasOneUse() ||
- !N1.hasOneUse())
+ N0.getOperand(0) != N1.getOperand(0) ||
+ !N0.hasOneUse() || !N1.hasOneUse())
return SDValue();
SDValue WideVec = N0.getOperand(0);
@@ -13315,8 +13325,8 @@ static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
NewMask.append(NumElts, -1);
// shuf (extract X, 0), (extract X, 4), M --> extract (shuf X, undef, M'), 0
- SDValue Shuf =
- DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT), NewMask);
+ SDValue Shuf = DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT),
+ NewMask);
// This is free: ymm -> xmm.
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuf,
DAG.getVectorIdxConstant(0, DL));
@@ -13515,8 +13525,8 @@ static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
if (!V.getValueType().isVector()) {
assert(V.getScalarValueSizeInBits() == NumEltBits &&
"Unexpected scalar size");
- MVT BroadcastVT =
- MVT::getVectorVT(V.getSimpleValueType(), VT.getVectorNumElements());
+ MVT BroadcastVT = MVT::getVectorVT(V.getSimpleValueType(),
+ VT.getVectorNumElements());
return DAG.getBitcast(VT, DAG.getNode(Opcode, DL, BroadcastVT, V));
}
@@ -13541,8 +13551,8 @@ static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
// elements are zeroable.
static bool matchShuffleAsInsertPS(SDValue &V1, SDValue &V2,
unsigned &InsertPSMask,
- const APInt &Zeroable, ArrayRef<int> Mask,
- SelectionDAG &DAG) {
+ const APInt &Zeroable,
+ ArrayRef<int> Mask, SelectionDAG &DAG) {
assert(V1.getSimpleValueType().is128BitVector() && "Bad operand type!");
assert(V2.getSimpleValueType().is128BitVector() && "Bad operand type!");
assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
@@ -14100,8 +14110,8 @@ static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// There are special ways we can lower some single-element blends.
if (NumV2Elements == 1)
- if (SDValue V = lowerShuffleAsElementInsertion(DL, MVT::v4i32, V1, V2, Mask,
- Zeroable, Subtarget, DAG))
+ if (SDValue V = lowerShuffleAsElementInsertion(
+ DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
return V;
// We have different paths for blend lowering, but they all must use the
@@ -14231,7 +14241,7 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
};
if ((NumHToL + NumHToH) == 0 || (NumLToL + NumLToH) == 0) {
- int PSHUFDMask[4] = {-1, -1, -1, -1};
+ int PSHUFDMask[4] = { -1, -1, -1, -1 };
SmallVector<std::pair<int, int>, 4> DWordPairs;
int DOffset = ((NumHToL + NumHToH) == 0 ? 0 : 2);
@@ -14335,8 +14345,7 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
int OneInput = ThreeAInputs ? BToAInputs[0] : AToAInputs[0];
int TripleInputSum = 0 + 1 + 2 + 3 + (4 * TripleInputOffset);
int TripleNonInputIdx =
- TripleInputSum -
- std::accumulate(TripleInputs.begin(), TripleInputs.end(), 0);
+ TripleInputSum - std::accumulate(TripleInputs.begin(), TripleInputs.end(), 0);
TripleDWord = TripleNonInputIdx / 2;
// We use xor with one to compute the adjacent DWord to whichever one the
@@ -14414,9 +14423,9 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
// Adjust the mask to match the new locations of A and B.
for (int &M : Mask)
- if (M >= 0 && M / 2 == ADWord)
+ if (M >= 0 && M/2 == ADWord)
M = 2 * BDWord + M % 2;
- else if (M >= 0 && M / 2 == BDWord)
+ else if (M >= 0 && M/2 == BDWord)
M = 2 * ADWord + M % 2;
// Recurse back into this routine to re-compute state now that this isn't
@@ -14444,33 +14453,33 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
[&PSHUFDMask](ArrayRef<int> InPlaceInputs, ArrayRef<int> IncomingInputs,
MutableArrayRef<int> SourceHalfMask,
MutableArrayRef<int> HalfMask, int HalfOffset) {
- if (InPlaceInputs.empty())
- return;
- if (InPlaceInputs.size() == 1) {
- SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
- InPlaceInputs[0] - HalfOffset;
- PSHUFDMask[InPlaceInputs[0] / 2] = InPlaceInputs[0] / 2;
- return;
- }
- if (IncomingInputs.empty()) {
- // Just fix all of the in place inputs.
- for (int Input : InPlaceInputs) {
- SourceHalfMask[Input - HalfOffset] = Input - HalfOffset;
- PSHUFDMask[Input / 2] = Input / 2;
- }
- return;
- }
+ if (InPlaceInputs.empty())
+ return;
+ if (InPlaceInputs.size() == 1) {
+ SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
+ InPlaceInputs[0] - HalfOffset;
+ PSHUFDMask[InPlaceInputs[0] / 2] = InPlaceInputs[0] / 2;
+ return;
+ }
+ if (IncomingInputs.empty()) {
+ // Just fix all of the in place inputs.
+ for (int Input : InPlaceInputs) {
+ SourceHalfMask[Input - HalfOffset] = Input - HalfOffset;
+ PSHUFDMask[Input / 2] = Input / 2;
+ }
+ return;
+ }
- assert(InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!");
- SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
- InPlaceInputs[0] - HalfOffset;
- // Put the second input next to the first so that they are packed into
- // a dword. We find the adjacent index by toggling the low bit.
- int AdjIndex = InPlaceInputs[0] ^ 1;
- SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset;
- llvm::replace(HalfMask, InPlaceInputs[1], AdjIndex);
- PSHUFDMask[AdjIndex / 2] = AdjIndex / 2;
- };
+ assert(InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!");
+ SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
+ InPlaceInputs[0] - HalfOffset;
+ // Put the second input next to the first so that they are packed into
+ // a dword. We find the adjacent index by toggling the low bit.
+ int AdjIndex = InPlaceInputs[0] ^ 1;
+ SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset;
+ llvm::replace(HalfMask, InPlaceInputs[1], AdjIndex);
+ PSHUFDMask[AdjIndex / 2] = AdjIndex / 2;
+ };
fixInPlaceInputs(LToLInputs, HToLInputs, PSHUFLMask, LoMask, 0);
fixInPlaceInputs(HToHInputs, LToHInputs, PSHUFHMask, HiMask, 4);
@@ -14479,12 +14488,10 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
// FIXME: This operation could almost certainly be simplified dramatically to
// look more like the 3-1 fixing operation.
auto moveInputsToRightHalf = [&PSHUFDMask](
- MutableArrayRef<int> IncomingInputs,
- ArrayRef<int> ExistingInputs,
- MutableArrayRef<int> SourceHalfMask,
- MutableArrayRef<int> HalfMask,
- MutableArrayRef<int> FinalSourceHalfMask,
- int SourceOffset, int DestOffset) {
+ MutableArrayRef<int> IncomingInputs, ArrayRef<int> ExistingInputs,
+ MutableArrayRef<int> SourceHalfMask, MutableArrayRef<int> HalfMask,
+ MutableArrayRef<int> FinalSourceHalfMask, int SourceOffset,
+ int DestOffset) {
auto isWordClobbered = [](ArrayRef<int> SourceHalfMask, int Word) {
return SourceHalfMask[Word] >= 0 && SourceHalfMask[Word] != Word;
};
@@ -14680,11 +14687,9 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
/// Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the
/// blend if only one input is used.
-static SDValue lowerShuffleAsBlendOfPSHUFBs(const SDLoc &DL, MVT VT, SDValue V1,
- SDValue V2, ArrayRef<int> Mask,
- const APInt &Zeroable,
- SelectionDAG &DAG, bool &V1InUse,
- bool &V2InUse) {
+static SDValue lowerShuffleAsBlendOfPSHUFBs(
+ const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
+ const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) {
assert(!is128BitLaneCrossingShuffleMask(VT, Mask) &&
"Lane crossing shuffle masks not supported");
@@ -14779,8 +14784,8 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Broadcast;
// Try to use bit rotation instructions.
- if (SDValue Rotate =
- lowerShuffleAsBitRotate(DL, MVT::v8i16, V1, Mask, Subtarget, DAG))
+ if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v8i16, V1, Mask,
+ Subtarget, DAG))
return Rotate;
// Use dedicated unpack instructions for masks that match their pattern.
@@ -14815,14 +14820,14 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// See if we can use SSE4A Extraction / Insertion.
if (Subtarget.hasSSE4A())
- if (SDValue V =
- lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask, Zeroable, DAG))
+ if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask,
+ Zeroable, DAG))
return V;
// There are special ways we can lower some single-element blends.
if (NumV2Inputs == 1)
- if (SDValue V = lowerShuffleAsElementInsertion(DL, MVT::v8i16, V1, V2, Mask,
- Zeroable, Subtarget, DAG))
+ if (SDValue V = lowerShuffleAsElementInsertion(
+ DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
return V;
// We have different paths for blend lowering, but they all must use the
@@ -14938,8 +14943,8 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// can both shuffle and set up the inefficient blend.
if (!IsBlendSupported && Subtarget.hasSSSE3()) {
bool V1InUse, V2InUse;
- return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask, Zeroable,
- DAG, V1InUse, V2InUse);
+ return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask,
+ Zeroable, DAG, V1InUse, V2InUse);
}
// We can always bit-blend if we have to so the fallback strategy is to
@@ -15072,8 +15077,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// See if we can use SSE4A Extraction / Insertion.
if (Subtarget.hasSSE4A())
- if (SDValue V =
- lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG))
+ if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask,
+ Zeroable, DAG))
return V;
int NumV2Elements = count_if(Mask, [](int M) { return M >= 16; });
@@ -15086,8 +15091,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Broadcast;
// Try to use bit rotation instructions.
- if (SDValue Rotate =
- lowerShuffleAsBitRotate(DL, MVT::v16i8, V1, Mask, Subtarget, DAG))
+ if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v16i8, V1, Mask,
+ Subtarget, DAG))
return Rotate;
if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask, DAG))
@@ -15128,7 +15133,7 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
int PreDupI16Shuffle[] = {-1, -1, -1, -1, -1, -1, -1, -1};
SmallDenseMap<int, int, 8> LaneMap;
for (int I : InPlaceInputs) {
- PreDupI16Shuffle[I / 2] = I / 2;
+ PreDupI16Shuffle[I/2] = I/2;
LaneMap[I] = I;
}
int j = TargetLo ? 0 : 4, je = j + 4;
@@ -15142,8 +15147,7 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
++j;
if (j == je)
- // We can't place the inputs into a single half with a simple i16
- // shuffle, so bail.
+ // We can't place the inputs into a single half with a simple i16 shuffle, so bail.
return SDValue();
// Map this input with the i16 shuffle.
@@ -15264,8 +15268,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// Use PALIGNR+Permute if possible - permute might become PSHUFB but the
// PALIGNR will be cheaper than the second PSHUFB+OR.
- if (SDValue V = lowerShuffleAsByteRotateAndPermute(DL, MVT::v16i8, V1, V2,
- Mask, Subtarget, DAG))
+ if (SDValue V = lowerShuffleAsByteRotateAndPermute(
+ DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
return V;
}
@@ -15274,8 +15278,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// There are special ways we can lower some single-element blends.
if (NumV2Elements == 1)
- if (SDValue V = lowerShuffleAsElementInsertion(DL, MVT::v16i8, V1, V2, Mask,
- Zeroable, Subtarget, DAG))
+ if (SDValue V = lowerShuffleAsElementInsertion(
+ DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
return V;
if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
@@ -15367,8 +15371,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
if (M >= 0)
M /= 2;
} else {
- // Otherwise just unpack the low half of V into VLoHalf and the high half
- // into VHiHalf so that we can blend them as i16s.
+ // Otherwise just unpack the low half of V into VLoHalf and the high half into
+ // VHiHalf so that we can blend them as i16s.
SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL);
VLoHalf = DAG.getBitcast(
@@ -15377,10 +15381,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
}
- SDValue LoV =
- DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
- SDValue HiV =
- DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);
+ SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
+ SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);
return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
}
@@ -15389,8 +15391,9 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
///
/// This routine breaks down the specific type of 128-bit shuffle and
/// dispatches to the lowering routines accordingly.
-static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
- SDValue V1, SDValue V2, const APInt &Zeroable,
+static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
+ MVT VT, SDValue V1, SDValue V2,
+ const APInt &Zeroable,
const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
if (VT == MVT::v8bf16) {
@@ -15572,7 +15575,7 @@ static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
assert(!V2.isUndef() && "This routine must not be used to lower single-input "
- "shuffles as it could then recurse on itself.");
+ "shuffles as it could then recurse on itself.");
int Size = Mask.size();
// If this can be modeled as a broadcast of two elements followed by a blend,
@@ -15911,8 +15914,8 @@ static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
// instruction bytes needed to explicitly generate the zero vector.
// Blends are faster and handle all the non-lane-crossing cases.
- if (SDValue Blend =
- lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
+ if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable,
+ Subtarget, DAG))
return Blend;
// If either input operand is a zero vector, use VPERM2X128 because its mask
@@ -15938,8 +15941,8 @@ static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
// Try to use SHUF128 if possible.
if (Subtarget.hasVLX()) {
if (WidenedMask[0] < 2 && WidenedMask[1] >= 2) {
- unsigned PermMask =
- ((WidenedMask[0] % 2) << 0) | ((WidenedMask[1] % 2) << 1);
+ unsigned PermMask = ((WidenedMask[0] % 2) << 0) |
+ ((WidenedMask[1] % 2) << 1);
return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2,
DAG.getTargetConstant(PermMask, DL, MVT::i8));
}
@@ -15963,7 +15966,7 @@ static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
(WidenedMask[1] >= 0 || IsHighZero) && "Undef half?");
unsigned PermMask = 0;
- PermMask |= IsLowZero ? 0x08 : (WidenedMask[0] << 0);
+ PermMask |= IsLowZero ? 0x08 : (WidenedMask[0] << 0);
PermMask |= IsHighZero ? 0x80 : (WidenedMask[1] << 4);
// Check the immediate mask and replace unused sources with undef.
@@ -16154,9 +16157,9 @@ static SDValue lowerShuffleAsLanePermuteAndRepeatedMask(
/// adjusted to access the extracted halves of the original shuffle operands is
/// returned in HalfMask. HalfIdx1 and HalfIdx2 return whether the upper or
/// lower half of each input operand is accessed.
-static bool getHalfShuffleMask(ArrayRef<int> Mask,
- MutableArrayRef<int> HalfMask, int &HalfIdx1,
- int &HalfIdx2) {
+static bool
+getHalfShuffleMask(ArrayRef<int> Mask, MutableArrayRef<int> HalfMask,
+ int &HalfIdx1, int &HalfIdx2) {
assert((Mask.size() == HalfMask.size() * 2) &&
"Expected input mask to be twice as long as output");
@@ -16209,8 +16212,7 @@ static bool getHalfShuffleMask(ArrayRef<int> Mask,
static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2,
ArrayRef<int> HalfMask, int HalfIdx1,
int HalfIdx2, bool UndefLower,
- SelectionDAG &DAG,
- bool UseConcat = false) {
+ SelectionDAG &DAG, bool UseConcat = false) {
assert(V1.getValueType() == V2.getValueType() && "Different sized vectors?");
assert(V1.getValueType().isSimple() && "Expecting only simple types");
@@ -16572,7 +16574,7 @@ static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
assert(isUndefOrZeroOrInRange(Mask, 0, 2 * NumElts) &&
"Illegal shuffle mask");
- bool ZeroLane[2] = {true, true};
+ bool ZeroLane[2] = { true, true };
for (int i = 0; i < NumElts; ++i)
ZeroLane[i & 1] &= Zeroable[i];
@@ -16657,9 +16659,9 @@ static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT,
// The VTRUNCs will put 0s in the upper 12 bytes. Use them to put zeroes in
// the upper bits of the result using an unpckldq.
- SDValue Unpack = DAG.getVectorShuffle(
- MVT::v16i8, DL, V1, V2,
- {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23});
+ SDValue Unpack = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2,
+ { 0, 1, 2, 3, 16, 17, 18, 19,
+ 4, 5, 6, 7, 20, 21, 22, 23 });
// Insert the unpckldq into a zero vector to widen to v32i8.
return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v32i8,
DAG.getConstant(0, DL, MVT::v32i8), Unpack,
@@ -16896,8 +16898,8 @@ static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Blend;
// Check for being able to broadcast a single element.
- if (SDValue Broadcast =
- lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
+ if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask,
+ Subtarget, DAG))
return Broadcast;
// Try to use shift instructions if fast.
@@ -17004,8 +17006,8 @@ static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Blend;
// Check for being able to broadcast a single element.
- if (SDValue Broadcast =
- lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
+ if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask,
+ Subtarget, DAG))
return Broadcast;
if (!Subtarget.hasAVX2()) {
@@ -17152,8 +17154,8 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Blend;
// Check for being able to broadcast a single element.
- if (SDValue Broadcast =
- lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
+ if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask,
+ Subtarget, DAG))
return Broadcast;
// Try to use shift instructions if fast.
@@ -17320,8 +17322,7 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// Try to produce a fixed cross-128-bit lane permute followed by unpack
// because that should be faster than the variable permute alternatives.
- if (SDValue V =
- lowerShuffleWithUNPCK256(DL, MVT::v16i16, V1, V2, Mask, DAG))
+ if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, V1, V2, Mask, DAG))
return V;
// There are no generalized cross-lane shuffle operations available on i16
@@ -17340,8 +17341,8 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// As this is a single-input shuffle, the repeated mask should be
// a strictly valid v8i16 mask that we can pass through to the v8i16
// lowering to handle even the v16 case.
- return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v16i16, V1,
- RepeatedMask, Subtarget, DAG);
+ return lowerV8I16GeneralSingleInputShuffle(
+ DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG);
}
}
@@ -17360,8 +17361,8 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Result;
// Try to permute the lanes and then use a per-lane permute.
- if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v16i16, V1, V2,
- Mask, DAG, Subtarget))
+ if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
+ DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
return V;
// Try to match an interleave of two v16i16s and lower them as unpck and
@@ -17397,8 +17398,8 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return ZExt;
// Check for being able to broadcast a single element.
- if (SDValue Broadcast =
- lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
+ if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask,
+ Subtarget, DAG))
return Broadcast;
if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
@@ -17450,8 +17451,8 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, V1, V2, Mask, DAG))
return V;
- if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v32i8, V1, V2,
- Mask, DAG, Subtarget))
+ if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
+ DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
return V;
return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v32i8, V1, V2, Mask,
@@ -17473,16 +17474,16 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
return Result;
// Try to permute the lanes and then use a per-lane permute.
- if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v32i8, V1, V2,
- Mask, DAG, Subtarget))
+ if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
+ DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
return V;
// Look for {0, 8, 16, 24, 32, 40, 48, 56 } in the first 8 elements. Followed
// by zeroable elements in the remaining 24 elements. Turn this into two
// vmovqb instructions shuffled together.
if (Subtarget.hasVLX())
- if (SDValue V = lowerShuffleAsVTRUNCAndUnpack(DL, MVT::v32i8, V1, V2, Mask,
- Zeroable, DAG))
+ if (SDValue V = lowerShuffleAsVTRUNCAndUnpack(DL, MVT::v32i8, V1, V2,
+ Mask, Zeroable, DAG))
return V;
// Try to match an interleave of two v32i8s and lower them as unpck and
@@ -17537,8 +17538,7 @@ static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
return V;
if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
return V;
- return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
- /*SimpleOnly*/ false);
+ return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
}
MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits),
@@ -17793,7 +17793,8 @@ static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// If we have a single input shuffle with different shuffle patterns in the
// 128-bit lanes and don't lane cross, use variable mask VPERMILPS.
- if (V2.isUndef() && !is128BitLaneCrossingShuffleMask(MVT::v16f32, Mask)) {
+ if (V2.isUndef() &&
+ !is128BitLaneCrossingShuffleMask(MVT::v16f32, Mask)) {
SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true);
return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v16f32, V1, VPermMask);
}
@@ -18065,8 +18066,8 @@ static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
// Whenever we can lower this as a zext, that instruction is strictly faster
// than any alternative. It also allows us to fold memory operands into the
// shuffle in many cases.
- if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v64i8, V1, V2, Mask,
- Zeroable, Subtarget, DAG))
+ if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
+ DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
return ZExt;
// Use dedicated unpack instructions for masks that match their pattern.
@@ -18160,11 +18161,13 @@ static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
/// This routine either breaks down the specific type of a 512-bit x86 vector
/// shuffle or splits it into two 256-bit shuffles and fuses the results back
/// together based on the available instructions.
-static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
- SDValue V1, SDValue V2, const APInt &Zeroable,
+static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
+ MVT VT, SDValue V1, SDValue V2,
+ const APInt &Zeroable,
const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
- assert(Subtarget.hasAVX512() && "Cannot lower 512-bit vectors w/ basic ISA!");
+ assert(Subtarget.hasAVX512() &&
+ "Cannot lower 512-bit vectors w/ basic ISA!");
// If we have a single input to the zero element, insert that into V1 if we
// can do so cheaply.
@@ -18182,8 +18185,8 @@ static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
return V;
// Check for being able to broadcast a single element.
- if (SDValue Broadcast =
- lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask, Subtarget, DAG))
+ if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask,
+ Subtarget, DAG))
return Broadcast;
if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) {
@@ -18194,8 +18197,7 @@ static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
return V;
- return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
- /*SimpleOnly*/ false);
+ return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
}
if (VT == MVT::v32f16 || VT == MVT::v32bf16) {
@@ -18302,12 +18304,14 @@ static int match1BitShuffleAsKSHIFT(unsigned &Opcode, ArrayRef<int> Mask,
return -1;
}
+
// Lower vXi1 vector shuffles.
// There is no a dedicated instruction on AVX-512 that shuffles the masks.
// The only way to shuffle bits is to sign-extend the mask vector to SIMD
// vector, shuffle and then truncate it back.
-static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
- SDValue V1, SDValue V2, const APInt &Zeroable,
+static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
+ MVT VT, SDValue V1, SDValue V2,
+ const APInt &Zeroable,
const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
assert(Subtarget.hasAVX512() &&
@@ -18456,8 +18460,8 @@ static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
int NumElems = VT.getVectorNumElements();
if ((Subtarget.hasBWI() && (NumElems >= 32)) ||
(Subtarget.hasDQI() && (NumElems < 32)))
- return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT), Shuffle,
- ISD::SETGT);
+ return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT),
+ Shuffle, ISD::SETGT);
return DAG.getNode(ISD::TRUNCATE, DL, VT, Shuffle);
}
@@ -18586,7 +18590,7 @@ static SDValue canonicalizeShuffleMaskWithHorizOp(
unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG,
const X86Subtarget &Subtarget);
-/// Top-level lowering for x86 vector shuffles.
+ /// Top-level lowering for x86 vector shuffles.
///
/// This handles decomposition, canonicalization, and lowering of all x86
/// vector shuffles. Most of the specific lowering strategies are encapsulated
@@ -18662,8 +18666,8 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, const X86Subtarget &Subtarget,
// by obfuscating the operands with bitcasts.
// TODO: Avoid lowering directly from this top-level function: make this
// a query (canLowerAsBroadcast) and defer lowering to the type-based calls.
- if (SDValue Broadcast =
- lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask, Subtarget, DAG))
+ if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask,
+ Subtarget, DAG))
return Broadcast;
MVT NewEltVT = VT.isFloatingPoint()
@@ -18890,7 +18894,8 @@ SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
// Build a mask by testing the condition against zero.
MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
SDValue Mask = DAG.getSetCC(dl, MaskVT, Cond,
- DAG.getConstant(0, dl, CondVT), ISD::SETNE);
+ DAG.getConstant(0, dl, CondVT),
+ ISD::SETNE);
// Now return a new VSELECT using the mask.
return DAG.getSelect(dl, VT, Mask, LHS, RHS);
}
@@ -18997,7 +19002,7 @@ static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
}
if (VT == MVT::i32 || VT == MVT::i64)
- return Op;
+ return Op;
return SDValue();
}
@@ -19010,7 +19015,7 @@ static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
SDLoc dl(Vec);
MVT VecVT = Vec.getSimpleValueType();
SDValue Idx = Op.getOperand(1);
- auto *IdxC = dyn_cast<ConstantSDNode>(Idx);
+ auto* IdxC = dyn_cast<ConstantSDNode>(Idx);
MVT EltVT = Op.getSimpleValueType();
assert((VecVT.getVectorNumElements() <= 16 || Subtarget.hasBWI()) &&
@@ -19025,8 +19030,7 @@ static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
if (NumElts == 1) {
Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
MVT IntVT = MVT::getIntegerVT(Vec.getValueType().getVectorNumElements());
- return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8,
- DAG.getBitcast(IntVT, Vec));
+ return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, DAG.getBitcast(IntVT, Vec));
}
MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;
MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);
@@ -19084,13 +19088,14 @@ static APInt getExtractedDemandedElts(SDNode *N) {
return DemandedElts;
}
-SDValue X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue
+X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
+ SelectionDAG &DAG) const {
SDLoc dl(Op);
SDValue Vec = Op.getOperand(0);
MVT VecVT = Vec.getSimpleValueType();
SDValue Idx = Op.getOperand(1);
- auto *IdxC = dyn_cast<ConstantSDNode>(Idx);
+ auto* IdxC = dyn_cast<ConstantSDNode>(Idx);
if (VecVT.getVectorElementType() == MVT::i1)
return ExtractBitFromMaskVector(Op, DAG, Subtarget);
@@ -19121,10 +19126,10 @@ SDValue X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
// | | Ports pressure in cycles | |
// |Uops| 1 | 2 - D |3 - D | 4 | 5 | |
// ---------------------------------------------------------
- // |2^ | | 0.5 | 0.5 |1.0| |CP| vmovaps xmmword ptr [rsp-0x18],
- // xmm0 |1 |0.5| | | |0.5| | lea rax, ptr [rsp-0x18] |1
- // | |0.5, 0.5|0.5, 0.5| | |CP| mov al, byte ptr [rdi+rax*1] Total Num
- // Of Uops: 4
+ // |2^ | | 0.5 | 0.5 |1.0| |CP| vmovaps xmmword ptr [rsp-0x18], xmm0
+ // |1 |0.5| | | |0.5| | lea rax, ptr [rsp-0x18]
+ // |1 | |0.5, 0.5|0.5, 0.5| | |CP| mov al, byte ptr [rdi+rax*1]
+ // Total Num Of Uops: 4
return SDValue();
}
@@ -19229,7 +19234,7 @@ SDValue X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
// UNPCKHPD the element to the lowest double word, then movsd.
// Note if the lower 64 bits of the result of the UNPCKHPD is then stored
// to a f64mem, the whole operation is folded into a single MOVHPDmr.
- int Mask[2] = {1, -1};
+ int Mask[2] = { 1, -1 };
Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask);
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
DAG.getVectorIdxConstant(0, dl));
@@ -19254,10 +19259,9 @@ static SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG,
unsigned NumElts = VecVT.getVectorNumElements();
MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;
MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);
- SDValue ExtOp =
- DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
- DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec),
- DAG.getNode(ISD::SIGN_EXTEND, dl, ExtEltVT, Elt), Idx);
+ SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
+ DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec),
+ DAG.getNode(ISD::SIGN_EXTEND, dl, ExtEltVT, Elt), Idx);
return DAG.getNode(ISD::TRUNCATE, dl, VecVT, ExtOp);
}
@@ -19284,9 +19288,9 @@ SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
if (EltVT == MVT::bf16) {
MVT IVT = VT.changeVectorElementTypeToInteger();
- SDValue Res =
- DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVT, DAG.getBitcast(IVT, N0),
- DAG.getBitcast(MVT::i16, N1), N2);
+ SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVT,
+ DAG.getBitcast(IVT, N0),
+ DAG.getBitcast(MVT::i16, N1), N2);
return DAG.getBitcast(VT, Res);
}
@@ -19621,9 +19625,8 @@ static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
}
// Returns the appropriate wrapper opcode for a global reference.
-unsigned
-X86TargetLowering::getGlobalWrapperKind(const GlobalValue *GV,
- const unsigned char OpFlags) const {
+unsigned X86TargetLowering::getGlobalWrapperKind(
+ const GlobalValue *GV, const unsigned char OpFlags) const {
// References to absolute symbols are never PC-relative.
if (GV && GV->isAbsoluteSymbolRef())
return X86ISD::Wrapper;
@@ -19647,8 +19650,8 @@ X86TargetLowering::getGlobalWrapperKind(const GlobalValue *GV,
// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
// be used to form addressing mode. These wrapped nodes will be selected
// into MOV32ri.
-SDValue X86TargetLowering::LowerConstantPool(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue
+X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
// In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
@@ -19698,10 +19701,11 @@ SDValue X86TargetLowering::LowerExternalSymbol(SDValue Op,
return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false, nullptr);
}
-SDValue X86TargetLowering::LowerBlockAddress(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue
+X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
// Create the TargetBlockAddressAddress node.
- unsigned char OpFlags = Subtarget.classifyBlockAddressReference();
+ unsigned char OpFlags =
+ Subtarget.classifyBlockAddressReference();
const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
SDLoc dl(Op);
@@ -19806,8 +19810,8 @@ SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
return Result;
}
-SDValue X86TargetLowering::LowerGlobalAddress(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue
+X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false, nullptr);
}
@@ -19885,24 +19889,24 @@ static SDValue GetTLSADDR(SelectionDAG &DAG, GlobalAddressSDNode *GA,
}
// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit
-static SDValue LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA,
- SelectionDAG &DAG,
- const EVT PtrVT) {
+static SDValue
+LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
+ const EVT PtrVT) {
return GetTLSADDR(DAG, GA, PtrVT, X86::EAX, X86II::MO_TLSGD,
/*LoadGlobalBaseReg=*/true);
}
// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit LP64
-static SDValue LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA,
- SelectionDAG &DAG,
- const EVT PtrVT) {
+static SDValue
+LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
+ const EVT PtrVT) {
return GetTLSADDR(DAG, GA, PtrVT, X86::RAX, X86II::MO_TLSGD);
}
// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit ILP32
-static SDValue LowerToTLSGeneralDynamicModelX32(GlobalAddressSDNode *GA,
- SelectionDAG &DAG,
- const EVT PtrVT) {
+static SDValue
+LowerToTLSGeneralDynamicModelX32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
+ const EVT PtrVT) {
return GetTLSADDR(DAG, GA, PtrVT, X86::EAX, X86II::MO_TLSGD);
}
@@ -19934,8 +19938,9 @@ static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
// Build x at dtpoff.
unsigned char OperandFlags = X86II::MO_DTPOFF;
unsigned WrapperKind = X86ISD::Wrapper;
- SDValue TGA = DAG.getTargetGlobalAddress(
- GA->getGlobal(), dl, GA->getValueType(0), GA->getOffset(), OperandFlags);
+ SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
+ GA->getValueType(0),
+ GA->getOffset(), OperandFlags);
SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
// Add x at dtpoff with the base.
@@ -19976,8 +19981,9 @@ static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
// emit "addl x at ntpoff,%eax" (local exec)
// or "addl x at indntpoff,%eax" (initial exec)
// or "addl x at gotntpoff(%ebx) ,%eax" (initial exec, 32-bit pic)
- SDValue TGA = DAG.getTargetGlobalAddress(
- GA->getGlobal(), dl, GA->getValueType(0), GA->getOffset(), OperandFlags);
+ SDValue TGA =
+ DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
+ GA->getOffset(), OperandFlags);
SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
if (model == TLSModel::InitialExec) {
@@ -19996,8 +20002,8 @@ static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
}
-SDValue X86TargetLowering::LowerGlobalTLSAddress(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue
+X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
@@ -20018,20 +20024,20 @@ SDValue X86TargetLowering::LowerGlobalTLSAddress(SDValue Op,
if (model == TLSModel::GeneralDynamic || model == TLSModel::LocalDynamic)
model = TLSModel::InitialExec;
switch (model) {
- case TLSModel::GeneralDynamic:
- if (Subtarget.is64Bit()) {
- if (Subtarget.isTarget64BitLP64())
- return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT);
- return LowerToTLSGeneralDynamicModelX32(GA, DAG, PtrVT);
- }
- return LowerToTLSGeneralDynamicModel32(GA, DAG, PtrVT);
- case TLSModel::LocalDynamic:
- return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT, Subtarget.is64Bit(),
- Subtarget.isTarget64BitLP64());
- case TLSModel::InitialExec:
- case TLSModel::LocalExec:
- return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget.is64Bit(),
- PositionIndependent);
+ case TLSModel::GeneralDynamic:
+ if (Subtarget.is64Bit()) {
+ if (Subtarget.isTarget64BitLP64())
+ return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT);
+ return LowerToTLSGeneralDynamicModelX32(GA, DAG, PtrVT);
+ }
+ return LowerToTLSGeneralDynamicModel32(GA, DAG, PtrVT);
+ case TLSModel::LocalDynamic:
+ return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT, Subtarget.is64Bit(),
+ Subtarget.isTarget64BitLP64());
+ case TLSModel::InitialExec:
+ case TLSModel::LocalExec:
+ return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget.is64Bit(),
+ PositionIndependent);
}
llvm_unreachable("Unknown TLS model.");
}
@@ -20052,8 +20058,9 @@ SDValue X86TargetLowering::LowerGlobalTLSAddress(SDValue Op,
WrapperKind = X86ISD::WrapperRIP;
}
SDLoc DL(Op);
- SDValue Result = DAG.getTargetGlobalAddress(
- GA->getGlobal(), DL, GA->getValueType(0), GA->getOffset(), OpFlag);
+ SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
+ GA->getValueType(0),
+ GA->getOffset(), OpFlag);
SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result);
// With PIC32, the address is actually $g + Offset.
@@ -20067,7 +20074,7 @@ SDValue X86TargetLowering::LowerGlobalTLSAddress(SDValue Op,
SDValue Chain = DAG.getEntryNode();
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
- SDValue Args[] = {Chain, Offset};
+ SDValue Args[] = { Chain, Offset };
Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args);
Chain = DAG.getCALLSEQ_END(Chain, 0, 0, Chain.getValue(1), DL);
@@ -20135,9 +20142,9 @@ SDValue X86TargetLowering::LowerGlobalTLSAddress(SDValue Op,
res = DAG.getLoad(PtrVT, dl, Chain, res, MachinePointerInfo());
// Get the offset of start of .tls section
- SDValue TGA =
- DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
- GA->getOffset(), X86II::MO_SECREL);
+ SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
+ GA->getValueType(0),
+ GA->getOffset(), X86II::MO_SECREL);
SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA);
// The address of the thread local variable is the add of the thread
@@ -20197,8 +20204,8 @@ static SDValue LowerI64IntToFP_AVX512DQ(SDValue Op, const SDLoc &dl,
MVT SrcVT = Src.getSimpleValueType();
MVT VT = Op.getSimpleValueType();
- if (!Subtarget.hasDQI() || SrcVT != MVT::i64 || Subtarget.is64Bit() ||
- (VT != MVT::f32 && VT != MVT::f64))
+ if (!Subtarget.hasDQI() || SrcVT != MVT::i64 || Subtarget.is64Bit() ||
+ (VT != MVT::f32 && VT != MVT::f64))
return SDValue();
// Pack the i64 into a vector, do the operation and extract.
@@ -20263,22 +20270,22 @@ static SDValue LowerI64IntToFP16(SDValue Op, const SDLoc &dl, SelectionDAG &DAG,
static bool useVectorCast(unsigned Opcode, MVT FromVT, MVT ToVT,
const X86Subtarget &Subtarget) {
switch (Opcode) {
- case ISD::SINT_TO_FP:
- // TODO: Handle wider types with AVX/AVX512.
- if (!Subtarget.hasSSE2() || FromVT != MVT::v4i32)
- return false;
- // CVTDQ2PS or (V)CVTDQ2PD
- return ToVT == MVT::v4f32 || (Subtarget.hasAVX() && ToVT == MVT::v4f64);
+ case ISD::SINT_TO_FP:
+ // TODO: Handle wider types with AVX/AVX512.
+ if (!Subtarget.hasSSE2() || FromVT != MVT::v4i32)
+ return false;
+ // CVTDQ2PS or (V)CVTDQ2PD
+ return ToVT == MVT::v4f32 || (Subtarget.hasAVX() && ToVT == MVT::v4f64);
- case ISD::UINT_TO_FP:
- // TODO: Handle wider types and i64 elements.
- if (!Subtarget.hasAVX512() || FromVT != MVT::v4i32)
- return false;
- // VCVTUDQ2PS or VCVTUDQ2PD
- return ToVT == MVT::v4f32 || ToVT == MVT::v4f64;
+ case ISD::UINT_TO_FP:
+ // TODO: Handle wider types and i64 elements.
+ if (!Subtarget.hasAVX512() || FromVT != MVT::v4i32)
+ return false;
+ // VCVTUDQ2PS or VCVTUDQ2PD
+ return ToVT == MVT::v4f32 || ToVT == MVT::v4f64;
- default:
- return false;
+ default:
+ return false;
}
}
@@ -20458,7 +20465,7 @@ static SDValue lowerINT_TO_FP_vXi64(SDValue Op, const SDLoc &DL,
return SDValue();
SDValue Zero = DAG.getConstant(0, DL, MVT::v4i64);
- SDValue One = DAG.getConstant(1, DL, MVT::v4i64);
+ SDValue One = DAG.getConstant(1, DL, MVT::v4i64);
SDValue Sign = DAG.getNode(ISD::OR, DL, MVT::v4i64,
DAG.getNode(ISD::SRL, DL, MVT::v4i64, Src, One),
DAG.getNode(ISD::AND, DL, MVT::v4i64, Src, One));
@@ -20678,7 +20685,7 @@ std::pair<SDValue, SDValue> X86TargetLowering::BuildFILD(
Chain = Result.getValue(1);
}
- return {Result, Chain};
+ return { Result, Chain };
}
/// Horizontal vector math instructions may be slower than normal math with
@@ -20715,18 +20722,18 @@ static SDValue LowerUINT_TO_FP_i64(SDValue Op, const SDLoc &dl,
LLVMContext *Context = DAG.getContext();
// Build some magic constants.
- static const uint32_t CV0[] = {0x43300000, 0x45300000, 0, 0};
+ static const uint32_t CV0[] = { 0x43300000, 0x45300000, 0, 0 };
Constant *C0 = ConstantDataVector::get(*Context, CV0);
auto PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
SDValue CPIdx0 = DAG.getConstantPool(C0, PtrVT, Align(16));
- SmallVector<Constant *, 2> CV1;
+ SmallVector<Constant*,2> CV1;
CV1.push_back(
- ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
- APInt(64, 0x4330000000000000ULL))));
+ ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
+ APInt(64, 0x4330000000000000ULL))));
CV1.push_back(
- ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
- APInt(64, 0x4530000000000000ULL))));
+ ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
+ APInt(64, 0x4530000000000000ULL))));
Constant *C1 = ConstantVector::get(CV1);
SDValue CPIdx1 = DAG.getConstantPool(C1, PtrVT, Align(16));
@@ -20747,10 +20754,11 @@ static SDValue LowerUINT_TO_FP_i64(SDValue Op, const SDLoc &dl,
SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
SDValue Result;
- if (Subtarget.hasSSE3() && shouldUseHorizontalOp(true, DAG, Subtarget)) {
+ if (Subtarget.hasSSE3() &&
+ shouldUseHorizontalOp(true, DAG, Subtarget)) {
Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);
} else {
- SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1, -1});
+ SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1});
Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuffle, Sub);
}
Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result,
@@ -20776,7 +20784,8 @@ static SDValue LowerUINT_TO_FP_i32(SDValue Op, const SDLoc &dl,
// Or the load with the bias.
SDValue Or = DAG.getNode(
- ISD::OR, dl, MVT::v2i64, DAG.getBitcast(MVT::v2i64, Load),
+ ISD::OR, dl, MVT::v2i64,
+ DAG.getBitcast(MVT::v2i64, Load),
DAG.getBitcast(MVT::v2i64,
DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Bias)));
Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
@@ -20980,9 +20989,8 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, const SDLoc &DL,
SDValue VecBitcast = DAG.getBitcast(VecI16VT, V);
// Low will be bitcasted right away, so do not bother bitcasting back to its
// original type.
- Low =
- DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast, VecCstLowBitcast,
- DAG.getTargetConstant(0xaa, DL, MVT::i8));
+ Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
+ VecCstLowBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
// uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
// (uint4) 0x53000000, 0xaa);
SDValue VecCstHighBitcast = DAG.getBitcast(VecI16VT, VecCstHigh);
@@ -20990,8 +20998,7 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, const SDLoc &DL,
// High will be bitcasted right away, so do not bother bitcasting back to
// its original type.
High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
- VecCstHighBitcast,
- DAG.getTargetConstant(0xaa, DL, MVT::i8));
+ VecCstHighBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
} else {
SDValue VecCstMask = DAG.getConstant(0xffff, DL, VecIntVT);
// uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
@@ -21027,8 +21034,7 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, const SDLoc &DL,
return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh);
}
-static SDValue lowerUINT_TO_FP_vec(SDValue Op, const SDLoc &dl,
- SelectionDAG &DAG,
+static SDValue lowerUINT_TO_FP_vec(SDValue Op, const SDLoc &dl, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0;
SDValue N0 = Op.getOperand(OpNo);
@@ -21242,7 +21248,8 @@ SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
DstTy = MVT::i64;
}
- assert(DstTy.getSimpleVT() <= MVT::i64 && DstTy.getSimpleVT() >= MVT::i16 &&
+ assert(DstTy.getSimpleVT() <= MVT::i64 &&
+ DstTy.getSimpleVT() >= MVT::i16 &&
"Unknown FP_TO_INT to lower!");
// We lower FP->int64 into FISTP64 followed by a load from a temporary
@@ -21280,8 +21287,8 @@ SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
bool LosesInfo = false;
if (TheVT == MVT::f64)
// The rounding mode is irrelevant as the conversion should be exact.
- Status = Thresh.convert(APFloat::IEEEdouble(),
- APFloat::rmNearestTiesToEven, &LosesInfo);
+ Status = Thresh.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
+ &LosesInfo);
else if (TheVT == MVT::f80)
Status = Thresh.convert(APFloat::x87DoubleExtended(),
APFloat::rmNearestTiesToEven, &LosesInfo);
@@ -21291,8 +21298,8 @@ SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
SDValue ThreshVal = DAG.getConstantFP(Thresh, DL, TheVT);
- EVT ResVT =
- getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), TheVT);
+ EVT ResVT = getSetCCResultType(DAG.getDataLayout(),
+ *DAG.getContext(), TheVT);
SDValue Cmp;
if (IsStrict) {
Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE, Chain,
@@ -21321,8 +21328,8 @@ SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
DAG.getConstantFP(0.0, DL, TheVT));
if (IsStrict) {
- Value = DAG.getNode(ISD::STRICT_FSUB, DL, {TheVT, MVT::Other},
- {Chain, Value, FltOfs});
+ Value = DAG.getNode(ISD::STRICT_FSUB, DL, { TheVT, MVT::Other},
+ { Chain, Value, FltOfs });
Chain = Value.getValue(1);
} else
Value = DAG.getNode(ISD::FSUB, DL, TheVT, Value, FltOfs);
@@ -21336,7 +21343,7 @@ SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!");
Chain = DAG.getStore(Chain, DL, Value, StackSlot, MPI);
SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
- SDValue Ops[] = {Chain, StackSlot};
+ SDValue Ops[] = { Chain, StackSlot };
unsigned FLDSize = TheVT.getStoreSize();
assert(FLDSize <= MemSize && "Stack slot not big enough");
@@ -21349,9 +21356,10 @@ SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
// Build the FP_TO_INT*_IN_MEM
MachineMemOperand *MMO = MF.getMachineMemOperand(
MPI, MachineMemOperand::MOStore, MemSize, Align(MemSize));
- SDValue Ops[] = {Chain, Value, StackSlot};
- SDValue FIST = DAG.getMemIntrinsicNode(
- X86ISD::FP_TO_INT_IN_MEM, DL, DAG.getVTList(MVT::Other), Ops, DstTy, MMO);
+ SDValue Ops[] = { Chain, Value, StackSlot };
+ SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL,
+ DAG.getVTList(MVT::Other),
+ Ops, DstTy, MMO);
SDValue Res = DAG.getLoad(Op.getValueType(), DL, FIST, StackSlot, MPI);
Chain = Res.getValue(1);
@@ -21530,7 +21538,7 @@ static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
return In;
unsigned NumElems = SrcVT.getVectorNumElements();
- if (NumElems < 2 || !isPowerOf2_32(NumElems))
+ if (NumElems < 2 || !isPowerOf2_32(NumElems) )
return SDValue();
unsigned DstSizeInBits = DstVT.getSizeInBits();
@@ -21601,7 +21609,7 @@ static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
// Scale shuffle mask to avoid bitcasts and help ComputeNumSignBits.
SmallVector<int, 64> Mask;
int Scale = 64 / OutVT.getScalarSizeInBits();
- narrowShuffleMaskElts(Scale, {0, 2, 1, 3}, Mask);
+ narrowShuffleMaskElts(Scale, { 0, 2, 1, 3 }, Mask);
Res = DAG.getVectorShuffle(OutVT, DL, Res, Res, Mask);
if (DstVT.is256BitVector())
@@ -21845,12 +21853,14 @@ static SDValue LowerTruncateVecI1(SDValue Op, const SDLoc &DL,
if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) {
// We need to shift to get the lsb into sign position.
// Shift packed bytes not supported natively, bitcast to word
- MVT ExtVT = MVT::getVectorVT(MVT::i16, InVT.getSizeInBits() / 16);
- In = DAG.getNode(ISD::SHL, DL, ExtVT, DAG.getBitcast(ExtVT, In),
+ MVT ExtVT = MVT::getVectorVT(MVT::i16, InVT.getSizeInBits()/16);
+ In = DAG.getNode(ISD::SHL, DL, ExtVT,
+ DAG.getBitcast(ExtVT, In),
DAG.getConstant(ShiftInx, DL, ExtVT));
In = DAG.getBitcast(InVT, In);
}
- return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), In, ISD::SETGT);
+ return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT),
+ In, ISD::SETGT);
}
// Use TESTD/Q, extended vector to packed dword/qword.
assert((InVT.is256BitVector() || InVT.is128BitVector()) &&
@@ -21888,8 +21898,7 @@ static SDValue LowerTruncateVecI1(SDValue Op, const SDLoc &DL,
// We either have 8 elements or we're allowed to use 512-bit vectors.
// If we have VLX, we want to use the narrowest vector that can get the
// job done so we use vXi32.
- MVT EltVT =
- Subtarget.hasVLX() ? MVT::i32 : MVT::getIntegerVT(512 / NumElts);
+ MVT EltVT = Subtarget.hasVLX() ? MVT::i32 : MVT::getIntegerVT(512/NumElts);
MVT ExtVT = MVT::getVectorVT(EltVT, NumElts);
In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In);
InVT = ExtVT;
@@ -22003,9 +22012,10 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
// On AVX2, v8i32 -> v8i16 becomes PSHUFB.
if (Subtarget.hasInt256()) {
// The PSHUFB mask:
- static const int ShufMask1[] = {
- 0, 1, 4, 5, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1,
- 16, 17, 20, 21, 24, 25, 28, 29, -1, -1, -1, -1, -1, -1, -1, -1};
+ static const int ShufMask1[] = { 0, 1, 4, 5, 8, 9, 12, 13,
+ -1, -1, -1, -1, -1, -1, -1, -1,
+ 16, 17, 20, 21, 24, 25, 28, 29,
+ -1, -1, -1, -1, -1, -1, -1, -1 };
In = DAG.getBitcast(MVT::v32i8, In);
In = DAG.getVectorShuffle(MVT::v32i8, DL, In, In, ShufMask1);
In = DAG.getBitcast(MVT::v4i64, In);
@@ -22195,8 +22205,8 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
dl, {NVT, MVT::Other}, {Chain, Src});
Chain = Res.getValue(1);
} else {
- Res = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, NVT,
- Src);
+ Res = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl,
+ NVT, Src);
}
// TODO: Need to add exception check code for strict FP.
@@ -22298,8 +22308,8 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
DAG.getUNDEF(MVT::v2f32));
if (IsStrict) {
- unsigned Opc =
- IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
+ unsigned Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI
+ : X86ISD::STRICT_CVTTP2UI;
return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op->getOperand(0), Tmp});
}
unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
@@ -22424,7 +22434,7 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
if (IsStrict)
- return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
+ return DAG.getMergeValues({ Tmp.first, Tmp.second }, dl);
return Tmp.first;
}
@@ -22488,7 +22498,7 @@ SDValue X86TargetLowering::LRINT_LLRINTHelper(SDNode *N,
assert(DstVT == MVT::i64 && "Invalid LRINT/LLRINT to lower!");
Chain = DAG.getStore(Chain, DL, Src, StackPtr, MPI);
SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
- SDValue Ops[] = {Chain, StackPtr};
+ SDValue Ops[] = { Chain, StackPtr };
Src = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, SrcVT, MPI,
/*Align*/ std::nullopt,
@@ -22496,7 +22506,7 @@ SDValue X86TargetLowering::LRINT_LLRINTHelper(SDNode *N,
Chain = Src.getValue(1);
}
- SDValue StoreOps[] = {Chain, Src, StackPtr};
+ SDValue StoreOps[] = { Chain, Src, StackPtr };
Chain = DAG.getMemIntrinsicNode(X86ISD::FIST, DL, DAG.getVTList(MVT::Other),
StoreOps, DstVT, MPI, /*Align*/ std::nullopt,
MachineMemOperand::MOStore);
@@ -22504,8 +22514,8 @@ SDValue X86TargetLowering::LRINT_LLRINTHelper(SDNode *N,
return DAG.getLoad(DstVT, DL, Chain, StackPtr, MPI);
}
-SDValue X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue
+X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const {
// This is based on the TargetLowering::expandFP_TO_INT_SAT implementation,
// but making use of X86 specifics to produce better instruction sequences.
SDNode *Node = Op.getNode();
@@ -22575,12 +22585,12 @@ SDValue X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
APFloat MinFloat(Sem);
APFloat MaxFloat(Sem);
- APFloat::opStatus MinStatus =
- MinFloat.convertFromAPInt(MinInt, IsSigned, APFloat::rmTowardZero);
- APFloat::opStatus MaxStatus =
- MaxFloat.convertFromAPInt(MaxInt, IsSigned, APFloat::rmTowardZero);
- bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
- !(MaxStatus & APFloat::opStatus::opInexact);
+ APFloat::opStatus MinStatus = MinFloat.convertFromAPInt(
+ MinInt, IsSigned, APFloat::rmTowardZero);
+ APFloat::opStatus MaxStatus = MaxFloat.convertFromAPInt(
+ MaxInt, IsSigned, APFloat::rmTowardZero);
+ bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact)
+ && !(MaxStatus & APFloat::opStatus::opInexact);
SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);
@@ -22607,8 +22617,8 @@ SDValue X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
SDValue MinClamped =
DAG.getNode(ISD::PSEUDO_FMAX, dl, SrcVT, Src, MinFloatNode);
// Clamp by MaxFloat from above. NaN cannot occur.
- SDValue BothClamped =
- DAG.getNode(X86ISD::FMINC, dl, SrcVT, MinClamped, MaxFloatNode);
+ SDValue BothClamped = DAG.getNode(
+ X86ISD::FMINC, dl, SrcVT, MinClamped, MaxFloatNode);
// Convert clamped value to integer.
SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, DstVT, BothClamped);
@@ -22620,8 +22630,8 @@ SDValue X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
// Otherwise, select zero if Src is NaN.
SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
- return DAG.getSelectCC(dl, Src, Src, ZeroInt, FpToInt,
- ISD::CondCode::SETUO);
+ return DAG.getSelectCC(
+ dl, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
}
SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
@@ -22652,8 +22662,8 @@ SDValue X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
}
// If Src OGT MaxFloat, select MaxInt.
- Select = DAG.getSelectCC(dl, Src, MaxFloatNode, MaxIntNode, Select,
- ISD::CondCode::SETOGT);
+ Select = DAG.getSelectCC(
+ dl, Src, MaxFloatNode, MaxIntNode, Select, ISD::CondCode::SETOGT);
// In the unsigned case we are done, because we mapped NaN to MinInt, which
// is already zero. The promoted case was already handled above.
@@ -22663,7 +22673,8 @@ SDValue X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
// Otherwise, select 0 if Src is NaN.
SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
- return DAG.getSelectCC(dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);
+ return DAG.getSelectCC(
+ dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);
}
SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
@@ -22732,7 +22743,7 @@ SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
std::move(Args));
SDValue Res;
- std::tie(Res, Chain) = LowerCallTo(CLI);
+ std::tie(Res,Chain) = LowerCallTo(CLI);
if (IsStrict)
Res = DAG.getMergeValues({Res, Chain}, DL);
@@ -23006,14 +23017,14 @@ static SDValue lowerAddSubToHorizontalOp(SDValue Op, const SDLoc &DL,
// TODO: Allow commuted (f)sub by negating the result of (F)HSUB?
unsigned HOpcode;
switch (Op.getOpcode()) {
- // clang-format off
+ // clang-format off
case ISD::ADD: HOpcode = X86ISD::HADD; break;
case ISD::SUB: HOpcode = X86ISD::HSUB; break;
case ISD::FADD: HOpcode = X86ISD::FHADD; break;
case ISD::FSUB: HOpcode = X86ISD::FHSUB; break;
default:
llvm_unreachable("Trying to lower unsupported opcode to horizontal op");
- // clang-format on
+ // clang-format on
}
unsigned LExtIndex = LHS.getConstantOperandVal(1);
unsigned RExtIndex = RHS.getConstantOperandVal(1);
@@ -23071,7 +23082,7 @@ static SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) {
bool Ignored;
APFloat Point5Pred = APFloat(0.5f);
Point5Pred.convert(Sem, APFloat::rmNearestTiesToEven, &Ignored);
- Point5Pred.next(/*nextDown*/ true);
+ Point5Pred.next(/*nextDown*/true);
SDValue Adder = DAG.getNode(ISD::FCOPYSIGN, dl, VT,
DAG.getConstantFP(Point5Pred, dl, VT), N0);
@@ -23121,16 +23132,16 @@ static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
unsigned EltBits = VT.getScalarSizeInBits();
// For FABS, mask is 0x7f...; for FNEG, mask is 0x80...
- APInt MaskElt =
- IsFABS ? APInt::getSignedMaxValue(EltBits) : APInt::getSignMask(EltBits);
+ APInt MaskElt = IsFABS ? APInt::getSignedMaxValue(EltBits) :
+ APInt::getSignMask(EltBits);
const fltSemantics &Sem = VT.getFltSemantics();
SDValue Mask = DAG.getConstantFP(APFloat(Sem, MaskElt), dl, LogicVT);
SDValue Op0 = Op.getOperand(0);
bool IsFNABS = !IsFABS && (Op0.getOpcode() == ISD::FABS);
- unsigned LogicOp = IsFABS ? X86ISD::FAND
- : IsFNABS ? X86ISD::FOR
- : X86ISD::FXOR;
+ unsigned LogicOp = IsFABS ? X86ISD::FAND :
+ IsFNABS ? X86ISD::FOR :
+ X86ISD::FXOR;
SDValue Operand = IsFNABS ? Op0.getOperand(0) : Op0;
if (VT.isVector() || IsF128)
@@ -23233,8 +23244,7 @@ static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
}
/// Helper for attempting to create a X86ISD::BT node.
-static SDValue getBT(SDValue Src, SDValue BitNo, const SDLoc &DL,
- SelectionDAG &DAG) {
+static SDValue getBT(SDValue Src, SDValue BitNo, const SDLoc &DL, SelectionDAG &DAG) {
// If Src is i8, promote it to i32 with any_extend. There is no i8 BT
// instruction. Since the shift amount is in-range-or-undefined, we know
// that doing a bittest on the i32 value is ok. We extend to i32 because
@@ -23843,7 +23853,8 @@ static bool hasNonFlagsUse(SDValue Op) {
// the node alone and emit a 'cmp' or 'test' instruction.
static bool isProfitableToUseFlagOp(SDValue Op) {
for (SDNode *U : Op->users())
- if (U->getOpcode() != ISD::CopyToReg && U->getOpcode() != ISD::SETCC &&
+ if (U->getOpcode() != ISD::CopyToReg &&
+ U->getOpcode() != ISD::SETCC &&
U->getOpcode() != ISD::STORE)
return false;
@@ -23859,20 +23870,14 @@ static SDValue EmitTest(SDValue Op, X86::CondCode X86CC, const SDLoc &dl,
bool NeedCF = false;
bool NeedOF = false;
switch (X86CC) {
- default:
- break;
- case X86::COND_A:
- case X86::COND_AE:
- case X86::COND_B:
- case X86::COND_BE:
+ default: break;
+ case X86::COND_A: case X86::COND_AE:
+ case X86::COND_B: case X86::COND_BE:
NeedCF = true;
break;
- case X86::COND_G:
- case X86::COND_GE:
- case X86::COND_L:
- case X86::COND_LE:
- case X86::COND_O:
- case X86::COND_NO: {
+ case X86::COND_G: case X86::COND_GE:
+ case X86::COND_L: case X86::COND_LE:
+ case X86::COND_O: case X86::COND_NO: {
// Check if we really need to set the
// Overflow flag. If NoSignedWrap is present
// that is not actually needed.
@@ -23924,14 +23929,14 @@ static SDValue EmitTest(SDValue Op, X86::CondCode X86CC, const SDLoc &dl,
// Otherwise use a regular EFLAGS-setting instruction.
switch (ArithOp.getOpcode()) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("unexpected operator!");
case ISD::ADD: Opcode = X86ISD::ADD; break;
case ISD::SUB: Opcode = X86ISD::SUB; break;
case ISD::XOR: Opcode = X86ISD::XOR; break;
case ISD::AND: Opcode = X86ISD::AND; break;
case ISD::OR: Opcode = X86ISD::OR; break;
- // clang-format on
+ // clang-format on
}
NumOperands = 2;
@@ -23946,9 +23951,8 @@ static SDValue EmitTest(SDValue Op, X86::CondCode X86CC, const SDLoc &dl,
case ISD::USUBO: {
// /USUBO/SSUBO will become a X86ISD::SUB and we can use its Z flag.
SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
- return DAG
- .getNode(X86ISD::SUB, dl, VTs, Op->getOperand(0), Op->getOperand(1))
- .getValue(1);
+ return DAG.getNode(X86ISD::SUB, dl, VTs, Op->getOperand(0),
+ Op->getOperand(1)).getValue(1);
}
default:
break;
@@ -23977,9 +23981,8 @@ static SDValue EmitCmp(SDValue Op0, SDValue Op1, X86::CondCode X86CC,
EVT CmpVT = Op0.getValueType();
- assert((CmpVT == MVT::i8 || CmpVT == MVT::i16 || CmpVT == MVT::i32 ||
- CmpVT == MVT::i64) &&
- "Unexpected VT!");
+ assert((CmpVT == MVT::i8 || CmpVT == MVT::i16 ||
+ CmpVT == MVT::i32 || CmpVT == MVT::i64) && "Unexpected VT!");
// Only promote the compare up to I32 if it is a 16 bit operation
// with an immediate. 16 bit immediates are to be avoided unless the target
@@ -24106,8 +24109,9 @@ bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const {
/// The minimum architected relative accuracy is 2^-12. We need one
/// Newton-Raphson step to have a good float result (24 bits of precision).
-SDValue X86TargetLowering::getSqrtEstimate(SDValue Op, SelectionDAG &DAG,
- int Enabled, int &RefinementSteps,
+SDValue X86TargetLowering::getSqrtEstimate(SDValue Op,
+ SelectionDAG &DAG, int Enabled,
+ int &RefinementSteps,
bool &UseOneConstNR,
bool Reciprocal) const {
SDLoc DL(Op);
@@ -24214,7 +24218,9 @@ SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
/// This is because we still need one division to calculate the reciprocal and
/// then we need two multiplies by that reciprocal as replacements for the
/// original divisions.
-unsigned X86TargetLowering::combineRepeatedFPDivisors() const { return 2; }
+unsigned X86TargetLowering::combineRepeatedFPDivisors() const {
+ return 2;
+}
SDValue
X86TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
@@ -24222,7 +24228,7 @@ X86TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
SmallVectorImpl<SDNode *> &Created) const {
AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
if (isIntDivCheap(N->getValueType(0), Attr))
- return SDValue(N, 0); // Lower SDIV as SDIV
+ return SDValue(N,0); // Lower SDIV as SDIV
assert((Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()) &&
"Unexpected divisor!");
@@ -24332,7 +24338,7 @@ static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
// 6 - NLE
// 7 - ORD
switch (SetCCOpcode) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected SETCC condition");
case ISD::SETOEQ:
case ISD::SETEQ: SSECC = 0; break;
@@ -24354,7 +24360,7 @@ static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
case ISD::SETO: SSECC = 7; break;
case ISD::SETUEQ: SSECC = 8; break;
case ISD::SETONE: SSECC = 12; break;
- // clang-format on
+ // clang-format on
}
if (Swap)
std::swap(Op0, Op1);
@@ -24643,13 +24649,13 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
Cmp1 = DAG.getNode(
Opc, dl, {VT, MVT::Other},
{Chain, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8)});
- Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
- Cmp0.getValue(1), Cmp1.getValue(1));
+ Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Cmp0.getValue(1),
+ Cmp1.getValue(1));
} else {
- Cmp0 = DAG.getNode(Opc, dl, VT, Op0, Op1,
- DAG.getTargetConstant(CC0, dl, MVT::i8));
- Cmp1 = DAG.getNode(Opc, dl, VT, Op0, Op1,
- DAG.getTargetConstant(CC1, dl, MVT::i8));
+ Cmp0 = DAG.getNode(
+ Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8));
+ Cmp1 = DAG.getNode(
+ Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8));
}
Cmp = DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);
} else {
@@ -24659,8 +24665,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
{Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
Chain = Cmp.getValue(1);
} else
- Cmp = DAG.getNode(Opc, dl, VT, Op0, Op1,
- DAG.getTargetConstant(SSECC, dl, MVT::i8));
+ Cmp = DAG.getNode(
+ Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
}
} else {
// Handle all other FP comparisons here.
@@ -24672,8 +24678,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
{Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
Chain = Cmp.getValue(1);
} else
- Cmp = DAG.getNode(Opc, dl, VT, Op0, Op1,
- DAG.getTargetConstant(SSECC, dl, MVT::i8));
+ Cmp = DAG.getNode(
+ Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
}
if (VT.getFixedSizeInBits() >
@@ -24724,7 +24730,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
// Translate compare code to XOP PCOM compare mode.
unsigned CmpMode = 0;
switch (Cond) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected SETCC condition");
case ISD::SETULT:
case ISD::SETLT: CmpMode = 0x00; break;
@@ -24736,7 +24742,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
case ISD::SETGE: CmpMode = 0x03; break;
case ISD::SETEQ: CmpMode = 0x04; break;
case ISD::SETNE: CmpMode = 0x05; break;
- // clang-format on
+ // clang-format on
}
// Are we comparing unsigned or signed integers?
@@ -24834,13 +24840,13 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
bool Invert = false;
unsigned Opc;
switch (Cond) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected condition code");
case ISD::SETUGT: Invert = true; [[fallthrough]];
case ISD::SETULE: Opc = ISD::UMIN; break;
case ISD::SETULT: Invert = true; [[fallthrough]];
case ISD::SETUGE: Opc = ISD::UMAX; break;
- // clang-format on
+ // clang-format on
}
SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
@@ -24864,10 +24870,10 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
// operations may be required for some comparisons.
unsigned Opc = (Cond == ISD::SETEQ || Cond == ISD::SETNE) ? X86ISD::PCMPEQ
: X86ISD::PCMPGT;
- bool Swap = Cond == ISD::SETLT || Cond == ISD::SETULT || Cond == ISD::SETGE ||
- Cond == ISD::SETUGE;
- bool Invert =
- Cond == ISD::SETNE || (Cond != ISD::SETEQ && ISD::isTrueWhenEqual(Cond));
+ bool Swap = Cond == ISD::SETLT || Cond == ISD::SETULT ||
+ Cond == ISD::SETGE || Cond == ISD::SETUGE;
+ bool Invert = Cond == ISD::SETNE ||
+ (Cond != ISD::SETEQ && ISD::isTrueWhenEqual(Cond));
if (Swap)
std::swap(Op0, Op1);
@@ -24885,7 +24891,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
Op1 = DAG.getBitcast(MVT::v4i32, Op1);
SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
- static const int MaskHi[] = {1, 1, 3, 3};
+ static const int MaskHi[] = { 1, 1, 3, 3 };
SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
return DAG.getBitcast(VT, Result);
@@ -24896,7 +24902,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
Op1 = DAG.getAllOnesConstant(dl, MVT::v4i32);
SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
- static const int MaskHi[] = {1, 1, 3, 3};
+ static const int MaskHi[] = { 1, 1, 3, 3 };
SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
return DAG.getBitcast(VT, Result);
@@ -24935,8 +24941,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
// Create masks for only the low parts/high parts of the 64 bit integers.
- static const int MaskHi[] = {1, 1, 3, 3};
- static const int MaskLo[] = {0, 0, 2, 2};
+ static const int MaskHi[] = { 1, 1, 3, 3 };
+ static const int MaskLo[] = { 0, 0, 2, 2 };
SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);
SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
@@ -24963,7 +24969,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
// Make sure the lower and upper halves are both all-ones.
- static const int Mask[] = {1, 0, 3, 2};
+ static const int Mask[] = { 1, 0, 3, 2 };
SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);
Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf);
@@ -24978,8 +24984,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
// bits of the inputs before performing those operations.
if (FlipSigns) {
MVT EltVT = VT.getVectorElementType();
- SDValue SM =
- DAG.getConstant(APInt::getSignMask(EltVT.getSizeInBits()), dl, VT);
+ SDValue SM = DAG.getConstant(APInt::getSignMask(EltVT.getSizeInBits()), dl,
+ VT);
Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SM);
Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SM);
}
@@ -24996,7 +25002,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
// Try to select this as a KORTEST+SETCC or KTEST+SETCC if possible.
static SDValue EmitAVX512Test(SDValue Op0, SDValue Op1, ISD::CondCode CC,
const SDLoc &dl, SelectionDAG &DAG,
- const X86Subtarget &Subtarget, SDValue &X86CC) {
+ const X86Subtarget &Subtarget,
+ SDValue &X86CC) {
assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
// Must be a bitcast from vXi1.
@@ -25143,8 +25150,7 @@ SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
Op.getOpcode() == ISD::STRICT_FSETCCS;
MVT VT = Op->getSimpleValueType(0);
- if (VT.isVector())
- return LowerVSETCC(Op, Subtarget, DAG);
+ if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
assert(VT == MVT::i8 && "SetCC type must be 8-bit integer");
SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
@@ -25239,8 +25245,7 @@ SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
}
-SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const {
SDValue LHS = Op.getOperand(0);
SDValue RHS = Op.getOperand(1);
SDValue Carry = Op.getOperand(2);
@@ -25252,8 +25257,8 @@ SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op,
// Recreate the carry if needed.
EVT CarryVT = Carry.getValueType();
- Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32), Carry,
- DAG.getAllOnesConstant(DL, CarryVT));
+ Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
+ Carry, DAG.getAllOnesConstant(DL, CarryVT));
SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1));
@@ -25273,8 +25278,7 @@ getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) {
unsigned BaseOp = 0;
SDLoc DL(Op);
switch (Op.getOpcode()) {
- default:
- llvm_unreachable("Unknown ovf instruction!");
+ default: llvm_unreachable("Unknown ovf instruction!");
case ISD::SADDO:
BaseOp = X86ISD::ADD;
Cond = X86::COND_O;
@@ -25348,8 +25352,7 @@ static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG) {
SDValue VOp0 = V.getOperand(0);
unsigned InBits = VOp0.getValueSizeInBits();
unsigned Bits = V.getValueSizeInBits();
- return DAG.MaskedValueIsZero(VOp0,
- APInt::getHighBitsSet(InBits, InBits - Bits));
+ return DAG.MaskedValueIsZero(VOp0, APInt::getHighBitsSet(InBits,InBits-Bits));
}
// Lower various (select (icmp CmpVal, 0), LHS, RHS) custom patterns.
@@ -25684,7 +25687,7 @@ static SDValue emitConjunctionForCCMP(SDValue Val, X86::CondCode &OutCC,
SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
bool AddTest = true;
- SDValue Cond = Op.getOperand(0);
+ SDValue Cond = Op.getOperand(0);
SDValue Op1 = Op.getOperand(1);
SDValue Op2 = Op.getOperand(2);
SDLoc DL(Op);
@@ -25847,13 +25850,14 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
// If condition flag is set by a X86ISD::CMP, then use it as the condition
// setting operand in place of the X86ISD::SETCC.
unsigned CondOpcode = Cond.getOpcode();
- if (CondOpcode == X86ISD::SETCC || CondOpcode == X86ISD::SETCC_CARRY) {
+ if (CondOpcode == X86ISD::SETCC ||
+ CondOpcode == X86ISD::SETCC_CARRY) {
CC = Cond.getOperand(0);
SDValue Cmp = Cond.getOperand(1);
bool IllegalFPCMov = false;
- if (VT.isFloatingPoint() && !VT.isVector() && !isScalarFPTypeInSSEReg(VT) &&
- Subtarget.canUseCMOV()) // FPStack?
+ if (VT.isFloatingPoint() && !VT.isVector() &&
+ !isScalarFPTypeInSSEReg(VT) && Subtarget.canUseCMOV()) // FPStack?
IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue());
if ((isX86LogicalCmp(Cmp) && !IllegalFPCMov) ||
@@ -25916,15 +25920,14 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
// X86 doesn't have an i8 cmov. If both operands are the result of a truncate
// widen the cmov and push the truncate through. This avoids introducing a new
// branch during isel and doesn't add any extensions.
- if (Op.getValueType() == MVT::i8 && Op1.getOpcode() == ISD::TRUNCATE &&
- Op2.getOpcode() == ISD::TRUNCATE) {
+ if (Op.getValueType() == MVT::i8 &&
+ Op1.getOpcode() == ISD::TRUNCATE && Op2.getOpcode() == ISD::TRUNCATE) {
SDValue T1 = Op1.getOperand(0), T2 = Op2.getOperand(0);
if (T1.getValueType() == T2.getValueType() &&
// Exclude CopyFromReg to avoid partial register stalls.
- T1.getOpcode() != ISD::CopyFromReg &&
- T2.getOpcode() != ISD::CopyFromReg) {
- SDValue Cmov =
- DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1, CC, Cond);
+ T1.getOpcode() != ISD::CopyFromReg && T2.getOpcode()!=ISD::CopyFromReg){
+ SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1,
+ CC, Cond);
return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
}
}
@@ -25940,14 +25943,14 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
!X86::mayFoldLoad(Op2, Subtarget))) {
Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
- SDValue Ops[] = {Op2, Op1, CC, Cond};
+ SDValue Ops[] = { Op2, Op1, CC, Cond };
SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops);
return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
}
// X86ISD::CMOV means set the result (which is operand 1) to the RHS if
// condition is true.
- SDValue Ops[] = {Op2, Op1, CC, Cond};
+ SDValue Ops[] = { Op2, Op1, CC, Cond };
return DAG.getNode(X86ISD::CMOV, DL, Op.getValueType(), Ops, Op->getFlags());
}
@@ -26057,9 +26060,9 @@ static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op,
InVT = In.getSimpleValueType();
}
- // SSE41 targets can use the pmov[sz]x* instructions directly for 128-bit
- // results, so are legal and shouldn't occur here. AVX2/AVX512 pmovsx*
- // instructions still need to be handled here for 256/512-bit results.
+ // SSE41 targets can use the pmov[sz]x* instructions directly for 128-bit results,
+ // so are legal and shouldn't occur here. AVX2/AVX512 pmovsx* instructions still
+ // need to be handled here for 256/512-bit results.
if (Subtarget.hasInt256()) {
assert(VT.getSizeInBits() > 128 && "Unexpected 128-bit vector extension");
@@ -26068,8 +26071,9 @@ static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op,
// FIXME: Apparently we create inreg operations that could be regular
// extends.
- unsigned ExtOpc = Opc == ISD::SIGN_EXTEND_VECTOR_INREG ? ISD::SIGN_EXTEND
- : ISD::ZERO_EXTEND;
+ unsigned ExtOpc =
+ Opc == ISD::SIGN_EXTEND_VECTOR_INREG ? ISD::SIGN_EXTEND
+ : ISD::ZERO_EXTEND;
return DAG.getNode(ExtOpc, dl, VT, In);
}
@@ -26187,9 +26191,9 @@ static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
SDValue OpLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, In);
unsigned NumElems = InVT.getVectorNumElements();
- SmallVector<int, 8> ShufMask(NumElems, -1);
- for (unsigned i = 0; i != NumElems / 2; ++i)
- ShufMask[i] = i + NumElems / 2;
+ SmallVector<int,8> ShufMask(NumElems, -1);
+ for (unsigned i = 0; i != NumElems/2; ++i)
+ ShufMask[i] = i + NumElems/2;
SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
OpHi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, OpHi);
@@ -26351,10 +26355,11 @@ static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
// TODO: It is possible to support ZExt by zeroing the undef values during
// the shuffle phase or after the shuffle.
static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget,
- SelectionDAG &DAG) {
+ SelectionDAG &DAG) {
MVT RegVT = Op.getSimpleValueType();
assert(RegVT.isVector() && "We only custom lower vector loads.");
- assert(RegVT.isInteger() && "We only custom lower integer vector loads.");
+ assert(RegVT.isInteger() &&
+ "We only custom lower integer vector loads.");
LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
SDLoc dl(Ld);
@@ -26398,7 +26403,7 @@ static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
SDValue X86TargetLowering::LowerConditionalBranch(SDValue Op,
SelectionDAG &DAG) const {
SDValue Chain = Op.getOperand(0);
- SDValue Cond = Op.getOperand(1);
+ SDValue Cond = Op.getOperand(1);
SDValue Dest;
if (Op.getOpcode() == ISD::BRCOND)
Dest = Op.getOperand(2);
@@ -26458,7 +26463,7 @@ SDValue X86TargetLowering::LowerConditionalBranch(SDValue Op,
if (User->getOpcode() == ISD::BR) {
SDValue FalseBB = User->getOperand(1);
SDNode *NewBR =
- DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
+ DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
assert(NewBR == User);
(void)NewBR;
Dest = FalseBB;
@@ -26522,8 +26527,9 @@ SDValue X86TargetLowering::LowerConditionalBranch(SDValue Op,
// bytes in one go. Touching the stack at 4K increments is necessary to ensure
// that the guard pages used by the OS virtual memory manager are allocated in
// correct sequence.
-SDValue X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue
+X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
+ SelectionDAG &DAG) const {
MachineFunction &MF = DAG.getMachineFunction();
bool SplitStack = MF.shouldSplitStack();
bool EmitStackProbeCall = hasStackProbeSymbol(MF);
@@ -26534,7 +26540,7 @@ SDValue X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
// Get the inputs.
SDNode *Node = Op.getNode();
SDValue Chain = Op.getOperand(0);
- SDValue Size = Op.getOperand(1);
+ SDValue Size = Op.getOperand(1);
MaybeAlign Alignment(Op.getConstantOperandVal(2));
EVT VT = Node->getValueType(0);
@@ -26657,9 +26663,8 @@ SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
MemOps.push_back(Store);
// Store ptr to reg_save_area.
- FIN = DAG.getNode(
- ISD::ADD, DL, PtrVT, FIN,
- DAG.getIntPtrConstant(Subtarget.isTarget64BitLP64() ? 8 : 4, DL));
+ FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(
+ Subtarget.isTarget64BitLP64() ? 8 : 4, DL));
SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT);
Store = DAG.getStore(
Op.getOperand(0), DL, RSFIN, FIN,
@@ -26669,7 +26674,8 @@ SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
}
SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
- assert(Subtarget.is64Bit() && "LowerVAARG only handles 64-bit va_arg!");
+ assert(Subtarget.is64Bit() &&
+ "LowerVAARG only handles 64-bit va_arg!");
assert(Op.getNumOperands() == 4);
MachineFunction &MF = DAG.getMachineFunction();
@@ -26693,11 +26699,11 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
// selection mechanism works only for the basic types.
assert(ArgVT != MVT::f80 && "va_arg for f80 not yet implemented");
if (ArgVT.isFloatingPoint() && ArgSize <= 16 /*bytes*/) {
- ArgMode = 2; // Argument passed in XMM register. Use fp_offset.
+ ArgMode = 2; // Argument passed in XMM register. Use fp_offset.
} else {
assert(ArgVT.isInteger() && ArgSize <= 32 /*bytes*/ &&
"Unhandled argument type in LowerVAARG");
- ArgMode = 1; // Argument passed in GPR64 register(s). Use gp_offset.
+ ArgMode = 1; // Argument passed in GPR64 register(s). Use gp_offset.
}
if (ArgMode == 2) {
@@ -26731,7 +26737,7 @@ static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
// where a va_list is still an i8*.
assert(Subtarget.is64Bit() && "This code only handles 64-bit va_copy!");
if (Subtarget.isCallingConvWin64(
- DAG.getMachineFunction().getFunction().getCallingConv()))
+ DAG.getMachineFunction().getFunction().getCallingConv()))
// Probably a Win64 va_copy.
return DAG.expandVACopy(Op.getNode());
@@ -26824,12 +26830,9 @@ static int getSEHRegistrationNodeSize(const Function *Fn) {
// The RegNodeSize is 6 32-bit words for SEH and 4 for C++ EH. See
// WinEHStatePass for the full struct definition.
switch (classifyEHPersonality(Fn->getPersonalityFn())) {
- case EHPersonality::MSVC_X86SEH:
- return 24;
- case EHPersonality::MSVC_CXX:
- return 16;
- default:
- break;
+ case EHPersonality::MSVC_X86SEH: return 24;
+ case EHPersonality::MSVC_CXX: return 16;
+ default: break;
}
report_fatal_error(
"can only recover FP for 32-bit MSVC EH personality functions");
@@ -26919,13 +26922,13 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SDLoc dl(Op);
unsigned IntNo = Op.getConstantOperandVal(0);
MVT VT = Op.getSimpleValueType();
- const IntrinsicData *IntrData = getIntrinsicWithoutChain(IntNo);
+ const IntrinsicData* IntrData = getIntrinsicWithoutChain(IntNo);
// Propagate flags from original node to transformed node(s).
SelectionDAG::FlagInserter FlagsInserter(DAG, Op->getFlags());
if (IntrData) {
- switch (IntrData->Type) {
+ switch(IntrData->Type) {
case INTR_TYPE_1OP: {
// We specify 2 possible opcodes for intrinsics with rounding modes.
// First, we check if the intrinsic may have non-default rounding mode,
@@ -27051,8 +27054,9 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
if (!isRoundModeCurDirection(Rnd))
return SDValue();
}
- return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src),
- Mask, PassThru, Subtarget, DAG);
+ return getVectorMaskingNode(
+ DAG.getNode(IntrData->Opc0, dl, VT, Src), Mask, PassThru,
+ Subtarget, DAG);
}
case INTR_TYPE_1OP_MASK_SAE: {
SDValue Src = Op.getOperand(1);
@@ -27093,9 +27097,9 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
if (!isRoundModeCurDirection(Rnd))
return SDValue();
}
- return getScalarMaskingNode(
- DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2), Mask, passThru,
- Subtarget, DAG);
+ return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1,
+ Src2),
+ Mask, passThru, Subtarget, DAG);
}
assert(Op.getNumOperands() == (6U + HasRounding) &&
@@ -27109,9 +27113,9 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
else if (!isRoundModeCurDirection(Sae))
return SDValue();
}
- return getScalarMaskingNode(
- DAG.getNode(Opc, dl, VT, Src1, Src2, RoundingMode), Mask, passThru,
- Subtarget, DAG);
+ return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1,
+ Src2, RoundingMode),
+ Mask, passThru, Subtarget, DAG);
}
case INTR_TYPE_SCALAR_MASK_RND: {
SDValue Src1 = Op.getOperand(1);
@@ -27146,8 +27150,8 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
else
return SDValue();
- return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2), Mask,
- passThru, Subtarget, DAG);
+ return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
+ Mask, passThru, Subtarget, DAG);
}
case INTR_TYPE_2OP_MASK: {
SDValue Src1 = Op.getOperand(1);
@@ -27183,8 +27187,8 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
return SDValue();
}
- return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2), Mask,
- PassThru, Subtarget, DAG);
+ return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
+ Mask, PassThru, Subtarget, DAG);
}
case INTR_TYPE_3OP_SCALAR_MASK_SAE: {
SDValue Src1 = Op.getOperand(1);
@@ -27233,12 +27237,12 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
// Reverse the operands to match VSELECT order.
return DAG.getNode(IntrData->Opc0, dl, VT, Src3, Src2, Src1);
}
- case VPERM_2OP: {
+ case VPERM_2OP : {
SDValue Src1 = Op.getOperand(1);
SDValue Src2 = Op.getOperand(2);
// Swap Src1 and Src2 in the node creation
- return DAG.getNode(IntrData->Opc0, dl, VT, Src2, Src1);
+ return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1);
}
case CFMA_OP_MASKZ:
case CFMA_OP_MASK: {
@@ -27282,8 +27286,8 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SDValue Imm = Op.getOperand(2);
SDValue Mask = Op.getOperand(3);
SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm);
- SDValue FPclassMask =
- getScalarMaskingNode(FPclass, Mask, SDValue(), Subtarget, DAG);
+ SDValue FPclassMask = getScalarMaskingNode(FPclass, Mask, SDValue(),
+ Subtarget, DAG);
// Need to fill with zeros to ensure the bitcast will produce zeroes
// for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
@@ -27308,7 +27312,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
if (!isRoundModeCurDirection(Sae))
return SDValue();
}
- // default rounding mode
+ //default rounding mode
return DAG.getNode(IntrData->Opc0, dl, MaskVT,
{Op.getOperand(1), Op.getOperand(2), CC, Mask});
}
@@ -27327,12 +27331,12 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
else if (!isRoundModeCurDirection(Sae))
return SDValue();
}
- // default rounding mode
+ //default rounding mode
if (!Cmp.getNode())
Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Src2, CC);
- SDValue CmpMask =
- getScalarMaskingNode(Cmp, Mask, SDValue(), Subtarget, DAG);
+ SDValue CmpMask = getScalarMaskingNode(Cmp, Mask, SDValue(),
+ Subtarget, DAG);
// Need to fill with zeros to ensure the bitcast will produce zeroes
// for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
@@ -27501,8 +27505,8 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
assert(IntrData->Opc0 == X86ISD::BEXTRI && "Unexpected opcode");
uint64_t Imm = Op.getConstantOperandVal(2);
- SDValue Control =
- DAG.getTargetConstant(Imm & 0xffff, dl, Op.getValueType());
+ SDValue Control = DAG.getTargetConstant(Imm & 0xffff, dl,
+ Op.getValueType());
return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
Op.getOperand(1), Control);
}
@@ -27524,7 +27528,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
Op.getOperand(3), GenCF.getValue(1));
}
SDValue SetCC = getSETCC(X86::COND_B, Res.getValue(1), dl, DAG);
- SDValue Results[] = {SetCC, Res};
+ SDValue Results[] = { SetCC, Res };
return DAG.getMergeValues(Results, dl);
}
case CVTPD2PS_MASK:
@@ -27607,8 +27611,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
}
switch (IntNo) {
- default:
- return SDValue(); // Don't custom lower most intrinsics.
+ default: return SDValue(); // Don't custom lower most intrinsics.
// ptest and testp intrinsics. The intrinsic these come from are designed to
// return an integer value, not just an instruction so lower it to the ptest
@@ -27642,8 +27645,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
unsigned TestOpc = X86ISD::PTEST;
X86::CondCode X86CC;
switch (IntNo) {
- default:
- llvm_unreachable("Bad fallthrough in Intrinsic lowering.");
+ default: llvm_unreachable("Bad fallthrough in Intrinsic lowering.");
case Intrinsic::x86_avx512_ktestc_b:
case Intrinsic::x86_avx512_ktestc_w:
case Intrinsic::x86_avx512_ktestc_d:
@@ -27714,8 +27716,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
unsigned Opcode;
X86::CondCode X86CC;
switch (IntNo) {
- default:
- llvm_unreachable("Impossible intrinsic"); // Can't reach here.
+ default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
case Intrinsic::x86_sse42_pcmpistria128:
Opcode = X86ISD::PCMPISTR;
X86CC = X86::COND_A;
@@ -27885,8 +27886,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
unsigned NewIntrinsic;
switch (IntNo) {
- default:
- llvm_unreachable("Impossible intrinsic"); // Can't reach here.
+ default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
case Intrinsic::x86_mmx_pslli_w:
NewIntrinsic = Intrinsic::x86_mmx_psll_w;
break;
@@ -27963,16 +27963,16 @@ static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
- SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale};
+ SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
SDValue Res =
DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops,
MemIntr->getMemoryVT(), MemIntr->getMemOperand());
return DAG.getMergeValues({Res, Res.getValue(1)}, dl);
}
-static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG, SDValue Src,
- SDValue Mask, SDValue Base, SDValue Index,
- SDValue ScaleOp, SDValue Chain,
+static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
+ SDValue Src, SDValue Mask, SDValue Base,
+ SDValue Index, SDValue ScaleOp, SDValue Chain,
const X86Subtarget &Subtarget) {
MVT VT = Op.getSimpleValueType();
SDLoc dl(Op);
@@ -28001,7 +28001,7 @@ static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG, SDValue Src,
MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
- SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale};
+ SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
SDValue Res =
DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops,
MemIntr->getMemoryVT(), MemIntr->getMemOperand());
@@ -28009,9 +28009,9 @@ static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG, SDValue Src,
}
static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
- SDValue Src, SDValue Mask, SDValue Base,
- SDValue Index, SDValue ScaleOp, SDValue Chain,
- const X86Subtarget &Subtarget) {
+ SDValue Src, SDValue Mask, SDValue Base,
+ SDValue Index, SDValue ScaleOp, SDValue Chain,
+ const X86Subtarget &Subtarget) {
SDLoc dl(Op);
auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
// Scale must be constant.
@@ -28053,8 +28053,8 @@ static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
TLI.getPointerTy(DAG.getDataLayout()));
SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
SDValue Segment = DAG.getRegister(0, MVT::i32);
- MVT MaskVT = MVT::getVectorVT(
- MVT::i1, Index.getSimpleValueType().getVectorNumElements());
+ MVT MaskVT =
+ MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements());
SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
SDValue Ops[] = {VMask, Base, Scale, Index, Disp, Segment, Chain};
SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops);
@@ -28070,11 +28070,11 @@ static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
/// expanded intrinsics implicitly defines extra registers (i.e. not just
/// EDX:EAX).
static SDValue expandIntrinsicWChainHelper(SDNode *N, const SDLoc &DL,
- SelectionDAG &DAG,
- unsigned TargetOpcode,
- unsigned SrcReg,
- const X86Subtarget &Subtarget,
- SmallVectorImpl<SDValue> &Results) {
+ SelectionDAG &DAG,
+ unsigned TargetOpcode,
+ unsigned SrcReg,
+ const X86Subtarget &Subtarget,
+ SmallVectorImpl<SDValue> &Results) {
SDValue Chain = N->getOperand(0);
SDValue Glue;
@@ -28114,7 +28114,7 @@ static SDValue expandIntrinsicWChainHelper(SDNode *N, const SDLoc &DL,
}
// Use a buildpair to merge the two 32-bit values into a 64-bit one.
- SDValue Ops[] = {LO, HI};
+ SDValue Ops[] = { LO, HI };
SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
Results.push_back(Pair);
Results.push_back(Chain);
@@ -28131,9 +28131,9 @@ static void getReadTimeStampCounter(SDNode *N, const SDLoc &DL, unsigned Opcode,
// The processor's time-stamp counter (a 64-bit MSR) is stored into the
// EDX:EAX registers. EDX is loaded with the high-order 32 bits of the MSR
// and the EAX register is loaded with the low-order 32 bits.
- SDValue Glue =
- expandIntrinsicWChainHelper(N, DL, DAG, Opcode,
- /* NoRegister */ 0, Subtarget, Results);
+ SDValue Glue = expandIntrinsicWChainHelper(N, DL, DAG, Opcode,
+ /* NoRegister */0, Subtarget,
+ Results);
if (Opcode != X86::RDTSCP)
return;
@@ -28191,9 +28191,10 @@ static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG) {
}
/// Emit Truncating Store with signed or unsigned saturation.
-static SDValue EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &DL,
- SDValue Val, SDValue Ptr, EVT MemVT,
- MachineMemOperand *MMO, SelectionDAG &DAG) {
+static SDValue
+EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val,
+ SDValue Ptr, EVT MemVT, MachineMemOperand *MMO,
+ SelectionDAG &DAG) {
SDVTList VTs = DAG.getVTList(MVT::Other);
SDValue Ops[] = {Chain, Val, Ptr};
unsigned Opc = SignedSat ? X86ISD::VTRUNCSTORES : X86ISD::VTRUNCSTOREUS;
@@ -28202,12 +28203,11 @@ static SDValue EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &DL,
/// Emit Masked Truncating Store with signed or unsigned saturation.
static SDValue EmitMaskedTruncSStore(bool SignedSat, SDValue Chain,
- const SDLoc &DL, SDValue Val, SDValue Ptr,
- SDValue Mask, EVT MemVT,
- MachineMemOperand *MMO,
- SelectionDAG &DAG) {
+ const SDLoc &DL,
+ SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT,
+ MachineMemOperand *MMO, SelectionDAG &DAG) {
SDVTList VTs = DAG.getVTList(MVT::Other);
- SDValue Ops[] = {Chain, Val, Ptr, Mask};
+ SDValue Ops[] = { Chain, Val, Ptr, Mask };
unsigned Opc = SignedSat ? X86ISD::VMTRUNCSTORES : X86ISD::VMTRUNCSTOREUS;
return DAG.getMemIntrinsicNode(Opc, DL, VTs, Ops, MemVT, MMO);
}
@@ -28275,8 +28275,9 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
SDLoc dl(Op);
// Create a WRPKRU node, pass the input to the EAX parameter, and pass 0
// to the EDX and ECX parameters.
- return DAG.getNode(X86ISD::WRPKRU, dl, MVT::Other, Op.getOperand(0),
- Op.getOperand(2), DAG.getConstant(0, dl, MVT::i32),
+ return DAG.getNode(X86ISD::WRPKRU, dl, MVT::Other,
+ Op.getOperand(0), Op.getOperand(2),
+ DAG.getConstant(0, dl, MVT::i32),
DAG.getConstant(0, dl, MVT::i32));
}
case llvm::Intrinsic::asan_check_memaccess: {
@@ -28307,8 +28308,7 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
unsigned Opcode;
switch (IntNo) {
- default:
- llvm_unreachable("Impossible intrinsic");
+ default: llvm_unreachable("Impossible intrinsic");
case Intrinsic::x86_umwait:
Opcode = X86ISD::UMWAIT;
break;
@@ -28321,8 +28321,9 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
break;
}
- SDValue Operation = DAG.getNode(Opcode, dl, VTs, Chain, Op->getOperand(2),
- Op->getOperand(3), Op->getOperand(4));
+ SDValue Operation =
+ DAG.getNode(Opcode, dl, VTs, Chain, Op->getOperand(2),
+ Op->getOperand(3), Op->getOperand(4));
SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
Operation.getValue(1));
@@ -28334,8 +28335,7 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
unsigned Opcode;
switch (IntNo) {
- default:
- llvm_unreachable("Impossible intrinsic!");
+ default: llvm_unreachable("Impossible intrinsic!");
case Intrinsic::x86_enqcmd:
Opcode = X86ISD::ENQCMD;
break;
@@ -28359,8 +28359,7 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
unsigned Opcode;
switch (IntNo) {
- default:
- llvm_unreachable("Impossible intrinsic");
+ default: llvm_unreachable("Impossible intrinsic");
case Intrinsic::x86_aesenc128kl:
Opcode = X86ISD::AESENC128KL;
break;
@@ -28398,8 +28397,7 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
unsigned Opcode;
switch (IntNo) {
- default:
- llvm_unreachable("Impossible intrinsic");
+ default: llvm_unreachable("Impossible intrinsic");
case Intrinsic::x86_aesencwide128kl:
Opcode = X86ISD::AESENCWIDE128KL;
break;
@@ -28580,9 +28578,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
}
SDLoc dl(Op);
- switch (IntrData->Type) {
- default:
- llvm_unreachable("Unknown Intrinsic Type");
+ switch(IntrData->Type) {
+ default: llvm_unreachable("Unknown Intrinsic Type");
case RDSEED:
case RDRAND: {
// Emit the node with the right value type.
@@ -28603,32 +28600,32 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
}
case GATHER_AVX2: {
SDValue Chain = Op.getOperand(0);
- SDValue Src = Op.getOperand(2);
- SDValue Base = Op.getOperand(3);
+ SDValue Src = Op.getOperand(2);
+ SDValue Base = Op.getOperand(3);
SDValue Index = Op.getOperand(4);
- SDValue Mask = Op.getOperand(5);
+ SDValue Mask = Op.getOperand(5);
SDValue Scale = Op.getOperand(6);
return getAVX2GatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
Scale, Chain, Subtarget);
}
case GATHER: {
- // gather(v1, mask, index, base, scale);
+ //gather(v1, mask, index, base, scale);
SDValue Chain = Op.getOperand(0);
- SDValue Src = Op.getOperand(2);
- SDValue Base = Op.getOperand(3);
+ SDValue Src = Op.getOperand(2);
+ SDValue Base = Op.getOperand(3);
SDValue Index = Op.getOperand(4);
- SDValue Mask = Op.getOperand(5);
+ SDValue Mask = Op.getOperand(5);
SDValue Scale = Op.getOperand(6);
- return getGatherNode(Op, DAG, Src, Mask, Base, Index, Scale, Chain,
- Subtarget);
+ return getGatherNode(Op, DAG, Src, Mask, Base, Index, Scale,
+ Chain, Subtarget);
}
case SCATTER: {
- // scatter(base, mask, index, v1, scale);
+ //scatter(base, mask, index, v1, scale);
SDValue Chain = Op.getOperand(0);
- SDValue Base = Op.getOperand(2);
- SDValue Mask = Op.getOperand(3);
+ SDValue Base = Op.getOperand(2);
+ SDValue Mask = Op.getOperand(3);
SDValue Index = Op.getOperand(4);
- SDValue Src = Op.getOperand(5);
+ SDValue Src = Op.getOperand(5);
SDValue Scale = Op.getOperand(6);
return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
Scale, Chain, Subtarget);
@@ -28639,9 +28636,9 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
"Wrong prefetch hint in intrinsic: should be 2 or 3");
unsigned Opcode = (HintVal == 2 ? IntrData->Opc1 : IntrData->Opc0);
SDValue Chain = Op.getOperand(0);
- SDValue Mask = Op.getOperand(2);
+ SDValue Mask = Op.getOperand(2);
SDValue Index = Op.getOperand(3);
- SDValue Base = Op.getOperand(4);
+ SDValue Base = Op.getOperand(4);
SDValue Scale = Op.getOperand(5);
return getPrefetchNode(Opcode, Op, DAG, Mask, Base, Index, Scale, Chain,
Subtarget);
@@ -28676,8 +28673,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
SDValue SetCC = getSETCC(X86::COND_NE, InTrans, dl, DAG);
SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
- return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Ret,
- SDValue(InTrans.getNode(), 1));
+ return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
+ Ret, SDValue(InTrans.getNode(), 1));
}
case TRUNCATE_TO_MEM_VI8:
case TRUNCATE_TO_MEM_VI16:
@@ -28690,7 +28687,7 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
MemIntrinsicSDNode *MemIntr = dyn_cast<MemIntrinsicSDNode>(Op);
assert(MemIntr && "Expected MemIntrinsicSDNode!");
- EVT MemVT = MemIntr->getMemoryVT();
+ EVT MemVT = MemIntr->getMemoryVT();
uint16_t TruncationOp = IntrData->Opc0;
switch (TruncationOp) {
@@ -28785,7 +28782,7 @@ SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
Register FrameReg =
RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction());
- SDLoc dl(Op); // FIXME probably not meaningful
+ SDLoc dl(Op); // FIXME probably not meaningful
unsigned Depth = Op.getConstantOperandVal(0);
assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
(FrameReg == X86::EBP && VT == MVT::i32)) &&
@@ -28802,7 +28799,7 @@ SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
// FIXME? Maybe this could be a TableGen attribute on some registers and
// this table could be generated automatically from RegInfo.
-Register X86TargetLowering::getRegisterByName(const char *RegName, LLT VT,
+Register X86TargetLowering::getRegisterByName(const char* RegName, LLT VT,
const MachineFunction &MF) const {
const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
@@ -28866,10 +28863,10 @@ bool X86TargetLowering::needsFixedCatchObjects() const {
}
SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
- SDValue Chain = Op.getOperand(0);
- SDValue Offset = Op.getOperand(1);
- SDValue Handler = Op.getOperand(2);
- SDLoc dl(Op);
+ SDValue Chain = Op.getOperand(0);
+ SDValue Offset = Op.getOperand(1);
+ SDValue Handler = Op.getOperand(2);
+ SDLoc dl (Op);
EVT PtrVT = getPointerTy(DAG.getDataLayout());
const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
@@ -28880,9 +28877,9 @@ SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT);
Register StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX;
- SDValue StoreAddr =
- DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
- DAG.getIntPtrConstant(RegInfo->getSlotSize(), dl));
+ SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
+ DAG.getIntPtrConstant(RegInfo->getSlotSize(),
+ dl));
StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StoreAddr, Offset);
Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());
Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr);
@@ -28905,20 +28902,19 @@ SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
(void)TII->getGlobalBaseReg(&DAG.getMachineFunction());
}
return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
- DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0),
- Op.getOperand(1));
+ DAG.getVTList(MVT::i32, MVT::Other),
+ Op.getOperand(0), Op.getOperand(1));
}
SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
- return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other, Op.getOperand(0),
- Op.getOperand(1));
+ return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
+ Op.getOperand(0), Op.getOperand(1));
}
-SDValue
-X86TargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue X86TargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
+ SelectionDAG &DAG) const {
SDLoc DL(Op);
return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other,
Op.getOperand(0));
@@ -28934,7 +28930,7 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
SDValue Trmp = Op.getOperand(1); // trampoline
SDValue FPtr = Op.getOperand(2); // nested function
SDValue Nest = Op.getOperand(3); // 'nest' parameter value
- SDLoc dl(Op);
+ SDLoc dl (Op);
const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
@@ -28943,7 +28939,7 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
SDValue OutChains[6];
// Large code-model.
- const unsigned char JMP64r = 0xFF; // 64-bit jmp through register opcode.
+ const unsigned char JMP64r = 0xFF; // 64-bit jmp through register opcode.
const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode.
const unsigned char N86R10 = TRI->getEncodingValue(X86::R10) & 0x7;
@@ -28991,7 +28987,7 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
} else {
const Function *Func =
- cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
+ cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
CallingConv::ID CC = Func->getCallingConv();
unsigned NestReg;
@@ -29013,8 +29009,7 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
unsigned Idx = 0;
for (FunctionType::param_iterator I = FTy->param_begin(),
- E = FTy->param_end();
- I != E; ++I, ++Idx)
+ E = FTy->param_end(); I != E; ++I, ++Idx)
if (Attrs.hasParamAttr(Idx, Attribute::InReg)) {
const DataLayout &DL = DAG.getDataLayout();
// FIXME: should only count parameters that are lowered to integers.
@@ -29120,16 +29115,18 @@ SDValue X86TargetLowering::LowerGET_ROUNDING(SDValue Op,
Chain = CWD.getValue(1);
// Mask and turn the control bits into a shift for the lookup table.
- SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i16,
- DAG.getNode(ISD::AND, DL, MVT::i16, CWD,
- DAG.getConstant(0xc00, DL, MVT::i16)),
- DAG.getConstant(9, DL, MVT::i8));
+ SDValue Shift =
+ DAG.getNode(ISD::SRL, DL, MVT::i16,
+ DAG.getNode(ISD::AND, DL, MVT::i16,
+ CWD, DAG.getConstant(0xc00, DL, MVT::i16)),
+ DAG.getConstant(9, DL, MVT::i8));
Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Shift);
SDValue LUT = DAG.getConstant(0x2d, DL, MVT::i32);
- SDValue RetVal = DAG.getNode(ISD::AND, DL, MVT::i32,
- DAG.getNode(ISD::SRL, DL, MVT::i32, LUT, Shift),
- DAG.getConstant(3, DL, MVT::i32));
+ SDValue RetVal =
+ DAG.getNode(ISD::AND, DL, MVT::i32,
+ DAG.getNode(ISD::SRL, DL, MVT::i32, LUT, Shift),
+ DAG.getConstant(3, DL, MVT::i32));
RetVal = DAG.getZExtOrTrunc(RetVal, DL, VT);
@@ -29437,15 +29434,17 @@ static SDValue LowerVectorCTLZ_AVX512CDI(SDValue Op, SelectionDAG &DAG,
MVT EltVT = VT.getVectorElementType();
unsigned NumElems = VT.getVectorNumElements();
- assert((EltVT == MVT::i8 || EltVT == MVT::i16) && "Unsupported element type");
+ assert((EltVT == MVT::i8 || EltVT == MVT::i16) &&
+ "Unsupported element type");
// Split vector, it's Lo and Hi parts will be handled in next iteration.
- if (NumElems > 16 || (NumElems == 16 && !Subtarget.canExtendTo512DQ()))
+ if (NumElems > 16 ||
+ (NumElems == 16 && !Subtarget.canExtendTo512DQ()))
return splitVectorIntUnary(Op, DAG, dl);
MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);
assert((NewVT.is256BitVector() || NewVT.is512BitVector()) &&
- "Unsupported value type for operation");
+ "Unsupported value type for operation");
// Use native supported vector instruction vplzcntd.
Op = DAG.getNode(ISD::ZERO_EXTEND, dl, NewVT, Op.getOperand(0));
@@ -30401,7 +30400,7 @@ static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
// Merge the two vectors back together with a shuffle. This expands into 2
// shuffles.
- static const int ShufMask[] = {0, 4, 2, 6};
+ static const int ShufMask[] = { 0, 4, 2, 6 };
return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask);
}
@@ -30538,7 +30537,7 @@ static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
//
// Place the odd value at an even position (basically, shift all values 1
// step to the left):
- const int Mask[] = {1, -1, 3, -1, 5, -1, 7, -1,
+ const int Mask[] = {1, -1, 3, -1, 5, -1, 7, -1,
9, -1, 11, -1, 13, -1, 15, -1};
// <a|b|c|d> => <b|undef|d|undef>
SDValue Odd0 =
@@ -30588,7 +30587,7 @@ static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
// Only i8 vectors should need custom lowering after this.
assert((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) ||
- (VT == MVT::v64i8 && Subtarget.hasBWI())) &&
+ (VT == MVT::v64i8 && Subtarget.hasBWI())) &&
"Unsupported vector type");
// Lower v16i8/v32i8 as extension to v8i16/v16i16 vector pairs, multiply,
@@ -30743,8 +30742,7 @@ static SDValue LowerMULO(SDValue Op, const X86Subtarget &Subtarget,
return DAG.getMergeValues({Low, Ovf}, dl);
}
-SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
assert(Subtarget.isTargetWin64() && "Unexpected target");
EVT VT = Op.getValueType();
assert(VT.isInteger() && VT.getSizeInBits() == 128 &&
@@ -30759,13 +30757,13 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op,
RTLIB::Libcall LC;
bool isSigned;
switch (Op->getOpcode()) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected request for libcall!");
case ISD::SDIV: isSigned = true; LC = RTLIB::SDIV_I128; break;
case ISD::UDIV: isSigned = false; LC = RTLIB::UDIV_I128; break;
case ISD::SREM: isSigned = true; LC = RTLIB::SREM_I128; break;
case ISD::UREM: isSigned = false; LC = RTLIB::UREM_I128; break;
- // clang-format on
+ // clang-format on
}
SDLoc dl(Op);
@@ -30907,9 +30905,9 @@ static bool supportedVectorShiftWithImm(EVT VT, const X86Subtarget &Subtarget,
// The shift amount is a variable, but it is the same for all vector lanes.
// These instructions are defined together with shift-immediate.
-static bool supportedVectorShiftWithBaseAmnt(EVT VT,
- const X86Subtarget &Subtarget,
- unsigned Opcode) {
+static
+bool supportedVectorShiftWithBaseAmnt(EVT VT, const X86Subtarget &Subtarget,
+ unsigned Opcode) {
return supportedVectorShiftWithImm(VT, Subtarget, Opcode);
}
@@ -30938,7 +30936,7 @@ static bool supportedVectorVarShift(EVT VT, const X86Subtarget &Subtarget,
return true;
bool LShift = VT.is128BitVector() || VT.is256BitVector();
- bool AShift = LShift && VT != MVT::v2i64 && VT != MVT::v4i64;
+ bool AShift = LShift && VT != MVT::v2i64 && VT != MVT::v4i64;
return (Opcode == ISD::SRA) ? AShift : LShift;
}
@@ -33064,8 +33062,7 @@ X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
auto SSID = AI->getSyncScopeID();
// We must restrict the ordering to avoid generating loads with Release or
// ReleaseAcquire orderings.
- auto Order =
- AtomicCmpXchgInst::getStrongestFailureOrdering(AI->getOrdering());
+ auto Order = AtomicCmpXchgInst::getStrongestFailureOrdering(AI->getOrdering());
// Before the load we need a fence. Here is an example lifted from
// http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf showing why a fence
@@ -33134,28 +33131,31 @@ static SDValue emitLockedStackOp(SelectionDAG &DAG,
if (Subtarget.is64Bit()) {
SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
- SDValue Ops[] = {DAG.getRegister(X86::RSP, MVT::i64), // Base
- DAG.getTargetConstant(1, DL, MVT::i8), // Scale
- DAG.getRegister(0, MVT::i64), // Index
- DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
- DAG.getRegister(0, MVT::i16), // Segment.
- Zero,
- Chain};
- SDNode *Res =
- DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32, MVT::Other, Ops);
+ SDValue Ops[] = {
+ DAG.getRegister(X86::RSP, MVT::i64), // Base
+ DAG.getTargetConstant(1, DL, MVT::i8), // Scale
+ DAG.getRegister(0, MVT::i64), // Index
+ DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
+ DAG.getRegister(0, MVT::i16), // Segment.
+ Zero,
+ Chain};
+ SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
+ MVT::Other, Ops);
return SDValue(Res, 1);
}
SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
- SDValue Ops[] = {DAG.getRegister(X86::ESP, MVT::i32), // Base
- DAG.getTargetConstant(1, DL, MVT::i8), // Scale
- DAG.getRegister(0, MVT::i32), // Index
- DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
- DAG.getRegister(0, MVT::i16), // Segment.
- Zero,
- Chain};
- SDNode *Res =
- DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32, MVT::Other, Ops);
+ SDValue Ops[] = {
+ DAG.getRegister(X86::ESP, MVT::i32), // Base
+ DAG.getTargetConstant(1, DL, MVT::i8), // Scale
+ DAG.getRegister(0, MVT::i32), // Index
+ DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
+ DAG.getRegister(0, MVT::i16), // Segment.
+ Zero,
+ Chain
+ };
+ SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
+ MVT::Other, Ops);
return SDValue(Res, 1);
}
@@ -33188,44 +33188,36 @@ static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget &Subtarget,
SDLoc DL(Op);
unsigned Reg = 0;
unsigned size = 0;
- switch (T.SimpleTy) {
- default:
- llvm_unreachable("Invalid value type!");
- case MVT::i8:
- Reg = X86::AL;
- size = 1;
- break;
- case MVT::i16:
- Reg = X86::AX;
- size = 2;
- break;
- case MVT::i32:
- Reg = X86::EAX;
- size = 4;
- break;
+ switch(T.SimpleTy) {
+ default: llvm_unreachable("Invalid value type!");
+ case MVT::i8: Reg = X86::AL; size = 1; break;
+ case MVT::i16: Reg = X86::AX; size = 2; break;
+ case MVT::i32: Reg = X86::EAX; size = 4; break;
case MVT::i64:
assert(Subtarget.is64Bit() && "Node not type legal!");
- Reg = X86::RAX;
- size = 8;
+ Reg = X86::RAX; size = 8;
break;
}
- SDValue cpIn =
- DAG.getCopyToReg(Op.getOperand(0), DL, Reg, Op.getOperand(2), SDValue());
- SDValue Ops[] = {cpIn.getValue(0), Op.getOperand(1), Op.getOperand(3),
- DAG.getTargetConstant(size, DL, MVT::i8), cpIn.getValue(1)};
+ SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
+ Op.getOperand(2), SDValue());
+ SDValue Ops[] = { cpIn.getValue(0),
+ Op.getOperand(1),
+ Op.getOperand(3),
+ DAG.getTargetConstant(size, DL, MVT::i8),
+ cpIn.getValue(1) };
SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
- SDValue Result =
- DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys, Ops, T, MMO);
+ SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
+ Ops, T, MMO);
SDValue cpOut =
- DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
+ DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
MVT::i32, cpOut.getValue(2));
SDValue Success = getSETCC(X86::COND_E, EFLAGS, DL, DAG);
- return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), cpOut, Success,
- EFLAGS.getValue(1));
+ return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
+ cpOut, Success, EFLAGS.getValue(1));
}
// Create X86ISD::MOVMSK, taking into account whether we need to split.
@@ -33293,8 +33285,7 @@ static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget,
}
assert((SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8 ||
- SrcVT == MVT::i64) &&
- "Unexpected VT!");
+ SrcVT == MVT::i64) && "Unexpected VT!");
assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
if (!(DstVT == MVT::f64 && SrcVT == MVT::i64) &&
@@ -33308,7 +33299,8 @@ static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget,
// Example: from MVT::v2i32 to MVT::v4i32.
MVT NewVT = MVT::getVectorVT(SrcVT.getVectorElementType(),
SrcVT.getVectorNumElements() * 2);
- Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT, Src, DAG.getUNDEF(SrcVT));
+ Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT, Src,
+ DAG.getUNDEF(SrcVT));
} else {
assert(SrcVT == MVT::i64 && !Subtarget.is64Bit() &&
"Unexpected source type in LowerBITCAST");
@@ -33454,8 +33446,7 @@ static SDValue LowerVectorCTPOP(SDValue Op, const SDLoc &DL,
if (Subtarget.hasVPOPCNTDQ()) {
unsigned NumElems = VT.getVectorNumElements();
assert((VT.getVectorElementType() == MVT::i8 ||
- VT.getVectorElementType() == MVT::i16) &&
- "Unexpected type");
+ VT.getVectorElementType() == MVT::i16) && "Unexpected type");
if (NumElems < 16 || (NumElems == 16 && Subtarget.canExtendTo512DQ())) {
MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);
Op = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, Op0);
@@ -33907,16 +33898,16 @@ static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG,
SDValue NewChain = DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Chain);
assert(!N->hasAnyUseOfValue(0));
// NOTE: The getUNDEF is needed to give something for the unused result 0.
- return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), DAG.getUNDEF(VT),
- NewChain);
+ return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
+ DAG.getUNDEF(VT), NewChain);
}
SDValue LockOp = lowerAtomicArithWithLOCK(N, DAG, Subtarget);
// RAUW the chain, but don't worry about the result, as it's unused.
assert(!N->hasAnyUseOfValue(0));
// NOTE: The getUNDEF is needed to give something for the unused result 0.
- return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), DAG.getUNDEF(VT),
- LockOp.getValue(1));
+ return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
+ DAG.getUNDEF(VT), LockOp.getValue(1));
}
static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
@@ -34023,17 +34014,17 @@ static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) {
// Set the carry flag.
SDValue Carry = Op.getOperand(2);
EVT CarryVT = Carry.getValueType();
- Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32), Carry,
- DAG.getAllOnesConstant(DL, CarryVT));
+ Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
+ Carry, DAG.getAllOnesConstant(DL, CarryVT));
bool IsAdd = Opc == ISD::UADDO_CARRY || Opc == ISD::SADDO_CARRY;
- SDValue Sum =
- DAG.getNode(IsAdd ? X86ISD::ADC : X86ISD::SBB, DL, VTs, Op.getOperand(0),
- Op.getOperand(1), Carry.getValue(1));
+ SDValue Sum = DAG.getNode(IsAdd ? X86ISD::ADC : X86ISD::SBB, DL, VTs,
+ Op.getOperand(0), Op.getOperand(1),
+ Carry.getValue(1));
bool IsSigned = Opc == ISD::SADDO_CARRY || Opc == ISD::SSUBO_CARRY;
- SDValue SetCC =
- getSETCC(IsSigned ? X86::COND_O : X86::COND_B, Sum.getValue(1), DL, DAG);
+ SDValue SetCC = getSETCC(IsSigned ? X86::COND_O : X86::COND_B,
+ Sum.getValue(1), DL, DAG);
if (N->getValueType(1) == MVT::i1)
SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
@@ -34129,8 +34120,8 @@ static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget,
if (!Subtarget.hasVLX() && !VT.is512BitVector() &&
!Index.getSimpleValueType().is512BitVector()) {
// Determine how much we need to widen by to get a 512-bit type.
- unsigned Factor =
- std::min(512 / VT.getSizeInBits(), 512 / IndexVT.getSizeInBits());
+ unsigned Factor = std::min(512/VT.getSizeInBits(),
+ 512/IndexVT.getSizeInBits());
unsigned NumElts = VT.getVectorNumElements() * Factor;
VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
@@ -34172,7 +34163,7 @@ static SDValue LowerMLOAD(SDValue Op, const X86Subtarget &Subtarget,
N->isExpandingLoad());
// Emit a blend.
SDValue Select = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
- return DAG.getMergeValues({Select, NewLoad.getValue(1)}, dl);
+ return DAG.getMergeValues({ Select, NewLoad.getValue(1) }, dl);
}
assert((!N->isExpandingLoad() || Subtarget.hasAVX512()) &&
@@ -34239,7 +34230,7 @@ static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget,
// This operation is legal for targets with VLX, but without
// VLX the vector should be widened to 512 bit
- unsigned NumEltsInWideVec = 512 / VT.getScalarSizeInBits();
+ unsigned NumEltsInWideVec = 512/VT.getScalarSizeInBits();
MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec);
// Mask element has to be i1.
@@ -34281,8 +34272,8 @@ static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget,
if (Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&
!IndexVT.is512BitVector()) {
// Determine how much we need to widen by to get a 512-bit type.
- unsigned Factor =
- std::min(512 / VT.getSizeInBits(), 512 / IndexVT.getSizeInBits());
+ unsigned Factor = std::min(512/VT.getSizeInBits(),
+ 512/IndexVT.getSizeInBits());
unsigned NumElts = VT.getVectorNumElements() * Factor;
@@ -34299,8 +34290,8 @@ static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget,
if (PassThru.isUndef())
PassThru = getZeroVector(VT, Subtarget, DAG, dl);
- SDValue Ops[] = {N->getChain(), PassThru, Mask,
- N->getBasePtr(), Index, N->getScale()};
+ SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
+ N->getScale() };
SDValue NewGather = DAG.getMemIntrinsicNode(
X86ISD::MGATHER, dl, DAG.getVTList(VT, MVT::Other), Ops, N->getMemoryVT(),
N->getMemOperand());
@@ -34480,7 +34471,7 @@ SDValue X86TargetLowering::visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL,
/// Provide custom lowering hooks for some operations.
SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
switch (Op.getOpcode()) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Should not custom lower this!");
case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, Subtarget, DAG);
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
@@ -34655,7 +34646,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
/// Replace a node with an illegal result type with a new node built out of
/// custom code.
void X86TargetLowering::ReplaceNodeResults(SDNode *N,
- SmallVectorImpl<SDValue> &Results,
+ SmallVectorImpl<SDValue>&Results,
SelectionDAG &DAG) const {
SDLoc dl(N);
unsigned Opc = N->getOpcode();
@@ -35191,8 +35182,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
SDValue Ovf = DAG.getSetCC(dl, N->getValueType(1), Hi, HiCmp, ISD::SETNE);
// Widen the result with by padding with undef.
- Res =
- DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res, DAG.getUNDEF(VT));
+ Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res,
+ DAG.getUNDEF(VT));
Results.push_back(Res);
Results.push_back(Ovf);
return;
@@ -35209,10 +35200,11 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
"Unexpected type action!");
unsigned NumConcat = 128 / InVT.getSizeInBits();
- EVT InWideVT =
- EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(),
- NumConcat * InVT.getVectorNumElements());
- EVT WideVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
+ EVT InWideVT = EVT::getVectorVT(*DAG.getContext(),
+ InVT.getVectorElementType(),
+ NumConcat * InVT.getVectorNumElements());
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
+ VT.getVectorElementType(),
NumConcat * VT.getVectorNumElements());
SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
@@ -35274,7 +35266,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
case ISD::UDIV:
case ISD::SREM:
case ISD::UREM: {
- SDValue V = LowerWin64_i128OP(SDValue(N, 0), DAG);
+ SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG);
Results.push_back(V);
return;
}
@@ -35352,9 +35344,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
Lo = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Lo);
Hi = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Hi);
- SDValue Res = DAG.getVectorShuffle(
- MVT::v16i8, dl, Lo, Hi,
- {0, 1, 2, 3, 16, 17, 18, 19, -1, -1, -1, -1, -1, -1, -1, -1});
+ SDValue Res = DAG.getVectorShuffle(MVT::v16i8, dl, Lo, Hi,
+ { 0, 1, 2, 3, 16, 17, 18, 19,
+ -1, -1, -1, -1, -1, -1, -1, -1 });
Results.push_back(Res);
return;
}
@@ -35386,7 +35378,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
SDValue In = N->getOperand(0);
EVT InVT = In.getValueType();
if (!Subtarget.hasSSE41() && VT == MVT::v4i64 &&
- (InVT == MVT::v4i16 || InVT == MVT::v4i8)) {
+ (InVT == MVT::v4i16 || InVT == MVT::v4i8)){
assert(getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector &&
"Unexpected type action!");
assert(Opc == ISD::SIGN_EXTEND && "Unexpected opcode");
@@ -35402,11 +35394,11 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
// Create an unpackl and unpackh to interleave the sign bits then bitcast
// to v2i64.
- SDValue Lo =
- DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits, {0, 4, 1, 5});
+ SDValue Lo = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
+ {0, 4, 1, 5});
Lo = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Lo);
- SDValue Hi =
- DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits, {2, 6, 3, 7});
+ SDValue Hi = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
+ {2, 6, 3, 7});
Hi = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Hi);
SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
@@ -35625,6 +35617,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
return;
}
+
if (VT == MVT::v2i32) {
assert((!IsStrict || IsSigned || Subtarget.hasAVX512()) &&
"Strict unsigned conversion requires AVX512");
@@ -35709,9 +35702,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
}
SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
- SDValue Res =
- DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT,
- DAG.getConstantFP(0.0, dl, VecInVT), Src, ZeroIdx);
+ SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT,
+ DAG.getConstantFP(0.0, dl, VecInVT), Src,
+ ZeroIdx);
SDValue Chain;
if (IsStrict) {
SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
@@ -35798,8 +35791,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
EVT SrcVT = Src.getValueType();
if (Subtarget.hasDQI() && Subtarget.hasVLX() && SrcVT == MVT::v2i64) {
if (IsStrict) {
- unsigned Opc =
- IsSigned ? X86ISD::STRICT_CVTSI2P : X86ISD::STRICT_CVTUI2P;
+ unsigned Opc = IsSigned ? X86ISD::STRICT_CVTSI2P
+ : X86ISD::STRICT_CVTUI2P;
SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},
{N->getOperand(0), Src});
Results.push_back(Res);
@@ -35813,7 +35806,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
if (SrcVT == MVT::v2i64 && !IsSigned && Subtarget.is64Bit() &&
Subtarget.hasSSE41() && !Subtarget.hasAVX512()) {
SDValue Zero = DAG.getConstant(0, dl, SrcVT);
- SDValue One = DAG.getConstant(1, dl, SrcVT);
+ SDValue One = DAG.getConstant(1, dl, SrcVT);
SDValue Sign = DAG.getNode(ISD::OR, dl, SrcVT,
DAG.getNode(ISD::SRL, dl, SrcVT, Src, One),
DAG.getNode(ISD::AND, dl, SrcVT, Src, One));
@@ -35879,9 +35872,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
if (IsStrict) {
SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other},
{N->getOperand(0), Or, VBias});
- SDValue Res =
- DAG.getNode(X86ISD::STRICT_VFPROUND, dl, {MVT::v4f32, MVT::Other},
- {Sub.getValue(1), Sub});
+ SDValue Res = DAG.getNode(X86ISD::STRICT_VFPROUND, dl,
+ {MVT::v4f32, MVT::Other},
+ {Sub.getValue(1), Sub});
Results.push_back(Res);
Results.push_back(Res.getValue(1));
} else {
@@ -35966,9 +35959,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
case ISD::INTRINSIC_W_CHAIN: {
unsigned IntNo = N->getConstantOperandVal(1);
switch (IntNo) {
- default:
- llvm_unreachable("Do not know how to custom type "
- "legalize this intrinsic operation!");
+ default : llvm_unreachable("Do not know how to custom type "
+ "legalize this intrinsic operation!");
case Intrinsic::x86_rdtsc:
return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget,
Results);
@@ -35981,7 +35973,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
return;
case Intrinsic::x86_rdpru:
expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPRU, X86::ECX, Subtarget,
- Results);
+ Results);
return;
case Intrinsic::x86_xgetbv:
expandIntrinsicWChainHelper(N, dl, DAG, X86::XGETBV, X86::ECX, Subtarget,
@@ -36038,12 +36030,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
}
SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
- Regs64bit ? X86::RAX : X86::EAX, HalfT,
- Result.getValue(1));
+ Regs64bit ? X86::RAX : X86::EAX,
+ HalfT, Result.getValue(1));
SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl,
- Regs64bit ? X86::RDX : X86::EDX, HalfT,
- cpOutL.getValue(2));
- SDValue OpsF[] = {cpOutL.getValue(0), cpOutH.getValue(0)};
+ Regs64bit ? X86::RDX : X86::EDX,
+ HalfT, cpOutL.getValue(2));
+ SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS,
MVT::i32, cpOutH.getValue(2));
@@ -36082,7 +36074,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
// Then extract the lower 64-bits.
MVT LdVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32;
SDVTList Tys = DAG.getVTList(LdVT, MVT::Other);
- SDValue Ops[] = {Node->getChain(), Node->getBasePtr()};
+ SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
SDValue Ld = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
MVT::i64, Node->getMemOperand());
if (Subtarget.hasSSE2()) {
@@ -36106,9 +36098,10 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
// First load this into an 80-bit X87 register. This will put the whole
// integer into the significand.
SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
- SDValue Ops[] = {Node->getChain(), Node->getBasePtr()};
- SDValue Result = DAG.getMemIntrinsicNode(
- X86ISD::FILD, dl, Tys, Ops, MVT::i64, Node->getMemOperand());
+ SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
+ SDValue Result = DAG.getMemIntrinsicNode(X86ISD::FILD,
+ dl, Tys, Ops, MVT::i64,
+ Node->getMemOperand());
SDValue Chain = Result.getValue(1);
// Now store the X87 register to a stack temporary and convert to i64.
@@ -36119,7 +36112,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
MachinePointerInfo MPI =
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
- SDValue StoreOps[] = {Chain, Result, StackPtr};
+ SDValue StoreOps[] = { Chain, Result, StackPtr };
Chain = DAG.getMemIntrinsicNode(
X86ISD::FIST, dl, DAG.getVTList(MVT::Other), StoreOps, MVT::i64,
MPI, std::nullopt /*Align*/, MachineMemOperand::MOStore);
@@ -36177,8 +36170,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
assert(getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector &&
"Unexpected type action!");
EVT WideVT = getTypeToTransformTo(*DAG.getContext(), DstVT);
- SDValue Res =
- DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, N->getOperand(0));
+ SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64,
+ N->getOperand(0));
Res = DAG.getBitcast(WideVT, Res);
Results.push_back(Res);
return;
@@ -36200,7 +36193,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
SDValue Mask = Gather->getMask();
assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");
SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT,
- Gather->getPassThru(), DAG.getUNDEF(VT));
+ Gather->getPassThru(),
+ DAG.getUNDEF(VT));
if (!Subtarget.hasVLX()) {
// We need to widen the mask, but the instruction will only use 2
// of its elements. So we can use undef.
@@ -36208,8 +36202,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
DAG.getUNDEF(MVT::v2i1));
Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Mask);
}
- SDValue Ops[] = {Gather->getChain(), PassThru, Mask,
- Gather->getBasePtr(), Index, Gather->getScale()};
+ SDValue Ops[] = { Gather->getChain(), PassThru, Mask,
+ Gather->getBasePtr(), Index, Gather->getScale() };
SDValue Res = DAG.getMemIntrinsicNode(
X86ISD::MGATHER, dl, DAG.getVTList(WideVT, MVT::Other), Ops,
Gather->getMemoryVT(), Gather->getMemOperand());
@@ -36254,7 +36248,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
return;
}
case ISD::ADDRSPACECAST: {
- SDValue V = LowerADDRSPACECAST(SDValue(N, 0), DAG);
+ SDValue V = LowerADDRSPACECAST(SDValue(N,0), DAG);
Results.push_back(V);
return;
}
@@ -36348,7 +36342,7 @@ bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
if (AM.HasBaseReg)
return false;
break;
- default: // Other stuff never works.
+ default: // Other stuff never works.
return false;
}
@@ -36451,13 +36445,12 @@ bool X86TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
if (Val.getOpcode() != ISD::LOAD)
return false;
- if (!VT1.isSimple() || !VT1.isInteger() || !VT2.isSimple() ||
- !VT2.isInteger())
+ if (!VT1.isSimple() || !VT1.isInteger() ||
+ !VT2.isSimple() || !VT2.isInteger())
return false;
switch (VT1.getSimpleVT().SimpleTy) {
- default:
- break;
+ default: break;
case MVT::i8:
case MVT::i16:
case MVT::i32:
@@ -36693,10 +36686,8 @@ static MachineBasicBlock *emitXBegin(MachineInstr &MI, MachineBasicBlock *MBB,
// sinkMBB:
// DstReg := phi(mainDstReg/mainBB, fallDstReg/fallBB)
BuildMI(*sinkMBB, sinkMBB->begin(), MIMD, TII->get(X86::PHI), DstReg)
- .addReg(mainDstReg)
- .addMBB(mainMBB)
- .addReg(fallDstReg)
- .addMBB(fallMBB);
+ .addReg(mainDstReg).addMBB(mainMBB)
+ .addReg(fallDstReg).addMBB(fallMBB);
MI.eraseFromParent();
return sinkMBB;
@@ -36762,8 +36753,8 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
unsigned TotalNumXMMRegs = 8;
bool UseGPOffset = (ArgMode == 1);
bool UseFPOffset = (ArgMode == 2);
- unsigned MaxOffset =
- TotalNumIntRegs * 8 + (UseFPOffset ? TotalNumXMMRegs * 16 : 0);
+ unsigned MaxOffset = TotalNumIntRegs * 8 +
+ (UseFPOffset ? TotalNumXMMRegs * 16 : 0);
/* Align ArgSize to a multiple of 8 */
unsigned ArgSizeA8 = (ArgSize + 7) & ~7;
@@ -36841,14 +36832,13 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
// Check if there is enough room left to pull this argument.
BuildMI(thisMBB, MIMD, TII->get(X86::CMP32ri))
- .addReg(OffsetReg)
- .addImm(MaxOffset + 8 - ArgSizeA8);
+ .addReg(OffsetReg)
+ .addImm(MaxOffset + 8 - ArgSizeA8);
// Branch to "overflowMBB" if offset >= max
// Fall through to "offsetMBB" otherwise
BuildMI(thisMBB, MIMD, TII->get(X86::JCC_1))
- .addMBB(overflowMBB)
- .addImm(X86::COND_AE);
+ .addMBB(overflowMBB).addImm(X86::COND_AE);
}
// In offsetMBB, emit code to use the reg_save_area.
@@ -36889,8 +36879,8 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
// Compute the offset for the next argument
Register NextOffsetReg = MRI.createVirtualRegister(OffsetRegClass);
BuildMI(offsetMBB, MIMD, TII->get(X86::ADD32ri), NextOffsetReg)
- .addReg(OffsetReg)
- .addImm(UseFPOffset ? 16 : 8);
+ .addReg(OffsetReg)
+ .addImm(UseFPOffset ? 16 : 8);
// Store it back into the va_list.
BuildMI(offsetMBB, MIMD, TII->get(X86::MOV32mr))
@@ -36903,7 +36893,8 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
.setMemRefs(StoreOnlyMMO);
// Jump to endMBB
- BuildMI(offsetMBB, MIMD, TII->get(X86::JMP_1)).addMBB(endMBB);
+ BuildMI(offsetMBB, MIMD, TII->get(X86::JMP_1))
+ .addMBB(endMBB);
}
//
@@ -36944,7 +36935,7 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
.addImm(~(uint64_t)(Alignment.value() - 1));
} else {
BuildMI(overflowMBB, MIMD, TII->get(TargetOpcode::COPY), OverflowDestReg)
- .addReg(OverflowAddrReg);
+ .addReg(OverflowAddrReg);
}
// Compute the next overflow address after this argument.
@@ -36970,11 +36961,10 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
// If we branched, emit the PHI to the front of endMBB.
if (offsetMBB) {
- BuildMI(*endMBB, endMBB->begin(), MIMD, TII->get(X86::PHI), DestReg)
- .addReg(OffsetDestReg)
- .addMBB(offsetMBB)
- .addReg(OverflowDestReg)
- .addMBB(overflowMBB);
+ BuildMI(*endMBB, endMBB->begin(), MIMD,
+ TII->get(X86::PHI), DestReg)
+ .addReg(OffsetDestReg).addMBB(offsetMBB)
+ .addReg(OverflowDestReg).addMBB(overflowMBB);
}
// Erase the pseudo instruction
@@ -36989,8 +36979,8 @@ X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
// kill marker, and set it if it should. Returns the correct kill
// marker value.
static bool checkAndUpdateEFLAGSKill(MachineBasicBlock::iterator SelectItr,
- MachineBasicBlock *BB,
- const TargetRegisterInfo *TRI) {
+ MachineBasicBlock* BB,
+ const TargetRegisterInfo* TRI) {
if (isPhysRegUsedAfter(X86::EFLAGS, SelectItr))
return false;
@@ -37457,21 +37447,11 @@ X86TargetLowering::EmitLoweredProbedAlloca(MachineInstr &MI,
//
// + ---- <- ------------ <- ------------- <- ------------ +
// | |
- // [free probe] -> [page alloc] -> [alloc probe] -> [tail alloc] + -> [dyn
- // probe] -> [page alloc] -> [dyn probe] -> [tail alloc] +
- // | |
- // + <-
- // -----------
- // <-
- // ------------
- // <-
- // -----------
- // <-
- // ------------
- // +
+ // [free probe] -> [page alloc] -> [alloc probe] -> [tail alloc] + -> [dyn probe] -> [page alloc] -> [dyn probe] -> [tail alloc] +
+ // | |
+ // + <- ----------- <- ------------ <- ----------- <- ------------ +
//
- // The property we want to enforce is to never have more than [page alloc]
- // between two probes.
+ // The property we want to enforce is to never have more than [page alloc] between two probes.
const unsigned XORMIOpc =
TFI.Uses64BitFramePtr ? X86::XOR64mi32 : X86::XOR32mi;
@@ -37563,61 +37543,56 @@ X86TargetLowering::EmitLoweredSegAlloca(MachineInstr &MI,
// Add code to the main basic block to check if the stack limit has been hit,
// and if so, jump to mallocMBB otherwise to bumpMBB.
BuildMI(BB, MIMD, TII->get(TargetOpcode::COPY), tmpSPVReg).addReg(physSPReg);
- BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::SUB64rr : X86::SUB32rr), SPLimitVReg)
- .addReg(tmpSPVReg)
- .addReg(sizeVReg);
- BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::CMP64mr : X86::CMP32mr))
- .addReg(0)
- .addImm(1)
- .addReg(0)
- .addImm(TlsOffset)
- .addReg(TlsReg)
- .addReg(SPLimitVReg);
+ BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::SUB64rr:X86::SUB32rr), SPLimitVReg)
+ .addReg(tmpSPVReg).addReg(sizeVReg);
+ BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::CMP64mr:X86::CMP32mr))
+ .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg)
+ .addReg(SPLimitVReg);
BuildMI(BB, MIMD, TII->get(X86::JCC_1)).addMBB(mallocMBB).addImm(X86::COND_G);
// bumpMBB simply decreases the stack pointer, since we know the current
// stacklet has enough space.
BuildMI(bumpMBB, MIMD, TII->get(TargetOpcode::COPY), physSPReg)
- .addReg(SPLimitVReg);
+ .addReg(SPLimitVReg);
BuildMI(bumpMBB, MIMD, TII->get(TargetOpcode::COPY), bumpSPPtrVReg)
- .addReg(SPLimitVReg);
+ .addReg(SPLimitVReg);
BuildMI(bumpMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB);
// Calls into a routine in libgcc to allocate more space from the heap.
const uint32_t *RegMask =
Subtarget.getRegisterInfo()->getCallPreservedMask(*MF, CallingConv::C);
if (IsLP64) {
- BuildMI(mallocMBB, MIMD, TII->get(X86::MOV64rr), X86::RDI).addReg(sizeVReg);
+ BuildMI(mallocMBB, MIMD, TII->get(X86::MOV64rr), X86::RDI)
+ .addReg(sizeVReg);
BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32))
- .addExternalSymbol("__morestack_allocate_stack_space")
- .addRegMask(RegMask)
- .addReg(X86::RDI, RegState::Implicit)
- .addReg(X86::RAX, RegState::ImplicitDefine);
+ .addExternalSymbol("__morestack_allocate_stack_space")
+ .addRegMask(RegMask)
+ .addReg(X86::RDI, RegState::Implicit)
+ .addReg(X86::RAX, RegState::ImplicitDefine);
} else if (Is64Bit) {
- BuildMI(mallocMBB, MIMD, TII->get(X86::MOV32rr), X86::EDI).addReg(sizeVReg);
+ BuildMI(mallocMBB, MIMD, TII->get(X86::MOV32rr), X86::EDI)
+ .addReg(sizeVReg);
BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32))
- .addExternalSymbol("__morestack_allocate_stack_space")
- .addRegMask(RegMask)
- .addReg(X86::EDI, RegState::Implicit)
- .addReg(X86::EAX, RegState::ImplicitDefine);
+ .addExternalSymbol("__morestack_allocate_stack_space")
+ .addRegMask(RegMask)
+ .addReg(X86::EDI, RegState::Implicit)
+ .addReg(X86::EAX, RegState::ImplicitDefine);
} else {
- BuildMI(mallocMBB, MIMD, TII->get(X86::SUB32ri), physSPReg)
- .addReg(physSPReg)
- .addImm(12);
+ BuildMI(mallocMBB, MIMD, TII->get(X86::SUB32ri), physSPReg).addReg(physSPReg)
+ .addImm(12);
BuildMI(mallocMBB, MIMD, TII->get(X86::PUSH32r)).addReg(sizeVReg);
BuildMI(mallocMBB, MIMD, TII->get(X86::CALLpcrel32))
- .addExternalSymbol("__morestack_allocate_stack_space")
- .addRegMask(RegMask)
- .addReg(X86::EAX, RegState::ImplicitDefine);
+ .addExternalSymbol("__morestack_allocate_stack_space")
+ .addRegMask(RegMask)
+ .addReg(X86::EAX, RegState::ImplicitDefine);
}
if (!Is64Bit)
- BuildMI(mallocMBB, MIMD, TII->get(X86::ADD32ri), physSPReg)
- .addReg(physSPReg)
- .addImm(16);
+ BuildMI(mallocMBB, MIMD, TII->get(X86::ADD32ri), physSPReg).addReg(physSPReg)
+ .addImm(16);
BuildMI(mallocMBB, MIMD, TII->get(TargetOpcode::COPY), mallocPtrVReg)
- .addReg(IsLP64 ? X86::RAX : X86::EAX);
+ .addReg(IsLP64 ? X86::RAX : X86::EAX);
BuildMI(mallocMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB);
// Set up the CFG correctly.
@@ -37672,8 +37647,7 @@ X86TargetLowering::EmitLoweredCatchRet(MachineInstr &MI,
RestoreMBB->setIsEHPad(true);
auto RestoreMBBI = RestoreMBB->begin();
- BuildMI(*RestoreMBB, RestoreMBBI, MIMD, TII.get(X86::JMP_4))
- .addMBB(TargetMBB);
+ BuildMI(*RestoreMBB, RestoreMBBI, MIMD, TII.get(X86::JMP_4)).addMBB(TargetMBB);
return BB;
}
@@ -37695,10 +37669,9 @@ X86TargetLowering::EmitLoweredTLSCall(MachineInstr &MI,
// FIXME: The 32-bit calls have non-standard calling conventions. Use a
// proper register mask.
const uint32_t *RegMask =
- Subtarget.is64Bit()
- ? Subtarget.getRegisterInfo()->getDarwinTLSCallPreservedMask()
- : Subtarget.getRegisterInfo()->getCallPreservedMask(*F,
- CallingConv::C);
+ Subtarget.is64Bit() ?
+ Subtarget.getRegisterInfo()->getDarwinTLSCallPreservedMask() :
+ Subtarget.getRegisterInfo()->getCallPreservedMask(*F, CallingConv::C);
if (Subtarget.is64Bit()) {
MachineInstrBuilder MIB =
BuildMI(*BB, MI, MIMD, TII->get(X86::MOV64rm), X86::RDI)
@@ -37954,7 +37927,8 @@ X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
MemOpndSlot = CurOp;
MVT PVT = getPointerTy(MF->getDataLayout());
- assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
+ assert((PVT == MVT::i64 || PVT == MVT::i32) &&
+ "Invalid Pointer Size!");
// For v = setjmp(buf), we generate
//
@@ -38002,19 +37976,19 @@ X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
LabelReg = MRI.createVirtualRegister(PtrRC);
if (Subtarget.is64Bit()) {
MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA64r), LabelReg)
- .addReg(X86::RIP)
- .addImm(0)
- .addReg(0)
- .addMBB(restoreMBB)
- .addReg(0);
+ .addReg(X86::RIP)
+ .addImm(0)
+ .addReg(0)
+ .addMBB(restoreMBB)
+ .addReg(0);
} else {
- const X86InstrInfo *XII = static_cast<const X86InstrInfo *>(TII);
+ const X86InstrInfo *XII = static_cast<const X86InstrInfo*>(TII);
MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA32r), LabelReg)
- .addReg(XII->getGlobalBaseReg(MF))
- .addImm(0)
- .addReg(0)
- .addMBB(restoreMBB, Subtarget.classifyBlockAddressReference())
- .addReg(0);
+ .addReg(XII->getGlobalBaseReg(MF))
+ .addImm(0)
+ .addReg(0)
+ .addMBB(restoreMBB, Subtarget.classifyBlockAddressReference())
+ .addReg(0);
}
} else
PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
@@ -38038,7 +38012,7 @@ X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
// Setup
MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::EH_SjLj_Setup))
- .addMBB(restoreMBB);
+ .addMBB(restoreMBB);
const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
MIB.addRegMask(RegInfo->getNoPreservedMask());
@@ -38065,9 +38039,9 @@ X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
Register FramePtr = RegInfo->getFrameRegister(*MF);
Register BasePtr = RegInfo->getBaseRegister();
unsigned Opm = Uses64BitFramePtr ? X86::MOV64rm : X86::MOV32rm;
- addRegOffset(BuildMI(restoreMBB, MIMD, TII->get(Opm), BasePtr), FramePtr,
- true, X86FI->getRestoreBasePointerOffset())
- .setMIFlag(MachineInstr::FrameSetup);
+ addRegOffset(BuildMI(restoreMBB, MIMD, TII->get(Opm), BasePtr),
+ FramePtr, true, X86FI->getRestoreBasePointerOffset())
+ .setMIFlag(MachineInstr::FrameSetup);
}
BuildMI(restoreMBB, MIMD, TII->get(X86::MOV32ri), restoreDstReg).addImm(1);
BuildMI(restoreMBB, MIMD, TII->get(X86::JMP_1)).addMBB(sinkMBB);
@@ -38282,10 +38256,11 @@ X86TargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands());
MVT PVT = getPointerTy(MF->getDataLayout());
- assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
+ assert((PVT == MVT::i64 || PVT == MVT::i32) &&
+ "Invalid Pointer Size!");
const TargetRegisterClass *RC =
- (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
+ (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
Register Tmp = MRI.createVirtualRegister(RC);
// Since FP is only updated here but NOT referenced, it's treated as GPR.
const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
@@ -38665,7 +38640,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
const MIMetadata MIMD(MI);
auto TMMImmToTMMReg = [](unsigned Imm) {
- assert(Imm < 8 && "Illegal tmm index");
+ assert (Imm < 8 && "Illegal tmm index");
return X86::TMM0 + Imm;
};
switch (MI.getOpcode()) {
@@ -38801,30 +38776,29 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
// OR 0b11 into bit 10 and 11. 0b11 is the encoding for round toward zero.
Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
BuildMI(*BB, MI, MIMD, TII->get(X86::OR32ri), NewCW)
- .addReg(OldCW, RegState::Kill)
- .addImm(0xC00);
+ .addReg(OldCW, RegState::Kill).addImm(0xC00);
// Extract to 16 bits.
Register NewCW16 =
MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass);
BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), NewCW16)
- .addReg(NewCW, RegState::Kill, X86::sub_16bit);
+ .addReg(NewCW, RegState::Kill, X86::sub_16bit);
// Prepare memory for FLDCW.
int NewCWFrameIdx =
MF->getFrameInfo().CreateStackObject(2, Align(2), false);
addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOV16mr)),
NewCWFrameIdx)
- .addReg(NewCW16, RegState::Kill);
+ .addReg(NewCW16, RegState::Kill);
// Reload the modified control word now...
- addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
- NewCWFrameIdx);
+ addFrameReference(BuildMI(*BB, MI, MIMD,
+ TII->get(X86::FLDCW16m)), NewCWFrameIdx);
// Get the X86 opcode to use.
unsigned Opc;
switch (MI.getOpcode()) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("illegal opcode!");
case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break;
case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break;
@@ -38835,7 +38809,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break;
case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break;
case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break;
- // clang-format on
+ // clang-format on
}
X86AddressMode AM = getAddressFromInstr(&MI, 0);
@@ -39051,8 +39025,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case X86::PTDPHF8PS: {
unsigned Opc;
switch (MI.getOpcode()) {
- default:
- llvm_unreachable("illegal opcode!");
+ default: llvm_unreachable("illegal opcode!");
// clang-format off
case X86::PTDPBSSD: Opc = X86::TDPBSSD; break;
case X86::PTDPBSUD: Opc = X86::TDPBSUD; break;
@@ -39098,8 +39071,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case X86::PTILESTORED: {
unsigned Opc;
switch (MI.getOpcode()) {
- default:
- llvm_unreachable("illegal opcode!");
+ default: llvm_unreachable("illegal opcode!");
#define GET_EGPR_IF_ENABLED(OPC) (Subtarget.hasEGPR() ? OPC##_EVEX : OPC)
case X86::PTILELOADD:
Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
@@ -39221,9 +39193,11 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
// X86 Optimization Hooks
//===----------------------------------------------------------------------===//
-bool X86TargetLowering::targetShrinkDemandedConstant(
- SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
- TargetLoweringOpt &TLO) const {
+bool
+X86TargetLowering::targetShrinkDemandedConstant(SDValue Op,
+ const APInt &DemandedBits,
+ const APInt &DemandedElts,
+ TargetLoweringOpt &TLO) const {
EVT VT = Op.getValueType();
unsigned Opcode = Op.getOpcode();
unsigned EltSize = VT.getScalarSizeInBits();
@@ -39408,15 +39382,16 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
unsigned NumElts = DemandedElts.getBitWidth();
unsigned Opc = Op.getOpcode();
EVT VT = Op.getValueType();
- assert((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN ||
- Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) &&
+ assert((Opc >= ISD::BUILTIN_OP_END ||
+ Opc == ISD::INTRINSIC_WO_CHAIN ||
+ Opc == ISD::INTRINSIC_W_CHAIN ||
+ Opc == ISD::INTRINSIC_VOID) &&
"Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!");
Known.resetAll();
switch (Opc) {
- default:
- break;
+ default: break;
case X86ISD::MUL_IMM: {
KnownBits Known2;
Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
@@ -39655,7 +39630,7 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
SDValue Op0 = Op.getOperand(0);
SDValue Op1 = Op.getOperand(1);
- if (auto *Cst1 = dyn_cast<ConstantSDNode>(Op1)) {
+ if (auto* Cst1 = dyn_cast<ConstantSDNode>(Op1)) {
unsigned Shift = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 0);
unsigned Length = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 8);
@@ -39881,8 +39856,7 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
unsigned NumElts = VT.getVectorNumElements();
if (Mask.size() == NumElts) {
SmallVector<APInt, 2> DemandedOps(NumOps, APInt(NumElts, 0));
- Known.Zero.setAllBits();
- Known.One.setAllBits();
+ Known.Zero.setAllBits(); Known.One.setAllBits();
for (unsigned i = 0; i != NumElts; ++i) {
if (!DemandedElts[i])
continue;
@@ -40027,18 +40001,16 @@ unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
case X86ISD::ANDNP: {
unsigned Tmp0 =
DAG.ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
- if (Tmp0 == 1)
- return 1; // Early out.
+ if (Tmp0 == 1) return 1; // Early out.
unsigned Tmp1 =
DAG.ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
return std::min(Tmp0, Tmp1);
}
case X86ISD::CMOV: {
- unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
- if (Tmp0 == 1)
- return 1; // Early out.
- unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth + 1);
+ unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth+1);
+ if (Tmp0 == 1) return 1; // Early out.
+ unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth+1);
return std::min(Tmp0, Tmp1);
}
}
@@ -40410,6 +40382,7 @@ static bool matchUnaryPermuteShuffle(MVT MaskVT, ArrayRef<int> Mask,
PermuteImm = (unsigned)ShiftAmt;
return true;
}
+
}
}
@@ -40469,8 +40442,7 @@ static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
// Attempt to match against either an unary or binary PACKSS/PACKUS shuffle.
if (((MaskVT == MVT::v8i16 || MaskVT == MVT::v16i8) && Subtarget.hasSSE2()) ||
- ((MaskVT == MVT::v16i16 || MaskVT == MVT::v32i8) &&
- Subtarget.hasInt256()) ||
+ ((MaskVT == MVT::v16i16 || MaskVT == MVT::v32i8) && Subtarget.hasInt256()) ||
((MaskVT == MVT::v32i16 || MaskVT == MVT::v64i8) && Subtarget.hasBWI())) {
if (matchShuffleWithPACK(MaskVT, SrcVT, V1, V2, Shuffle, Mask, DAG,
Subtarget)) {
@@ -41043,9 +41015,9 @@ static SDValue combineX86ShuffleChain(
SDValue LHS = isInRange(Mask[0], 0, 2) ? V1 : V2;
SDValue RHS = isInRange(Mask[1], 0, 2) ? V1 : V2;
return DAG.getNode(X86ISD::VPERM2X128, DL, RootVT,
- CanonicalizeShuffleInput(RootVT, LHS),
- CanonicalizeShuffleInput(RootVT, RHS),
- DAG.getTargetConstant(PermMask, DL, MVT::i8));
+ CanonicalizeShuffleInput(RootVT, LHS),
+ CanonicalizeShuffleInput(RootVT, RHS),
+ DAG.getTargetConstant(PermMask, DL, MVT::i8));
}
}
}
@@ -41139,8 +41111,8 @@ static SDValue combineX86ShuffleChain(
}
if (matchUnaryPermuteShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
- AllowIntDomain, DAG, Subtarget, Shuffle,
- ShuffleVT, PermuteImm) &&
+ AllowIntDomain, DAG, Subtarget, Shuffle, ShuffleVT,
+ PermuteImm) &&
(!IsMaskedShuffle ||
(NumRootElts == ShuffleVT.getVectorNumElements()))) {
if (Depth == 0 && RootOpc == Shuffle)
@@ -42023,9 +41995,11 @@ static SDValue combineX86ShufflesConstants(MVT VT, ArrayRef<SDValue> Ops,
}
namespace llvm {
-namespace X86 {
-enum { MaxShuffleCombineDepth = 8 };
-} // namespace X86
+ namespace X86 {
+ enum {
+ MaxShuffleCombineDepth = 8
+ };
+ } // namespace X86
} // namespace llvm
/// Fully generic combining of x86 shuffle instructions.
@@ -42429,8 +42403,7 @@ static SDValue combineX86ShufflesRecursively(
// The Op itself may be of different VT, so we need to scale the mask.
unsigned NumOpElts = Op.getValueType().getVectorNumElements();
- APInt OpScaledDemandedElts =
- APIntOps::ScaleBitMask(OpDemandedElts, NumOpElts);
+ APInt OpScaledDemandedElts = APIntOps::ScaleBitMask(OpDemandedElts, NumOpElts);
// Can this operand be simplified any further, given it's demanded elements?
if (SDValue NewOp = TLI.SimplifyMultipleUseDemandedVectorElts(
@@ -43260,7 +43233,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
ISD::isNormalLoad(Src.getNode())) {
LoadSDNode *LN = cast<LoadSDNode>(Src);
SDVTList Tys = DAG.getVTList(VT, MVT::Other);
- SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
+ SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
SDValue BcastLd =
DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
LN->getMemoryVT(), LN->getMemOperand());
@@ -43292,7 +43265,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
// Unless its volatile or atomic.
if (LN->isSimple()) {
SDVTList Tys = DAG.getVTList(VT, MVT::Other);
- SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
+ SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
SDValue BcastLd = DAG.getMemIntrinsicNode(
X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
LN->getPointerInfo(), LN->getBaseAlign(),
@@ -43310,7 +43283,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
LoadSDNode *LN = cast<LoadSDNode>(Src.getOperand(0));
if (LN->getMemoryVT().getSizeInBits() == 16) {
SDVTList Tys = DAG.getVTList(VT, MVT::Other);
- SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
+ SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
SDValue BcastLd =
DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
LN->getMemoryVT(), LN->getMemOperand());
@@ -43337,7 +43310,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
SDVTList Tys = DAG.getVTList(VT, MVT::Other);
SDValue Ptr = DAG.getMemBasePlusOffset(
LN->getBasePtr(), TypeSize::getFixed(Offset), DL);
- SDValue Ops[] = {LN->getChain(), Ptr};
+ SDValue Ops[] = { LN->getChain(), Ptr };
SDValue BcastLd = DAG.getMemIntrinsicNode(
X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
LN->getPointerInfo().getWithOffset(Offset), LN->getBaseAlign(),
@@ -43355,7 +43328,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
MemSDNode *LN = cast<MemIntrinsicSDNode>(Src);
if (LN->getMemoryVT().getSizeInBits() == VT.getScalarSizeInBits()) {
SDVTList Tys = DAG.getVTList(VT, MVT::Other);
- SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
+ SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
SDValue BcastLd =
DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
LN->getMemoryVT(), LN->getMemOperand());
@@ -43916,13 +43889,13 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
if (Op1.getOpcode() == X86ISD::VBROADCAST_LOAD && Op1.hasOneUse()) {
auto *MemIntr = cast<MemIntrinsicSDNode>(Op1);
if (MemIntr->getMemoryVT().getScalarSizeInBits() == 32) {
- SDValue Load =
- DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
- MemIntr->getBasePtr(), MemIntr->getMemOperand());
- SDValue Insert = DAG.getNode(
- X86ISD::INSERTPS, DL, VT, Op0,
- DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Load),
- DAG.getTargetConstant(InsertPSMask & 0x3f, DL, MVT::i8));
+ SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
+ MemIntr->getBasePtr(),
+ MemIntr->getMemOperand());
+ SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0,
+ DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT,
+ Load),
+ DAG.getTargetConstant(InsertPSMask & 0x3f, DL, MVT::i8));
DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
return Insert;
}
@@ -44104,8 +44077,8 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
if (Mask[0] == Mask[1] && Mask[2] == Mask[3] &&
(V.getOpcode() == X86ISD::PSHUFLW ||
V.getOpcode() == X86ISD::PSHUFHW) &&
- V.getOpcode() != N.getOpcode() && V.hasOneUse() &&
- V.getOperand(0).hasOneUse()) {
+ V.getOpcode() != N.getOpcode() &&
+ V.hasOneUse() && V.getOperand(0).hasOneUse()) {
SDValue D = peekThroughOneUseBitcasts(V.getOperand(0));
if (D.getOpcode() == X86ISD::PSHUFD) {
SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
@@ -44179,11 +44152,11 @@ static bool isAddSubOrSubAddMask(ArrayRef<int> Mask, bool &Op0Even) {
/// operation. If true is returned then the operands of ADDSUB(SUBADD) operation
/// are written to the parameters \p Opnd0 and \p Opnd1.
///
-/// We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle
-/// nodes so it is easier to generically match. We also insert dummy vector
-/// shuffle nodes for the operands which explicitly discard the lanes which are
-/// unused by this operation to try to flow through the rest of the combiner the
-/// fact that they're unused.
+/// We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle nodes
+/// so it is easier to generically match. We also insert dummy vector shuffle
+/// nodes for the operands which explicitly discard the lanes which are unused
+/// by this operation to try to flow through the rest of the combiner
+/// the fact that they're unused.
static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
bool &IsSubAdd, bool &HasAllowContract) {
@@ -44218,15 +44191,13 @@ static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
// commute the FADD operands.
SDValue LHS, RHS;
if (V1.getOpcode() == ISD::FSUB) {
- LHS = V1->getOperand(0);
- RHS = V1->getOperand(1);
+ LHS = V1->getOperand(0); RHS = V1->getOperand(1);
if ((V2->getOperand(0) != LHS || V2->getOperand(1) != RHS) &&
(V2->getOperand(0) != RHS || V2->getOperand(1) != LHS))
return false;
} else {
assert(V2.getOpcode() == ISD::FSUB && "Unexpected opcode");
- LHS = V2->getOperand(0);
- RHS = V2->getOperand(1);
+ LHS = V2->getOperand(0); RHS = V2->getOperand(1);
if ((V1->getOperand(0) != LHS || V1->getOperand(1) != RHS) &&
(V1->getOperand(0) != RHS || V1->getOperand(1) != LHS))
return false;
@@ -44237,8 +44208,8 @@ static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
return false;
// It's a subadd if the vector in the even parity is an FADD.
- IsSubAdd =
- Op0Even ? V1->getOpcode() == ISD::FADD : V2->getOpcode() == ISD::FADD;
+ IsSubAdd = Op0Even ? V1->getOpcode() == ISD::FADD
+ : V2->getOpcode() == ISD::FADD;
HasAllowContract =
V1->getFlags().hasAllowContract() && V2->getFlags().hasAllowContract();
@@ -44527,8 +44498,7 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
// TODO: Multiply by zero.
- // If RHS/LHS elements are known zero then we don't need the LHS/RHS
- // equivalent.
+ // If RHS/LHS elements are known zero then we don't need the LHS/RHS equivalent.
APInt DemandedLHSElts = DemandedSrcElts & ~RHSZero;
if (SimplifyDemandedVectorElts(LHS, DemandedLHSElts, LHSUndef, LHSZero, TLO,
Depth + 1))
@@ -45350,7 +45320,7 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
// For splats, unless we *only* demand the 0'th element,
// stop attempts at simplification here, we aren't going to improve things,
// this is better than any potential shuffle.
- if (!DemandedElts.isOne() && TLO.DAG.isSplatValue(Op, /*AllowUndefs*/ false))
+ if (!DemandedElts.isOne() && TLO.DAG.isSplatValue(Op, /*AllowUndefs*/false))
return false;
// Get target/faux shuffle mask.
@@ -45476,7 +45446,7 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
EVT VT = Op.getValueType();
unsigned BitWidth = OriginalDemandedBits.getBitWidth();
unsigned Opc = Op.getOpcode();
- switch (Opc) {
+ switch(Opc) {
case X86ISD::VTRUNC: {
KnownBits KnownOp;
SDValue Src = Op.getOperand(0);
@@ -45484,10 +45454,8 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
// Simplify the input, using demanded bit information.
APInt TruncMask = OriginalDemandedBits.zext(SrcVT.getScalarSizeInBits());
- APInt DemandedElts =
- OriginalDemandedElts.trunc(SrcVT.getVectorNumElements());
- if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, KnownOp, TLO,
- Depth + 1))
+ APInt DemandedElts = OriginalDemandedElts.trunc(SrcVT.getVectorNumElements());
+ if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, KnownOp, TLO, Depth + 1))
return true;
break;
}
@@ -45591,8 +45559,7 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
}
}
- // If we are only demanding sign bits then we can use the shift source
- // directly.
+ // If we are only demanding sign bits then we can use the shift source directly.
unsigned NumSignBits =
TLO.DAG.ComputeNumSignBits(Op0, OriginalDemandedElts, Depth + 1);
unsigned UpperDemandedBits = BitWidth - OriginalDemandedBits.countr_zero();
@@ -45783,8 +45750,8 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
return true;
KnownBits KnownVec;
- if (SimplifyDemandedBits(Vec, DemandedVecBits, DemandedVecElts, KnownVec,
- TLO, Depth + 1))
+ if (SimplifyDemandedBits(Vec, DemandedVecBits, DemandedVecElts,
+ KnownVec, TLO, Depth + 1))
return true;
if (SDValue V = SimplifyMultipleUseDemandedBits(
@@ -46447,13 +46414,13 @@ static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
// Helper to flip between AND/OR/XOR opcodes and their X86ISD FP equivalents.
static unsigned getAltBitOpcode(unsigned Opcode) {
- switch (Opcode) {
- // clang-format off
+ switch(Opcode) {
+ // clang-format off
case ISD::AND: return X86ISD::FAND;
case ISD::OR: return X86ISD::FOR;
case ISD::XOR: return X86ISD::FXOR;
case X86ISD::ANDNP: return X86ISD::FANDN;
- // clang-format on
+ // clang-format on
}
llvm_unreachable("Unknown bitwise opcode");
}
@@ -46676,7 +46643,8 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
// Convert a vXi1 constant build vector to the same width scalar integer.
static SDValue combinevXi1ConstantToInteger(SDValue Op, SelectionDAG &DAG) {
EVT SrcVT = Op.getValueType();
- assert(SrcVT.getVectorElementType() == MVT::i1 && "Expected a vXi1 vector");
+ assert(SrcVT.getVectorElementType() == MVT::i1 &&
+ "Expected a vXi1 vector");
assert(ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
"Expected a constant build vector");
@@ -46992,7 +46960,7 @@ static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
// and the vbroadcast_load are both integer or both fp. In some cases this
// will remove the bitcast entirely.
if (N0.getOpcode() == X86ISD::VBROADCAST_LOAD && N0.hasOneUse() &&
- VT.isFloatingPoint() != SrcVT.isFloatingPoint() && VT.isVector()) {
+ VT.isFloatingPoint() != SrcVT.isFloatingPoint() && VT.isVector()) {
auto *BCast = cast<MemIntrinsicSDNode>(N0);
unsigned SrcVTSize = SrcVT.getScalarSizeInBits();
unsigned MemSize = BCast->getMemoryVT().getScalarSizeInBits();
@@ -47005,7 +46973,7 @@ static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
LoadVT = MVT::getVectorVT(LoadVT, SrcVT.getVectorNumElements());
SDVTList Tys = DAG.getVTList(LoadVT, MVT::Other);
- SDValue Ops[] = {BCast->getChain(), BCast->getBasePtr()};
+ SDValue Ops[] = { BCast->getChain(), BCast->getBasePtr() };
SDValue ResNode =
DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, SDLoc(N), Tys, Ops,
MemVT, BCast->getMemOperand());
@@ -47055,7 +47023,7 @@ static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
bool LowUndef = true, AllUndefOrZero = true;
for (unsigned i = 1, e = SrcVT.getVectorNumElements(); i != e; ++i) {
SDValue Op = N0.getOperand(i);
- LowUndef &= Op.isUndef() || (i >= e / 2);
+ LowUndef &= Op.isUndef() || (i >= e/2);
AllUndefOrZero &= isNullConstantOrUndef(Op);
}
if (AllUndefOrZero) {
@@ -47170,12 +47138,12 @@ static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
// transferring the SSE operand to integer register and back.
unsigned FPOpcode;
switch (N0.getOpcode()) {
- // clang-format off
+ // clang-format off
case ISD::AND: FPOpcode = X86ISD::FAND; break;
case ISD::OR: FPOpcode = X86ISD::FOR; break;
case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
default: return SDValue();
- // clang-format on
+ // clang-format on
}
// Check if we have a bitcast from another integer type as well.
@@ -47276,7 +47244,7 @@ static SDValue createVPDPBUSD(SelectionDAG &DAG, SDValue LHS, SDValue RHS,
// Actually build the DotProduct, split as 256/512 bits for
// AVXVNNI/AVX512VNNI.
auto DpBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
- ArrayRef<SDValue> Ops) {
+ ArrayRef<SDValue> Ops) {
MVT VT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);
return DAG.getNode(X86ISD::VPDPBUSD, DL, VT, Ops);
};
@@ -47316,8 +47284,7 @@ static SDValue createPSADBW(SelectionDAG &DAG, SDValue N0, SDValue N1,
PSADBWBuilder);
}
-// Attempt to replace an all_of/any_of/parity style horizontal reduction with a
-// MOVMSK.
+// Attempt to replace an all_of/any_of/parity style horizontal reduction with a MOVMSK.
static SDValue combineVECREDUCE_LOGIC(SDNode *Reduce, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
// Bail without SSE2.
@@ -47592,9 +47559,9 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
if (Stages > 3) {
unsigned SadElems = SadVT.getVectorNumElements();
- for (unsigned i = Stages - 3; i > 0; --i) {
+ for(unsigned i = Stages - 3; i > 0; --i) {
SmallVector<int, 16> Mask(SadElems, -1);
- for (unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
+ for(unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
Mask[j] = MaskEnd + j;
SDValue Shuffle =
@@ -47910,10 +47877,10 @@ static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG,
SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
Vec.getOperand(0).getValueType().getScalarType(),
Vec.getOperand(0), Index);
- SDValue Ext1 =
- DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Vec.getOperand(1), Index);
- SDValue Ext2 =
- DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Vec.getOperand(2), Index);
+ SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
+ Vec.getOperand(1), Index);
+ SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
+ Vec.getOperand(2), Index);
return DAG.getNode(ISD::SELECT, DL, VT, Ext0, Ext1, Ext2);
}
@@ -48163,9 +48130,8 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
return DAG.getConstant(EltBits[Idx].zext(NumEltBits), dl, VT);
}
- // Convert extract_element(bitcast(<X x i1>) ->
- // bitcast(extract_subvector()). Improves lowering of bool masks on rust
- // which splits them into byte array.
+ // Convert extract_element(bitcast(<X x i1>) -> bitcast(extract_subvector()).
+ // Improves lowering of bool masks on rust which splits them into byte array.
if (InputVector.getOpcode() == ISD::BITCAST && (NumEltBits % 8) == 0) {
SDValue Src = peekThroughBitcasts(InputVector);
if (Src.getValueType().getScalarType() == MVT::i1 &&
@@ -48567,7 +48533,8 @@ static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG,
TargetLowering::DAGCombinerInfo &DCI,
const X86Subtarget &Subtarget) {
SDValue Cond = N->getOperand(0);
- if ((N->getOpcode() != ISD::VSELECT && N->getOpcode() != X86ISD::BLENDV) ||
+ if ((N->getOpcode() != ISD::VSELECT &&
+ N->getOpcode() != X86ISD::BLENDV) ||
ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
return SDValue();
@@ -49002,13 +48969,13 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
Cond1 == InnerSetCC.getOperand(1)) {
ISD::CondCode NewCC;
switch (CC == ISD::SETEQ ? InnerCC : CC) {
- // clang-format off
+ // clang-format off
case ISD::SETGT: NewCC = ISD::SETGE; break;
case ISD::SETLT: NewCC = ISD::SETLE; break;
case ISD::SETUGT: NewCC = ISD::SETUGE; break;
case ISD::SETULT: NewCC = ISD::SETULE; break;
default: NewCC = ISD::SETCC_INVALID; break;
- // clang-format on
+ // clang-format on
}
if (NewCC != ISD::SETCC_INVALID) {
Cond = DAG.getSetCC(DL, CondVT, Cond0, Cond1, NewCC);
@@ -49182,9 +49149,9 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
// 16-bit lacks a proper blendv.
// TODO: Can we merge this into the LowerVSETCC handling of uniform masks?
bool CanShiftBlend =
- TLI.isTypeLegal(VT) &&
- ((Subtarget.hasAVX() && EltBitWidth == 32) ||
- (Subtarget.hasAVX2() && EltBitWidth == 64) || (Subtarget.hasXOP()));
+ TLI.isTypeLegal(VT) && ((Subtarget.hasAVX() && EltBitWidth == 32) ||
+ (Subtarget.hasAVX2() && EltBitWidth == 64) ||
+ (Subtarget.hasXOP()));
if (CanShiftBlend && DAG.isKnownToBeAPowerOfTwo(Mask)) {
// Create a left-shift constant to get the mask bits over to the sign-bit.
// vsel ((X & C) == 0), LHS, RHS --> vsel ((shl X, C') < 0), RHS, LHS
@@ -49420,7 +49387,7 @@ static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
SDValue Op2 = Cmp.getOperand(1);
SDValue SetCC;
- const ConstantSDNode *C = nullptr;
+ const ConstantSDNode* C = nullptr;
bool needOppositeCond = (CC == X86::COND_E);
bool checkAgainstTrue = false; // Is it a comparison against 1?
@@ -49441,7 +49408,8 @@ static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
bool truncatedToBoolWithAnd = false;
// Skip (zext $x), (trunc $x), or (and $x, 1) node.
while (SetCC.getOpcode() == ISD::ZERO_EXTEND ||
- SetCC.getOpcode() == ISD::TRUNCATE || SetCC.getOpcode() == ISD::AND) {
+ SetCC.getOpcode() == ISD::TRUNCATE ||
+ SetCC.getOpcode() == ISD::AND) {
if (SetCC.getOpcode() == ISD::AND) {
int OpIdx = -1;
if (isOneConstant(SetCC.getOperand(0)))
@@ -49484,13 +49452,13 @@ static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
if (!FVal) {
SDValue Op = SetCC.getOperand(0);
// Skip 'zext' or 'trunc' node.
- if (Op.getOpcode() == ISD::ZERO_EXTEND || Op.getOpcode() == ISD::TRUNCATE)
+ if (Op.getOpcode() == ISD::ZERO_EXTEND ||
+ Op.getOpcode() == ISD::TRUNCATE)
Op = Op.getOperand(0);
// A special case for rdrand/rdseed, where 0 is set if false cond is
// found.
if ((Op.getOpcode() != X86ISD::RDRAND &&
- Op.getOpcode() != X86ISD::RDSEED) ||
- Op.getResNo() != 0)
+ Op.getOpcode() != X86ISD::RDSEED) || Op.getResNo() != 0)
return SDValue();
}
// Quit if false value is not the constant 0 or 1.
@@ -49535,8 +49503,7 @@ static bool checkBoolTestAndOrSetCCCombine(SDValue Cond, X86::CondCode &CC0,
SDValue SetCC0, SetCC1;
switch (Cond->getOpcode()) {
- default:
- return false;
+ default: return false;
case ISD::AND:
case X86ISD::AND:
isAnd = true;
@@ -49601,7 +49568,8 @@ static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) {
}
// If this is a check of the z flag of an add with 1, switch to the
// C flag.
- if (CarryCC == X86::COND_E && CarryOp1.getOpcode() == X86ISD::ADD &&
+ if (CarryCC == X86::COND_E &&
+ CarryOp1.getOpcode() == X86ISD::ADD &&
isOneConstant(CarryOp1.getOperand(1)))
return CarryOp1;
} else if (FoundAndLSB) {
@@ -50136,11 +50104,12 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
// Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. This is efficient
// for any integer data type, including i8/i16.
- if (FalseC->getAPIntValue() + 1 == TrueC->getAPIntValue()) {
+ if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
Cond = getSETCC(CC, Cond, DL, DAG);
// Zero extend the condition if needed.
- Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), Cond);
+ Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
+ FalseC->getValueType(0), Cond);
Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
SDValue(FalseC, 0));
return Cond;
@@ -50156,25 +50125,24 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
bool isFastMultiplier = false;
if (Diff.ult(10)) {
switch (Diff.getZExtValue()) {
- default:
- break;
- case 1: // result = add base, cond
- case 2: // result = lea base( , cond*2)
- case 3: // result = lea base(cond, cond*2)
- case 4: // result = lea base( , cond*4)
- case 5: // result = lea base(cond, cond*4)
- case 8: // result = lea base( , cond*8)
- case 9: // result = lea base(cond, cond*8)
+ default: break;
+ case 1: // result = add base, cond
+ case 2: // result = lea base( , cond*2)
+ case 3: // result = lea base(cond, cond*2)
+ case 4: // result = lea base( , cond*4)
+ case 5: // result = lea base(cond, cond*4)
+ case 8: // result = lea base( , cond*8)
+ case 9: // result = lea base(cond, cond*8)
isFastMultiplier = true;
break;
}
}
if (isFastMultiplier) {
- Cond = getSETCC(CC, Cond, DL, DAG);
+ Cond = getSETCC(CC, Cond, DL ,DAG);
// Zero extend the condition if needed.
- Cond =
- DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), Cond);
+ Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
+ Cond);
// Scale the condition by the difference.
if (Diff != 1)
Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
@@ -50752,6 +50720,7 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
EVT VT = N->getValueType(0);
SDLoc DL(N);
+
if (!Subtarget.hasSSE2())
return SDValue();
@@ -50767,14 +50736,16 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
bool IsSigned = Opc == ISD::SDIV || Opc == ISD::SREM;
// Scalar functions that do not use XMM registers must not get this combine.
- if (!VT.isVector() && (Subtarget.useSoftFloat() ||
- DAG.getMachineFunction().getFunction().hasFnAttribute(
- Attribute::NoImplicitFloat)))
+ if (!VT.isVector() &&
+ (Subtarget.useSoftFloat() ||
+ DAG.getMachineFunction().getFunction().hasFnAttribute(
+ Attribute::NoImplicitFloat)))
return SDValue();
// If the result is only read back as scalar extracts, scalarization computes
// just the demanded lanes.
- if (VT.isVector() && all_of(N->users(), [](const SDNode *U) {
+ if (VT.isVector() &&
+ all_of(N->users(), [](const SDNode *U) {
return U->getOpcode() == ISD::EXTRACT_VECTOR_ELT;
}))
return SDValue();
@@ -50824,11 +50795,11 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
// Nothing will split an illegal FP type after type legalization and the
// strict SAE divide is 512-bit only. Scalar f32/f64 are always legal.
- bool FPVTUsable =
- !VT.isVector() ||
- (IsStrict ? FPVT.getSizeInBits() <= 512
- : DCI.isBeforeLegalize() ||
- DAG.getTargetLoweringInfo().isTypeLegal(FPVT));
+ bool FPVTUsable = !VT.isVector() ||
+ (IsStrict
+ ? FPVT.getSizeInBits() <= 512
+ : DCI.isBeforeLegalize() ||
+ DAG.getTargetLoweringInfo().isTypeLegal(FPVT));
// Halve the divide while the integer halves stay legal.
if (!FPVTUsable) {
@@ -51131,7 +51102,8 @@ static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG,
// fold (shl (and (setcc_c), c1), c2) -> (and setcc_c, (c1 << c2))
// since the result of setcc_c is all zero's or all ones.
- if (VT.isInteger() && !VT.isVector() && N1C && N0.getOpcode() == ISD::AND &&
+ if (VT.isInteger() && !VT.isVector() &&
+ N1C && N0.getOpcode() == ISD::AND &&
N0.getOperand(1).getOpcode() == ISD::Constant) {
SDValue N00 = N0.getOperand(0);
APInt Mask = N0.getConstantOperandAPInt(1);
@@ -51215,7 +51187,7 @@ static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG,
if (SraConst.isNegative())
return SDValue();
- for (MVT SVT : {MVT::i8, MVT::i16, MVT::i32}) {
+ for (MVT SVT : { MVT::i8, MVT::i16, MVT::i32 }) {
unsigned ShiftSize = SVT.getSizeInBits();
// Only deal with (Size - ShlConst) being equal to 8, 16 or 32.
if (ShiftSize >= Size || ShlConst != Size - ShiftSize)
@@ -51578,8 +51550,8 @@ static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
// Try to combine a PACKUSWB/PACKSSWB implemented truncate with a regular
// truncate to create a larger truncate.
- if (Subtarget.hasAVX512() && N0.getOpcode() == ISD::TRUNCATE &&
- N1.isUndef() && VT == MVT::v16i8 &&
+ if (Subtarget.hasAVX512() &&
+ N0.getOpcode() == ISD::TRUNCATE && N1.isUndef() && VT == MVT::v16i8 &&
N0.getOperand(0).getValueType() == MVT::v8i32) {
if ((IsSigned && DAG.ComputeNumSignBits(N0) > 8) ||
(!IsSigned &&
@@ -51964,7 +51936,7 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
SDValue CMP00 = CMP0->getOperand(0);
SDValue CMP01 = CMP0->getOperand(1);
- EVT VT = CMP00.getValueType();
+ EVT VT = CMP00.getValueType();
if (VT == MVT::f32 || VT == MVT::f64 ||
(VT == MVT::f16 && Subtarget.hasFP16())) {
@@ -51990,10 +51962,8 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
}
if (!ExpectingFlags) {
- enum X86::CondCode cc0 =
- (enum X86::CondCode)N0.getConstantOperandVal(0);
- enum X86::CondCode cc1 =
- (enum X86::CondCode)N1.getConstantOperandVal(0);
+ enum X86::CondCode cc0 = (enum X86::CondCode)N0.getConstantOperandVal(0);
+ enum X86::CondCode cc1 = (enum X86::CondCode)N1.getConstantOperandVal(0);
if (cc1 == X86::COND_E || cc1 == X86::COND_NE) {
X86::CondCode tmp = cc0;
@@ -52001,7 +51971,7 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
cc1 = tmp;
}
- if ((cc0 == X86::COND_E && cc1 == X86::COND_NP) ||
+ if ((cc0 == X86::COND_E && cc1 == X86::COND_NP) ||
(cc0 == X86::COND_NE && cc1 == X86::COND_P)) {
// FIXME: need symbolic constants for these magic numbers.
// See X86ATTInstPrinter.cpp:printSSECC().
@@ -52011,8 +51981,7 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01,
DAG.getTargetConstant(x86cc, DL, MVT::i8));
// Need to fill with zeros to ensure the bitcast will produce zeroes
- // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee
- // that.
+ // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1,
DAG.getConstant(0, DL, MVT::v16i1),
FSetCC, DAG.getVectorIdxConstant(0, DL));
@@ -52044,8 +52013,8 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
SDValue OnesOrZeroesI = DAG.getBitcast(IntVT, OnesOrZeroesF);
SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
DAG.getConstant(1, DL, IntVT));
- SDValue OneBitOfTruth =
- DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ANDed);
+ SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
+ ANDed);
return OneBitOfTruth;
}
}
@@ -52240,8 +52209,7 @@ static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL,
assert(VT.isVector() && "Expected vector type");
assert((N.getOpcode() == ISD::ANY_EXTEND ||
N.getOpcode() == ISD::ZERO_EXTEND ||
- N.getOpcode() == ISD::SIGN_EXTEND) &&
- "Invalid Node");
+ N.getOpcode() == ISD::SIGN_EXTEND) && "Invalid Node");
SDValue Narrow = N.getOperand(0);
EVT NarrowVT = Narrow.getValueType();
@@ -52251,27 +52219,26 @@ static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL,
if (!Op)
return SDValue();
switch (N.getOpcode()) {
- default:
- llvm_unreachable("Unexpected opcode");
+ default: llvm_unreachable("Unexpected opcode");
case ISD::ANY_EXTEND:
return Op;
case ISD::ZERO_EXTEND:
return DAG.getZeroExtendInReg(Op, DL, NarrowVT);
case ISD::SIGN_EXTEND:
- return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op,
- DAG.getValueType(NarrowVT));
+ return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,
+ Op, DAG.getValueType(NarrowVT));
}
}
static unsigned convertIntLogicToFPLogicOpcode(unsigned Opcode) {
unsigned FPOpcode;
switch (Opcode) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected input node for FP logic conversion");
case ISD::AND: FPOpcode = X86ISD::FAND; break;
case ISD::OR: FPOpcode = X86ISD::FOR; break;
case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
- // clang-format on
+ // clang-format on
}
return FPOpcode;
}
@@ -52748,7 +52715,8 @@ static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG,
SmallVector<SDValue, 4> Ops(Src.getNumOperands(),
DAG.getConstant(0, dl, SubVecVT));
Ops[0] = SubVec;
- SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT, Ops);
+ SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT,
+ Ops);
EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcVT.getSizeInBits());
return DAG.getZExtOrTrunc(DAG.getBitcast(IntVT, Concat), dl, VT);
}
@@ -53100,7 +53068,7 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
return R;
- if (SDValue R = combineAndNotIntoANDNP(N, dl, DAG))
+ if (SDValue R = combineAndNotIntoANDNP(N, dl ,DAG))
return R;
if (SDValue ShiftRight = combineAndMaskToShift(N, dl, DAG, Subtarget))
@@ -53946,8 +53914,7 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
if (NotCond) {
SDValue R = DAG.getZExtOrTrunc(NotCond, dl, VT);
- R = DAG.getNode(ISD::MUL, dl, VT, R,
- DAG.getConstant(Val + 1, dl, VT));
+ R = DAG.getNode(ISD::MUL, dl, VT, R, DAG.getConstant(Val + 1, dl, VT));
R = DAG.getNode(ISD::SUB, dl, VT, R, DAG.getConstant(1, dl, VT));
return R;
}
@@ -54094,7 +54061,7 @@ static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
return SDValue();
switch (VT.getSimpleVT().SimpleTy) {
- // clang-format off
+ // clang-format off
default: return SDValue();
case MVT::v16i8:
case MVT::v8i16:
@@ -54224,8 +54191,8 @@ static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
// split across two registers. We can use a packusdw+perm to clamp to 0-65535
// and concatenate at the same time. Then we can use a final vpmovuswb to
// clip to 0-255.
- if (Subtarget.hasBWI() && !Subtarget.useAVX512Regs() && InVT == MVT::v16i32 &&
- VT == MVT::v16i8) {
+ if (Subtarget.hasBWI() && !Subtarget.useAVX512Regs() &&
+ InVT == MVT::v16i32 && VT == MVT::v16i8) {
if (SDValue USatVal = detectSSatPattern(In, VT, true)) {
// Emit a VPACKUSDW+VPERMQ followed by a VPMOVUSWB.
SDValue Mid = truncateVectorWithPACK(X86ISD::PACKUS, MVT::v16i16, USatVal,
@@ -54241,12 +54208,11 @@ static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
// FIXME: We could widen truncates to 512 to remove the VLX restriction.
// If the result type is 256-bits or larger and we have disable 512-bit
// registers, we should go ahead and use the pack instructions if possible.
- bool PreferAVX512 =
- ((Subtarget.hasAVX512() && InSVT == MVT::i32) ||
- (Subtarget.hasBWI() && InSVT == MVT::i16)) &&
- (InVT.getSizeInBits() > 128) &&
- (Subtarget.hasVLX() || InVT.getSizeInBits() > 256) &&
- !(!Subtarget.useAVX512Regs() && VT.getSizeInBits() >= 256);
+ bool PreferAVX512 = ((Subtarget.hasAVX512() && InSVT == MVT::i32) ||
+ (Subtarget.hasBWI() && InSVT == MVT::i16)) &&
+ (InVT.getSizeInBits() > 128) &&
+ (Subtarget.hasVLX() || InVT.getSizeInBits() > 256) &&
+ !(!Subtarget.useAVX512Regs() && VT.getSizeInBits() >= 256);
if (!PreferAVX512 && VT.getVectorNumElements() > 1 &&
isPowerOf2_32(VT.getVectorNumElements()) &&
@@ -54259,8 +54225,8 @@ static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
SDValue Mid = truncateVectorWithPACK(X86ISD::PACKSS, MidVT, USatVal, DL,
DAG, Subtarget);
assert(Mid && "Failed to pack!");
- SDValue V =
- truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG, Subtarget);
+ SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG,
+ Subtarget);
assert(V && "Failed to pack!");
return V;
} else if (SVT == MVT::i8 || Subtarget.hasSSE41())
@@ -54627,9 +54593,10 @@ reduceMaskedLoadToScalarLoad(MaskedLoadSDNode *ML, SelectionDAG &DAG,
CastVT = VT.changeVectorElementType(*DAG.getContext(), EltVT);
}
- SDValue Load = DAG.getLoad(EltVT, DL, ML->getChain(), Addr,
- ML->getPointerInfo().getWithOffset(Offset),
- Alignment, ML->getMemOperand()->getFlags());
+ SDValue Load =
+ DAG.getLoad(EltVT, DL, ML->getChain(), Addr,
+ ML->getPointerInfo().getWithOffset(Offset),
+ Alignment, ML->getMemOperand()->getFlags());
SDValue PassThru = DAG.getBitcast(CastVT, ML->getPassThru());
@@ -54660,8 +54627,8 @@ combineMaskedLoadConstantMask(MaskedLoadSDNode *ML, SelectionDAG &DAG,
if (LoadFirstElt && LoadLastElt) {
SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(),
ML->getMemOperand());
- SDValue Blend =
- DAG.getSelect(DL, VT, ML->getMask(), VecLd, ML->getPassThru());
+ SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd,
+ ML->getPassThru());
return DCI.CombineTo(ML, Blend, VecLd.getValue(1), true);
}
@@ -54683,8 +54650,8 @@ combineMaskedLoadConstantMask(MaskedLoadSDNode *ML, SelectionDAG &DAG,
VT, DL, ML->getChain(), ML->getBasePtr(), ML->getOffset(), ML->getMask(),
DAG.getUNDEF(VT), ML->getMemoryVT(), ML->getMemOperand(),
ML->getAddressingMode(), ML->getExtensionType());
- SDValue Blend =
- DAG.getSelect(DL, VT, ML->getMask(), NewML, ML->getPassThru());
+ SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML,
+ ML->getPassThru());
return DCI.CombineTo(ML, Blend, NewML.getValue(1), true);
}
@@ -54783,8 +54750,8 @@ static SDValue reduceMaskedStoreToScalarStore(MaskedStoreSDNode *MS,
// Store that element at the appropriate offset from the base pointer.
return DAG.getStore(MS->getChain(), DL, Extract, Addr,
- MS->getPointerInfo().getWithOffset(Offset), Alignment,
- MS->getMemOperand()->getFlags());
+ MS->getPointerInfo().getWithOffset(Offset),
+ Alignment, MS->getMemOperand()->getFlags());
}
static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
@@ -55140,16 +55107,15 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
// Turn vXi1 stores of constants into a scalar store.
if ((VT == MVT::v8i1 || VT == MVT::v16i1 || VT == MVT::v32i1 ||
- VT == MVT::v64i1) &&
- VT == StVT && TLI.isTypeLegal(VT) &&
+ VT == MVT::v64i1) && VT == StVT && TLI.isTypeLegal(VT) &&
ISD::isBuildVectorOfConstantSDNodes(StoredVal.getNode())) {
// If its a v64i1 store without 64-bit support, we need two stores.
if (!DCI.isBeforeLegalize() && VT == MVT::v64i1 && !Subtarget.is64Bit()) {
- SDValue Lo =
- DAG.getBuildVector(MVT::v32i1, dl, StoredVal->ops().slice(0, 32));
+ SDValue Lo = DAG.getBuildVector(MVT::v32i1, dl,
+ StoredVal->ops().slice(0, 32));
Lo = combinevXi1ConstantToInteger(Lo, DAG);
- SDValue Hi =
- DAG.getBuildVector(MVT::v32i1, dl, StoredVal->ops().slice(32, 32));
+ SDValue Hi = DAG.getBuildVector(MVT::v32i1, dl,
+ StoredVal->ops().slice(32, 32));
Hi = combinevXi1ConstantToInteger(Hi, DAG);
SDValue Ptr0 = St->getBasePtr();
@@ -55299,14 +55265,14 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
if (TLI.isTruncStoreLegal(VT, StVT, St->getAlign(),
St->getAddressSpace())) {
if (SDValue Val = detectSSatPattern(St->getValue(), St->getMemoryVT()))
- return EmitTruncSStore(true /* Signed saturation */, St->getChain(), dl,
- Val, St->getBasePtr(), St->getMemoryVT(),
- St->getMemOperand(), DAG);
- if (SDValue Val =
- detectUSatPattern(St->getValue(), St->getMemoryVT(), DAG, dl))
+ return EmitTruncSStore(true /* Signed saturation */, St->getChain(),
+ dl, Val, St->getBasePtr(),
+ St->getMemoryVT(), St->getMemOperand(), DAG);
+ if (SDValue Val = detectUSatPattern(St->getValue(), St->getMemoryVT(),
+ DAG, dl))
return EmitTruncSStore(false /* Unsigned saturation */, St->getChain(),
- dl, Val, St->getBasePtr(), St->getMemoryVT(),
- St->getMemOperand(), DAG);
+ dl, Val, St->getBasePtr(),
+ St->getMemoryVT(), St->getMemOperand(), DAG);
}
return SDValue();
@@ -56144,7 +56110,8 @@ static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL,
// (i16 (ssat (add (mul (zext (even elts (i8 A))), (sext (even elts (i8 B)))),
// (mul (zext (odd elts (i8 A)), (sext (odd elts (i8 B))))))))
static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
- const X86Subtarget &Subtarget, const SDLoc &DL) {
+ const X86Subtarget &Subtarget,
+ const SDLoc &DL) {
using namespace SDPatternMatch;
if (!VT.isVector() || !Subtarget.hasSSSE3())
return SDValue();
@@ -56218,8 +56185,8 @@ static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
std::swap(IdxN01, IdxN11);
}
// N0 indices be the even element. N1 indices must be the next odd element.
- if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 || IdxN01 != 2 * i ||
- IdxN11 != 2 * i + 1)
+ if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 ||
+ IdxN01 != 2 * i || IdxN11 != 2 * i + 1)
return SDValue();
SDValue N00In = N00Elt.getOperand(0);
SDValue N01In = N01Elt.getOperand(0);
@@ -56230,8 +56197,8 @@ static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
ZExtIn = N00In;
SExtIn = N01In;
}
- if (ZExtIn != N00In || SExtIn != N01In || ZExtIn != N10In ||
- SExtIn != N11In)
+ if (ZExtIn != N00In || SExtIn != N01In ||
+ ZExtIn != N10In || SExtIn != N11In)
return SDValue();
}
@@ -56251,13 +56218,14 @@ static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
// Shrink by adding truncate nodes and let DAGCombine fold with the
// sources.
EVT InVT = Ops[0].getValueType();
- assert(InVT.getScalarType() == MVT::i8 && "Unexpected scalar element type");
+ assert(InVT.getScalarType() == MVT::i8 &&
+ "Unexpected scalar element type");
assert(InVT == Ops[1].getValueType() && "Operands' types mismatch");
EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
InVT.getVectorNumElements() / 2);
return DAG.getNode(X86ISD::VPMADDUBSW, DL, ResVT, Ops[0], Ops[1]);
};
- return SplitOpsAndApply(DAG, Subtarget, DL, VT, {ZExtIn, SExtIn},
+ return SplitOpsAndApply(DAG, Subtarget, DL, VT, { ZExtIn, SExtIn },
PMADDBuilder);
}
@@ -56505,7 +56473,7 @@ static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc,
bool NegRes) {
if (NegMul) {
switch (Opcode) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected opcode");
case ISD::FMA: Opcode = X86ISD::FNMADD; break;
case ISD::STRICT_FMA: Opcode = X86ISD::STRICT_FNMADD; break;
@@ -56519,13 +56487,13 @@ static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc,
case X86ISD::FNMSUB: Opcode = X86ISD::FMSUB; break;
case X86ISD::STRICT_FNMSUB: Opcode = X86ISD::STRICT_FMSUB; break;
case X86ISD::FNMSUB_RND: Opcode = X86ISD::FMSUB_RND; break;
- // clang-format on
+ // clang-format on
}
}
if (NegAcc) {
switch (Opcode) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected opcode");
case ISD::FMA: Opcode = X86ISD::FMSUB; break;
case ISD::STRICT_FMA: Opcode = X86ISD::STRICT_FMSUB; break;
@@ -56543,7 +56511,7 @@ static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc,
case X86ISD::FMADDSUB_RND: Opcode = X86ISD::FMSUBADD_RND; break;
case X86ISD::FMSUBADD: Opcode = X86ISD::FMADDSUB; break;
case X86ISD::FMSUBADD_RND: Opcode = X86ISD::FMADDSUB_RND; break;
- // clang-format on
+ // clang-format on
}
}
@@ -56560,7 +56528,7 @@ static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc,
case X86ISD::FNMADD_RND: Opcode = X86ISD::FMSUB_RND; break;
case X86ISD::FNMSUB: Opcode = ISD::FMA; break;
case X86ISD::FNMSUB_RND: Opcode = X86ISD::FMADD_RND; break;
- // clang-format on
+ // clang-format on
}
}
@@ -56704,13 +56672,13 @@ static SDValue combineX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
Op1 = DAG.getBitcast(IntVT, Op1);
unsigned IntOpcode;
switch (N->getOpcode()) {
- // clang-format off
+ // clang-format off
default: llvm_unreachable("Unexpected FP logic op");
case X86ISD::FOR: IntOpcode = ISD::OR; break;
case X86ISD::FXOR: IntOpcode = ISD::XOR; break;
case X86ISD::FAND: IntOpcode = ISD::AND; break;
case X86ISD::FANDN: IntOpcode = X86ISD::ANDNP; break;
- // clang-format on
+ // clang-format on
}
SDValue IntOp = DAG.getNode(IntOpcode, dl, IntVT, Op0, Op1);
return DAG.getBitcast(VT, IntOp);
@@ -57150,18 +57118,17 @@ static SDValue combineFMinFMax(SDNode *N, SelectionDAG &DAG) {
// into FMINC and FMAXC, which are Commutative operations.
unsigned NewOp = 0;
switch (N->getOpcode()) {
- default:
- llvm_unreachable("unknown opcode");
- case ISD::PSEUDO_FMIN:
- NewOp = X86ISD::FMINC;
- break;
- case ISD::PSEUDO_FMAX:
- NewOp = X86ISD::FMAXC;
- break;
- }
+ default: llvm_unreachable("unknown opcode");
+ case ISD::PSEUDO_FMIN:
+ NewOp = X86ISD::FMINC;
+ break;
+ case ISD::PSEUDO_FMAX:
+ NewOp = X86ISD::FMAXC;
+ break;
+ }
- return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0), N->getOperand(0),
- N->getOperand(1));
+ return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0),
+ N->getOperand(0), N->getOperand(1));
}
static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG,
@@ -57208,8 +57175,8 @@ static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG,
if (!VT.isVector() && DAG.getMachineFunction().getFunction().hasMinSize())
return SDValue();
- EVT SetCCType =
- TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+ EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
+ VT);
// There are 4 possibilities involving NaN inputs, and these are the required
// outputs:
@@ -57259,8 +57226,8 @@ static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG,
MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);
if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
SDLoc dl(N);
- SDValue Convert =
- DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(InVT, VZLoad));
+ SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
+ DAG.getBitcast(InVT, VZLoad));
DCI.CombineTo(N, Convert);
DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
DCI.recursivelyDeleteUnusedNodes(LN);
@@ -57853,8 +57820,8 @@ static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
// Only combine legal element types.
EVT SVT = VT.getVectorElementType();
- if (SVT != MVT::i8 && SVT != MVT::i16 && SVT != MVT::i32 && SVT != MVT::i64 &&
- SVT != MVT::f32 && SVT != MVT::f64)
+ if (SVT != MVT::i8 && SVT != MVT::i16 && SVT != MVT::i32 &&
+ SVT != MVT::i64 && SVT != MVT::f32 && SVT != MVT::f64)
return SDValue();
// We don't have CMPP Instruction for vxf16
@@ -57894,15 +57861,16 @@ static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
SDLoc DL(N);
// (i32 (sext (i8 (x86isd::setcc_carry)))) -> (i32 (x86isd::setcc_carry))
- if (!DCI.isBeforeLegalizeOps() && N0.getOpcode() == X86ISD::SETCC_CARRY) {
+ if (!DCI.isBeforeLegalizeOps() &&
+ N0.getOpcode() == X86ISD::SETCC_CARRY) {
SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0),
- N0->getOperand(1));
+ N0->getOperand(1));
bool ReplaceOtherUses = !N0.hasOneUse();
DCI.CombineTo(N, Setcc);
// Replace other uses with a truncate of the widened setcc_carry.
if (ReplaceOtherUses) {
- SDValue Trunc =
- DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), Setcc);
+ SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
+ N0.getValueType(), Setcc);
DCI.CombineTo(N0.getNode(), Trunc);
}
@@ -58195,13 +58163,13 @@ static SDValue combineZext(SDNode *N, SelectionDAG &DAG,
if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ANY_EXTEND &&
N0.getOpcode() == X86ISD::SETCC_CARRY) {
SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, N0->getOperand(0),
- N0->getOperand(1));
+ N0->getOperand(1));
bool ReplaceOtherUses = !N0.hasOneUse();
DCI.CombineTo(N, Setcc);
// Replace other uses with a truncate of the widened setcc_carry.
if (ReplaceOtherUses) {
- SDValue Trunc =
- DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), Setcc);
+ SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
+ N0.getValueType(), Setcc);
DCI.CombineTo(N0.getNode(), Trunc);
}
@@ -59042,8 +59010,8 @@ static SDValue combineVectorCompareAndMaskUnaryOp(SDNode *N,
SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
// The AND node needs bitcasts to/from an integer vector type around it.
SDValue MaskConst = DAG.getBitcast(IntVT, SourceConst);
- SDValue NewAnd =
- DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0), MaskConst);
+ SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0),
+ MaskConst);
SDValue Res = DAG.getBitcast(VT, NewAnd);
if (IsStrict)
return DAG.getMergeValues({Res, SourceConst.getValue(1)}, DL);
@@ -59229,8 +59197,8 @@ static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG,
// use CVTSI2P.
assert(InVT == MVT::v2i64 && "Unexpected VT!");
SDValue Cast = DAG.getBitcast(MVT::v4i32, Op0);
- SDValue Shuf =
- DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast, {0, 2, -1, -1});
+ SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast,
+ { 0, 2, -1, -1 });
if (IsStrict)
return DAG.getNode(X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other},
{N->getOperand(0), Shuf});
@@ -59426,7 +59394,7 @@ static bool needCarryOrOverflowFlag(SDValue Flags) {
}
switch (CC) {
- // clang-format off
+ // clang-format off
default: break;
case X86::COND_A: case X86::COND_AE:
case X86::COND_B: case X86::COND_BE:
@@ -59434,7 +59402,7 @@ static bool needCarryOrOverflowFlag(SDValue Flags) {
case X86::COND_G: case X86::COND_GE:
case X86::COND_L: case X86::COND_LE:
return true;
- // clang-format on
+ // clang-format on
}
}
@@ -59574,12 +59542,11 @@ static SDValue combineCMP(SDNode *N, SelectionDAG &DAG,
// After this the truncate and arithmetic op must have a single use.
if (!Trunc.hasOneUse() || !Op.hasOneUse())
- return SDValue();
+ return SDValue();
unsigned NewOpc;
switch (Op.getOpcode()) {
- default:
- return SDValue();
+ default: return SDValue();
case ISD::AND:
// Skip and with constant. We have special handling for and with immediate
// during isel to generate test instructions.
@@ -59587,12 +59554,8 @@ static SDValue combineCMP(SDNode *N, SelectionDAG &DAG,
return SDValue();
NewOpc = X86ISD::AND;
break;
- case ISD::OR:
- NewOpc = X86ISD::OR;
- break;
- case ISD::XOR:
- NewOpc = X86ISD::XOR;
- break;
+ case ISD::OR: NewOpc = X86ISD::OR; break;
+ case ISD::XOR: NewOpc = X86ISD::XOR; break;
case ISD::ADD:
// If the carry or overflow flag is used, we can't truncate.
if (needCarryOrOverflowFlag(SDValue(N, 0)))
@@ -59820,8 +59783,9 @@ static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
-static SDValue matchPMADDWD(SelectionDAG &DAG, SDNode *N, const SDLoc &DL,
- EVT VT, const X86Subtarget &Subtarget) {
+static SDValue matchPMADDWD(SelectionDAG &DAG, SDNode *N,
+ const SDLoc &DL, EVT VT,
+ const X86Subtarget &Subtarget) {
using namespace SDPatternMatch;
// Example of pattern we try to detect:
@@ -59966,8 +59930,9 @@ static SDValue matchPMADDWD(SelectionDAG &DAG, SDNode *N, const SDLoc &DL,
// Attempt to turn this pattern into PMADDWD.
// (add (mul (sext (build_vector)), (sext (build_vector))),
// (mul (sext (build_vector)), (sext (build_vector)))
-static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N, const SDLoc &DL,
- EVT VT, const X86Subtarget &Subtarget) {
+static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N,
+ const SDLoc &DL, EVT VT,
+ const X86Subtarget &Subtarget) {
using namespace SDPatternMatch;
if (!Subtarget.hasSSE2())
@@ -60062,7 +60027,7 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N, const SDLoc &DL,
// If the output is narrower than an input, extract the low part of the input
// vector.
EVT OutVT16 = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
- VT.getVectorNumElements() * 2);
+ VT.getVectorNumElements() * 2);
if (OutVT16.bitsLT(In0.getValueType())) {
In0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In0,
DAG.getVectorIdxConstant(0, DL));
@@ -60071,7 +60036,8 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N, const SDLoc &DL,
In1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In1,
DAG.getVectorIdxConstant(0, DL));
}
- return SplitOpsAndApply(DAG, Subtarget, DL, VT, {In0, In1}, PMADDBuilder);
+ return SplitOpsAndApply(DAG, Subtarget, DL, VT, { In0, In1 },
+ PMADDBuilder);
}
// ADD(VPMADDWD(X,Y),VPMADDWD(Z,W)) -> VPMADDWD(SHUFFLE(X,Z), SHUFFLE(Y,W))
@@ -61086,8 +61052,8 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
unsigned Imm1 = Ops[1].getConstantOperandVal(2);
// TODO: Handle zero'd subvectors.
if ((Imm0 & 0x88) == 0 && (Imm1 & 0x88) == 0) {
- int Mask[4] = {(int)(Imm0 & 0x03), (int)((Imm0 >> 4) & 0x3),
- (int)(Imm1 & 0x03), (int)((Imm1 >> 4) & 0x3)};
+ int Mask[4] = {(int)(Imm0 & 0x03), (int)((Imm0 >> 4) & 0x3), (int)(Imm1 & 0x03),
+ (int)((Imm1 >> 4) & 0x3)};
MVT ShuffleVT = VT.isFloatingPoint() ? MVT::v8f64 : MVT::v8i64;
SDValue LHS = concatSubVectors(Ops[0].getOperand(0),
Ops[0].getOperand(1), DAG, DL);
@@ -61359,7 +61325,8 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
break;
}
- ISD::CondCode ICC = Opcode == X86ISD::PCMPEQ ? ISD::SETEQ : ISD::SETGT;
+ ISD::CondCode ICC =
+ Opcode == X86ISD::PCMPEQ ? ISD::SETEQ : ISD::SETGT;
ISD::CondCode FCC =
Opcode == X86ISD::PCMPEQ ? ISD::SETOEQ : ISD::SETOGT;
@@ -61873,8 +61840,7 @@ static SDValue combineCONCAT_VECTORS(SDNode *N, SelectionDAG &DAG,
APInt Constant = APInt::getZero(VT.getSizeInBits());
for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
auto *C = dyn_cast<ConstantSDNode>(peekThroughBitcasts(Ops[I]));
- if (!C)
- break;
+ if (!C) break;
Constant.insertBits(C->getAPIntValue(), I * SubSizeInBits);
if (I == (E - 1)) {
EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
@@ -61961,9 +61927,9 @@ static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
ISD::isBuildVectorAllZeros(Ins.getOperand(0).getNode()) &&
Ins.getOperand(1).getValueSizeInBits().getFixedValue() <=
SubVecVT.getFixedSizeInBits())
- return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
- getZeroVector(OpVT, Subtarget, DAG, dl),
- Ins.getOperand(1), N->getOperand(2));
+ return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
+ getZeroVector(OpVT, Subtarget, DAG, dl),
+ Ins.getOperand(1), N->getOperand(2));
}
// See if were inserting into a zero vXi1 vector and the subvector was
@@ -62655,7 +62621,7 @@ static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
LHS.getOperand(0).getValueType() == MVT::v4i32) {
SDLoc dl(N);
LHS = DAG.getVectorShuffle(MVT::v4i32, dl, LHS.getOperand(0),
- LHS.getOperand(0), {0, -1, 1, -1});
+ LHS.getOperand(0), { 0, -1, 1, -1 });
LHS = DAG.getBitcast(MVT::v2i64, LHS);
return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
}
@@ -62665,7 +62631,7 @@ static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
RHS.getOperand(0).getValueType() == MVT::v4i32) {
SDLoc dl(N);
RHS = DAG.getVectorShuffle(MVT::v4i32, dl, RHS.getOperand(0),
- RHS.getOperand(0), {0, -1, 1, -1});
+ RHS.getOperand(0), { 0, -1, 1, -1 });
RHS = DAG.getBitcast(MVT::v2i64, RHS);
return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
}
@@ -63141,16 +63107,16 @@ static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG,
// Widen to at least 8 input elements.
if (NumElts < 8) {
unsigned NumConcats = 8 / NumElts;
- SDValue Fill =
- NumElts == 4 ? DAG.getUNDEF(IntVT) : DAG.getConstant(0, dl, IntVT);
+ SDValue Fill = NumElts == 4 ? DAG.getUNDEF(IntVT)
+ : DAG.getConstant(0, dl, IntVT);
SmallVector<SDValue, 4> Ops(NumConcats, Fill);
Ops[0] = Src;
Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, Ops);
}
// Destination is vXf32 with at least 4 elements.
- EVT CvtVT =
- EVT::getVectorVT(*DAG.getContext(), MVT::f32, std::max(4U, NumElts));
+ EVT CvtVT = EVT::getVectorVT(*DAG.getContext(), MVT::f32,
+ std::max(4U, NumElts));
SDValue Cvt, Chain;
if (IsStrict) {
Cvt = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {CvtVT, MVT::Other},
@@ -63472,7 +63438,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
switch (N->getOpcode()) {
- // clang-format off
+ // clang-format off
default: break;
case ISD::SCALAR_TO_VECTOR:
return combineSCALAR_TO_VECTOR(N, DAG, Subtarget);
@@ -63853,8 +63819,7 @@ bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
bool Commute = false;
switch (Op.getOpcode()) {
- default:
- return false;
+ default: return false;
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
case ISD::ANY_EXTEND:
@@ -63894,7 +63859,8 @@ bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
((Commute && !isa<ConstantSDNode>(N1)) ||
(Op.getOpcode() != ISD::MUL && IsFoldableRMW(N0, Op))))
return false;
- if (IsFoldableAtomicRMW(N0, Op) || (Commute && IsFoldableAtomicRMW(N1, Op)))
+ if (IsFoldableAtomicRMW(N0, Op) ||
+ (Commute && IsFoldableAtomicRMW(N1, Op)))
return false;
}
}
@@ -63981,7 +63947,8 @@ X86TargetLowering::getConstraintType(StringRef Constraint) const {
default:
break;
}
- } else if (Constraint.size() == 2) {
+ }
+ else if (Constraint.size() == 2) {
switch (Constraint[0]) {
default:
break;
@@ -64170,7 +64137,8 @@ X86TargetLowering::getSingleConstraintMatchWeight(
/// Try to replace an X constraint, which matches anything, with another that
/// has more specific requirements based on the type of the corresponding
/// operand.
-const char *X86TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
+const char *X86TargetLowering::
+LowerXConstraint(EVT ConstraintVT) const {
// FP X constraints get lowered to SSE1/2 registers if available, otherwise
// 'f' like normal targets.
if (ConstraintVT.isFloatingPoint()) {
@@ -64216,8 +64184,7 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
SDValue Result;
char ConstraintLetter = Constraint[0];
switch (ConstraintLetter) {
- default:
- break;
+ default: break;
case 'I':
if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
if (C->getZExtValue() <= 31) {
@@ -64292,8 +64259,8 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
DAG.getSignedTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64);
break;
}
- // FIXME gcc accepts some relocatable values here too, but only in certain
- // memory models; it's complicated.
+ // FIXME gcc accepts some relocatable values here too, but only in certain
+ // memory models; it's complicated.
}
return;
}
@@ -64336,8 +64303,8 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
if (auto *CST = dyn_cast<ConstantSDNode>(Op)) {
bool IsBool = CST->getConstantIntValue()->getBitWidth() == 1;
BooleanContent BCont = getBooleanContents(MVT::i64);
- ISD::NodeType ExtOpc =
- IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;
+ ISD::NodeType ExtOpc = IsBool ? getExtendForContent(BCont)
+ : ISD::SIGN_EXTEND;
SDLoc DL(Op);
Result =
ExtOpc == ISD::ZERO_EXTEND
@@ -64418,8 +64385,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
if (Constraint.size() == 1) {
// GCC Constraint Letters
switch (Constraint[0]) {
- default:
- break;
+ default: break;
// 'A' means [ER]AX + [ER]DX.
case 'A':
if (Subtarget.is64Bit())
@@ -64447,7 +64413,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
return std::make_pair(0U, &X86::VK64RegClass);
}
break;
- case 'q': // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode.
+ case 'q': // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode.
if (Subtarget.is64Bit()) {
if (VT == MVT::i8 || VT == MVT::i1)
return std::make_pair(0U, useEGPRInlineAsm(Subtarget)
@@ -64469,7 +64435,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
}
[[fallthrough]];
// 32-bit fallthrough
- case 'Q': // Q_REGS
+ case 'Q': // Q_REGS
if (VT == MVT::i8 || VT == MVT::i1)
return std::make_pair(0U, &X86::GR8_ABCD_LRegClass);
if (VT == MVT::i16)
@@ -64480,8 +64446,8 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
if (VT != MVT::f80 && !VT.isVector() && VT != MVT::Other)
return std::make_pair(0U, &X86::GR64_ABCDRegClass);
break;
- case 'r': // GENERAL_REGS
- case 'l': // INDEX_REGS
+ case 'r': // GENERAL_REGS
+ case 'l': // INDEX_REGS
if (VT == MVT::i8 || VT == MVT::i1)
return std::make_pair(0U, useEGPRInlineAsm(Subtarget)
? &X86::GR8RegClass
@@ -64500,7 +64466,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
? &X86::GR64RegClass
: &X86::GR64_NOREX2RegClass);
break;
- case 'R': // LEGACY_REGS
+ case 'R': // LEGACY_REGS
if (VT == MVT::i8 || VT == MVT::i1)
return std::make_pair(0U, &X86::GR8_NOREXRegClass);
if (VT == MVT::i16)
@@ -64511,7 +64477,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
if (VT != MVT::f80 && !VT.isVector() && VT != MVT::Other)
return std::make_pair(0U, &X86::GR64_NOREXRegClass);
break;
- case 'f': // FP Stack registers.
+ case 'f': // FP Stack registers.
// If SSE is enabled for this VT, use f80 to ensure the isel moves the
// value to the correct fpstack register class.
if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT))
@@ -64521,19 +64487,16 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80)
return std::make_pair(0U, &X86::RFP80RegClass);
break;
- case 'y': // MMX_REGS if MMX allowed.
- if (!Subtarget.hasMMX())
- break;
+ case 'y': // MMX_REGS if MMX allowed.
+ if (!Subtarget.hasMMX()) break;
return std::make_pair(0U, &X86::VR64RegClass);
case 'v':
- case 'x': // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed
- if (!Subtarget.hasSSE1())
- break;
+ case 'x': // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed
+ if (!Subtarget.hasSSE1()) break;
bool VConstraint = (Constraint[0] == 'v');
switch (VT.SimpleTy) {
- default:
- break;
+ default: break;
// Scalar SSE types.
case MVT::f16:
if (VConstraint && Subtarget.hasFP16())
@@ -64624,8 +64587,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
case MVT::v16f32:
case MVT::v16i32:
case MVT::v8i64:
- if (!Subtarget.hasAVX512())
- break;
+ if (!Subtarget.hasAVX512()) break;
if (VConstraint)
return std::make_pair(0U, &X86::VR512RegClass);
return std::make_pair(0U, &X86::VR512_0_15RegClass);
@@ -64641,15 +64603,12 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
case '2':
return getRegForInlineAsmConstraint(TRI, "x", VT);
case 'm':
- if (!Subtarget.hasMMX())
- break;
+ if (!Subtarget.hasMMX()) break;
return std::make_pair(0U, &X86::VR64RegClass);
case 'z':
- if (!Subtarget.hasSSE1())
- break;
+ if (!Subtarget.hasSSE1()) break;
switch (VT.SimpleTy) {
- default:
- break;
+ default: break;
// Scalar SSE types.
case MVT::f16:
if (!Subtarget.hasFP16())
@@ -64764,15 +64723,14 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
// Use the default implementation in TargetLowering to convert the register
// constraint into a member of a register class.
- std::pair<Register, const TargetRegisterClass *> Res;
+ std::pair<Register, const TargetRegisterClass*> Res;
Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
// Not found as a standard register?
if (!Res.second) {
// Only match x87 registers if the VT is one SelectionDAGBuilder can convert
// to/from f80.
- if (VT == MVT::Other || VT == MVT::f32 || VT == MVT::f64 ||
- VT == MVT::f80) {
+ if (VT == MVT::Other || VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80) {
// Map st(0) -> st(7) -> ST0
if (Constraint.size() == 7 && Constraint[0] == '{' &&
tolower(Constraint[1]) == 's' && tolower(Constraint[2]) == 't' &&
@@ -64830,7 +64788,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
// turn into {ax},{dx}.
// MVT::Other is used to specify clobber names.
if (TRI->isTypeLegalForClass(*Res.second, VT) || VT == MVT::Other)
- return Res; // Correct type already, nothing to do.
+ return Res; // Correct type already, nothing to do.
// Get a matching integer of the correct size. i.e. "ax" with MVT::32 should
// return "eax". This should even work for things like getting 64bit integer
@@ -64842,8 +64800,7 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
// Therefore, use a helper method.
if (isGRClass(*Class)) {
unsigned Size = VT.getSizeInBits();
- if (Size == 1)
- Size = 8;
+ if (Size == 1) Size = 8;
if (Size != 8 && Size != 16 && Size != 32 && Size != 64)
return std::make_pair(0, nullptr);
Register DestReg = getX86SubSuperRegister(Res.first, Size);
@@ -64851,11 +64808,9 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
bool is64Bit = Subtarget.is64Bit();
const TargetRegisterClass *RC =
Size == 8 ? (is64Bit ? &X86::GR8RegClass : &X86::GR8_NOREXRegClass)
- : Size == 16
- ? (is64Bit ? &X86::GR16RegClass : &X86::GR16_NOREXRegClass)
- : Size == 32
- ? (is64Bit ? &X86::GR32RegClass : &X86::GR32_NOREXRegClass)
- : /*Size == 64*/ (is64Bit ? &X86::GR64RegClass : nullptr);
+ : Size == 16 ? (is64Bit ? &X86::GR16RegClass : &X86::GR16_NOREXRegClass)
+ : Size == 32 ? (is64Bit ? &X86::GR32RegClass : &X86::GR32_NOREXRegClass)
+ : /*Size == 64*/ (is64Bit ? &X86::GR64RegClass : nullptr);
if (Size == 64 && !is64Bit) {
// Model GCC's behavior here and select a fixed pair of 32-bit
// registers.
@@ -65107,7 +65062,8 @@ X86TargetLowering::getStackProbeSymbolName(const MachineFunction &MF) const {
return getLibcallImplName(StackProbeImpl);
}
-unsigned X86TargetLowering::getStackProbeSize(const MachineFunction &MF) const {
+unsigned
+X86TargetLowering::getStackProbeSize(const MachineFunction &MF) const {
// The default stack probe size is 4096 if the function has no stackprobesize
// attribute.
return MF.getFunction().getFnAttributeAsParsedInteger("stack-probe-size",
More information about the llvm-commits
mailing list