[llvm] [X86] Fix stack temporary leaks for x87 float returns (PR #213134)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 1 12:24:11 PDT 2026
https://github.com/AZero13 updated https://github.com/llvm/llvm-project/pull/213134
>From 22ddb78577c58519c2a6ae27e81cf119490b86e2 Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Thu, 30 Jul 2026 16:09:08 -0400
Subject: [PATCH 1/2] Add pre-commit test for x87 stack leak
[X86] Add pre-commit test for x87 float temporary stack leak
---
llvm/test/CodeGen/X86/stack-coloring-x87.ll | 98 ++++++++++++++++++++
llvm/test/CodeGen/X86/x87-stack-coloring.ll | 99 +++++++++++++++++++++
2 files changed, 197 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/stack-coloring-x87.ll
create mode 100644 llvm/test/CodeGen/X86/x87-stack-coloring.ll
diff --git a/llvm/test/CodeGen/X86/stack-coloring-x87.ll b/llvm/test/CodeGen/X86/stack-coloring-x87.ll
new file mode 100644
index 0000000000000..8990b810bd590
--- /dev/null
+++ b/llvm/test/CodeGen/X86/stack-coloring-x87.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc -O2 | FileCheck %s
+
+%struct.ByValStruct = type { [4 x i64] }
+declare void @use_byval(ptr byval(%struct.ByValStruct) %a)
+
+define void @test_byval(ptr %p1, ptr %p2, ptr %p3) {
+; CHECK-LABEL: test_byval:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: pushq %rsi
+; CHECK-NEXT: .seh_pushreg %rsi
+; CHECK-NEXT: pushq %rdi
+; CHECK-NEXT: .seh_pushreg %rdi
+; CHECK-NEXT: subq $136, %rsp
+; CHECK-NEXT: .seh_stackalloc 136
+; CHECK-NEXT: .seh_endprologue
+; CHECK-NEXT: movq %r8, %rsi
+; CHECK-NEXT: movq %rdx, %rdi
+; CHECK-NEXT: movq 24(%rcx), %rax
+; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq 16(%rcx), %rax
+; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq (%rcx), %rax
+; CHECK-NEXT: movq 8(%rcx), %rcx
+; CHECK-NEXT: movq %rcx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT: callq use_byval
+; CHECK-NEXT: movq 24(%rdi), %rax
+; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq 16(%rdi), %rax
+; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq (%rdi), %rax
+; CHECK-NEXT: movq 8(%rdi), %rcx
+; CHECK-NEXT: movq %rcx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT: callq use_byval
+; CHECK-NEXT: movq 24(%rsi), %rax
+; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq 16(%rsi), %rax
+; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq (%rsi), %rax
+; CHECK-NEXT: movq 8(%rsi), %rcx
+; CHECK-NEXT: movq %rcx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT: callq use_byval
+; CHECK-NEXT: nop
+; CHECK-NEXT: .seh_startepilogue
+; CHECK-NEXT: addq $136, %rsp
+; CHECK-NEXT: popq %rdi
+; CHECK-NEXT: popq %rsi
+; CHECK-NEXT: .seh_endepilogue
+; CHECK-NEXT: retq
+; CHECK-NEXT: .seh_endproc
+entry:
+ call void @use_byval(ptr byval(%struct.ByValStruct) %p1)
+ call void @use_byval(ptr byval(%struct.ByValStruct) %p2)
+ call void @use_byval(ptr byval(%struct.ByValStruct) %p3)
+ ret void
+}
+
+declare void @use_f80(x86_fp80)
+define void @test_f80(x86_fp80 %p1, x86_fp80 %p2, x86_fp80 %p3) {
+; CHECK-LABEL: test_f80:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: subq $104, %rsp
+; CHECK-NEXT: .seh_stackalloc 104
+; CHECK-NEXT: .seh_endprologue
+; CHECK-NEXT: fldt (%r8)
+; CHECK-NEXT: fstpt {{[-0-9]+}}(%r{{[sb]}}p) # 10-byte Folded Spill
+; CHECK-NEXT: fldt (%rdx)
+; CHECK-NEXT: fstpt {{[-0-9]+}}(%r{{[sb]}}p) # 10-byte Folded Spill
+; CHECK-NEXT: fldt (%rcx)
+; CHECK-NEXT: fstpt {{[0-9]+}}(%rsp)
+; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT: callq use_f80
+; CHECK-NEXT: fldt {{[-0-9]+}}(%r{{[sb]}}p) # 10-byte Folded Reload
+; CHECK-NEXT: fstpt {{[0-9]+}}(%rsp)
+; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT: callq use_f80
+; CHECK-NEXT: fldt {{[-0-9]+}}(%r{{[sb]}}p) # 10-byte Folded Reload
+; CHECK-NEXT: fstpt {{[0-9]+}}(%rsp)
+; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT: callq use_f80
+; CHECK-NEXT: nop
+; CHECK-NEXT: .seh_startepilogue
+; CHECK-NEXT: addq $104, %rsp
+; CHECK-NEXT: .seh_endepilogue
+; CHECK-NEXT: retq
+; CHECK-NEXT: .seh_endproc
+entry:
+ call void @use_f80(x86_fp80 %p1)
+ call void @use_f80(x86_fp80 %p2)
+ call void @use_f80(x86_fp80 %p3)
+ ret void
+}
diff --git a/llvm/test/CodeGen/X86/x87-stack-coloring.ll b/llvm/test/CodeGen/X86/x87-stack-coloring.ll
new file mode 100644
index 0000000000000..eeca372cf3ac5
--- /dev/null
+++ b/llvm/test/CodeGen/X86/x87-stack-coloring.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+sse2,+x87 | FileCheck %s
+
+; ModuleID = 'example_big.c'
+source_filename = "example_big.c"
+target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:128-f64:32:64-f80:32-n8:16:32-S128"
+target triple = "i386-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local float @returns_float() local_unnamed_addr #0 {
+; CHECK-LABEL: returns_float:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: subl $44, %esp
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: calll rand at PLT
+; CHECK-NEXT: cvtsi2ss %eax, %xmm0
+; CHECK-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: addl $12, %esp
+; CHECK-NEXT: .cfi_def_cfa_offset 4
+; CHECK-NEXT: retl
+entry:
+ %call = tail call i32 @rand() #3
+ %conv = sitofp i32 %call to float
+ ret float %conv
+}
+
+declare i32 @rand() local_unnamed_addr #1
+
+; Function Attrs: nounwind uwtable
+define dso_local float @example() local_unnamed_addr #2 {
+; CHECK-LABEL: example:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: subl $44, %esp
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: calll returns_float
+; CHECK-NEXT: fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: movss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: calll returns_float
+; CHECK-NEXT: fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT: movss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: addss {{[0-9]+}}(%esp), %xmm0
+; CHECK-NEXT: movss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: calll returns_float
+; CHECK-NEXT: fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT: movss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: addss {{[0-9]+}}(%esp), %xmm0
+; CHECK-NEXT: movss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: calll returns_float
+; CHECK-NEXT: fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT: movss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: addss {{[0-9]+}}(%esp), %xmm0
+; CHECK-NEXT: movss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: calll returns_float
+; CHECK-NEXT: fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT: movss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: addss {{[0-9]+}}(%esp), %xmm0
+; CHECK-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: addl $12, %esp
+; CHECK-NEXT: .cfi_def_cfa_offset 4
+; CHECK-NEXT: retl
+entry:
+ %call = tail call float @returns_float()
+ %call1 = tail call float @returns_float()
+ %add = fadd float %call, %call1
+ %call2 = tail call float @returns_float()
+ %add3 = fadd float %add, %call2
+ %call4 = tail call float @returns_float()
+ %add5 = fadd float %add3, %call4
+ %call6 = tail call float @returns_float()
+ %add7 = fadd float %add5, %call6
+ ret float %add7
+}
+
+attributes #0 = { noinline nounwind uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
+attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
+attributes #2 = { nounwind uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0, !1, !2, !3}
+!llvm.ident = !{!4}
+!llvm.errno.tbaa = !{!5}
+
+!0 = !{i32 1, !"NumRegisterParameters", i32 0}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 24.0.0git (https://github.com/AZero13/llvm-project.git 1b9e4c34a31ca71c2016b58ee23ee2729e3a58c3)"}
+!5 = !{!6, !7, i64 0}
+!6 = !{!"__libc_errno", !7, i64 0}
+!7 = !{!"int", !8, i64 0}
+!8 = !{!"omnipotent char", !9, i64 0}
+!9 = !{!"Simple C/C++ TBAA"}
>From ef8d01b1b0af77dc9248a671ac6ff50ab7da3a07 Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Thu, 30 Jul 2026 16:18:42 -0400
Subject: [PATCH 2/2] [X86] Fix stack temporary leaks for x87 float returns
When passing arguments indirectly (like byval structs or x87 floats), LLVM creates temporary stack slots in the backend.
Right now, these temporary stack slots leak memory across function calls.
To fix:
Add LIFETIME_START / LIFETIME_END markers around temporary argument slots in LowerCall.
Allow StackColoring to safely merge stack slots even if they don't have a matching IR AllocaInst.
Parallelize indirect argument setup in LowerCall using MemOpChains so the instruction scheduler isn't artificially constrained.
---
llvm/lib/CodeGen/StackColoring.cpp | 81 +-
llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 1916 +++++++++--------
llvm/test/CodeGen/X86/avx512-regcall-Mask.ll | 18 +-
llvm/test/CodeGen/X86/dag-stlf-mismatch.ll | 3 +-
.../test/CodeGen/X86/fp128-libcalls-strict.ll | 6 +-
llvm/test/CodeGen/X86/fp128-libcalls.ll | 16 +-
llvm/test/CodeGen/X86/i128-fp128-abi.ll | 24 +-
llvm/test/CodeGen/X86/sse-intel-ocl.ll | 20 +-
llvm/test/CodeGen/X86/stack-coloring-x87.ll | 12 +-
.../X86/vector-constrained-fp-intrinsics.ll | 164 +-
llvm/test/CodeGen/X86/win64-byval.ll | 6 +-
llvm/test/CodeGen/X86/x87-stack-coloring.ll | 74 +-
12 files changed, 1263 insertions(+), 1077 deletions(-)
diff --git a/llvm/lib/CodeGen/StackColoring.cpp b/llvm/lib/CodeGen/StackColoring.cpp
index ea1f33f791765..0352f4f0b220f 100644
--- a/llvm/lib/CodeGen/StackColoring.cpp
+++ b/llvm/lib/CodeGen/StackColoring.cpp
@@ -928,31 +928,6 @@ void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
for (const std::pair<int, int> &SI : SlotRemap) {
const AllocaInst *From = MFI->getObjectAllocation(SI.first);
const AllocaInst *To = MFI->getObjectAllocation(SI.second);
- assert(To && From && "Invalid allocation object");
- Allocas[From] = To;
-
- // If From is before wo, its possible that there is a use of From between
- // them.
- if (From->comesBefore(To))
- const_cast<AllocaInst *>(To)->moveBefore(
- const_cast<AllocaInst *>(From)->getIterator());
-
- // AA might be used later for instruction scheduling, and we need it to be
- // able to deduce the correct aliasing releationships between pointers
- // derived from the alloca being remapped and the target of that remapping.
- // The only safe way, without directly informing AA about the remapping
- // somehow, is to directly update the IR to reflect the change being made
- // here.
- Instruction *Inst = const_cast<AllocaInst *>(To);
- if (From->getType() != To->getType()) {
- BitCastInst *Cast = new BitCastInst(Inst, From->getType());
- Cast->insertAfter(Inst->getIterator());
- Inst = Cast;
- }
-
- // We keep both slots to maintain AliasAnalysis metadata later.
- MergedAllocas.insert(From);
- MergedAllocas.insert(To);
// Transfer the stack protector layout tag, but make sure that SSPLK_AddrOf
// does not overwrite SSPLK_SmallArray or SSPLK_LargeArray, and make sure
@@ -966,21 +941,49 @@ void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
FromKind != MachineFrameInfo::SSPLK_AddrOf)))
MFI->setObjectSSPLayout(SI.second, FromKind);
- // The new alloca might not be valid in a llvm.dbg.declare for this
- // variable, so poison out the use to make the verifier happy.
- AllocaInst *FromAI = const_cast<AllocaInst *>(From);
- if (FromAI->isUsedByMetadata())
- ValueAsMetadata::handleRAUW(FromAI, PoisonValue::get(FromAI->getType()));
- for (auto &Use : FromAI->uses()) {
- if (BitCastInst *BCI = dyn_cast<BitCastInst>(Use.get()))
- if (BCI->isUsedByMetadata())
- ValueAsMetadata::handleRAUW(BCI, PoisonValue::get(BCI->getType()));
- }
+ if (From && To) {
+ Allocas[From] = To;
+
+ // If From is before wo, its possible that there is a use of From between
+ // them.
+ if (From->comesBefore(To))
+ const_cast<AllocaInst *>(To)->moveBefore(
+ const_cast<AllocaInst *>(From)->getIterator());
+
+ // AA might be used later for instruction scheduling, and we need it to be
+ // able to deduce the correct aliasing releationships between pointers
+ // derived from the alloca being remapped and the target of that
+ // remapping. The only safe way, without directly informing AA about the
+ // remapping somehow, is to directly update the IR to reflect the change
+ // being made here.
+ Instruction *Inst = const_cast<AllocaInst *>(To);
+ if (From->getType() != To->getType()) {
+ BitCastInst *Cast = new BitCastInst(Inst, From->getType());
+ Cast->insertAfter(Inst->getIterator());
+ Inst = Cast;
+ }
- // Note that this will not replace uses in MMOs (which we'll update below),
- // or anywhere else (which is why we won't delete the original
- // instruction).
- FromAI->replaceAllUsesWith(Inst);
+ // We keep both slots to maintain AliasAnalysis metadata later.
+ MergedAllocas.insert(From);
+ MergedAllocas.insert(To);
+
+ // The new alloca might not be valid in a llvm.dbg.declare for this
+ // variable, so poison out the use to make the verifier happy.
+ AllocaInst *FromAI = const_cast<AllocaInst *>(From);
+ if (FromAI->isUsedByMetadata())
+ ValueAsMetadata::handleRAUW(FromAI,
+ PoisonValue::get(FromAI->getType()));
+ for (auto &Use : FromAI->uses()) {
+ if (BitCastInst *BCI = dyn_cast<BitCastInst>(Use.get()))
+ if (BCI->isUsedByMetadata())
+ ValueAsMetadata::handleRAUW(BCI, PoisonValue::get(BCI->getType()));
+ }
+
+ // Note that this will not replace uses in MMOs (which we'll update
+ // below), or anywhere else (which is why we won't delete the original
+ // instruction).
+ FromAI->replaceAllUsesWith(Inst);
+ }
}
// Remap all instructions to the new stack slots.
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index d078117061677..39351a06d6de4 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -39,7 +39,8 @@ using namespace llvm;
STATISTIC(NumLoadMoved, "Number of loads moved below TokenFactor");
-static cl::opt<bool> AndImmShrink("x86-and-imm-shrink", cl::init(true),
+static cl::opt<bool> AndImmShrink(
+ "x86-and-imm-shrink", cl::init(true),
cl::desc("Enable setting constant bits to reduce size of mask immediates"),
cl::Hidden);
@@ -54,575 +55,556 @@ extern cl::opt<bool> IndirectBranchTracking;
//===----------------------------------------------------------------------===//
namespace {
- /// This corresponds to X86AddressMode, but uses SDValue's instead of register
- /// numbers for the leaves of the matched tree.
- struct X86ISelAddressMode {
- enum {
- RegBase,
- FrameIndexBase
- } BaseType = RegBase;
-
- // This is really a union, discriminated by BaseType!
- SDValue Base_Reg;
- int Base_FrameIndex = 0;
-
- unsigned Scale = 1;
- SDValue IndexReg;
- int32_t Disp = 0;
- SDValue Segment;
- const GlobalValue *GV = nullptr;
- const Constant *CP = nullptr;
- const BlockAddress *BlockAddr = nullptr;
- const char *ES = nullptr;
- MCSymbol *MCSym = nullptr;
- int JT = -1;
- Align Alignment; // CP alignment.
- unsigned char SymbolFlags = X86II::MO_NO_FLAG; // X86II::MO_*
- bool NegateIndex = false;
- // True when this address is being matched to be emitted as a LEA rather
- // than folded into a memory operand. Unlike a memory operand, a LEA turns
- // the folded arithmetic into real instructions, so it is not profitable to
- // split an already-materialized (multi-use) value here. (Issue #51707)
- bool IsForLEA = false;
-
- X86ISelAddressMode() = default;
-
- bool hasSymbolicDisplacement() const {
- return GV != nullptr || CP != nullptr || ES != nullptr ||
- MCSym != nullptr || JT != -1 || BlockAddr != nullptr;
- }
+/// This corresponds to X86AddressMode, but uses SDValue's instead of register
+/// numbers for the leaves of the matched tree.
+struct X86ISelAddressMode {
+ enum { RegBase, FrameIndexBase } BaseType = RegBase;
+
+ // This is really a union, discriminated by BaseType!
+ SDValue Base_Reg;
+ int Base_FrameIndex = 0;
+
+ unsigned Scale = 1;
+ SDValue IndexReg;
+ int32_t Disp = 0;
+ SDValue Segment;
+ const GlobalValue *GV = nullptr;
+ const Constant *CP = nullptr;
+ const BlockAddress *BlockAddr = nullptr;
+ const char *ES = nullptr;
+ MCSymbol *MCSym = nullptr;
+ int JT = -1;
+ Align Alignment; // CP alignment.
+ unsigned char SymbolFlags = X86II::MO_NO_FLAG; // X86II::MO_*
+ bool NegateIndex = false;
+ // True when this address is being matched to be emitted as a LEA rather
+ // than folded into a memory operand. Unlike a memory operand, a LEA turns
+ // the folded arithmetic into real instructions, so it is not profitable to
+ // split an already-materialized (multi-use) value here. (Issue #51707)
+ bool IsForLEA = false;
+
+ X86ISelAddressMode() = default;
+
+ bool hasSymbolicDisplacement() const {
+ return GV != nullptr || CP != nullptr || ES != nullptr ||
+ MCSym != nullptr || JT != -1 || BlockAddr != nullptr;
+ }
- bool hasBaseOrIndexReg() const {
- return BaseType == FrameIndexBase ||
- IndexReg.getNode() != nullptr || Base_Reg.getNode() != nullptr;
- }
+ bool hasBaseOrIndexReg() const {
+ return BaseType == FrameIndexBase || IndexReg.getNode() != nullptr ||
+ Base_Reg.getNode() != nullptr;
+ }
- /// Return true if this addressing mode is already RIP-relative.
- bool isRIPRelative() const {
- if (BaseType != RegBase) return false;
- if (RegisterSDNode *RegNode =
- dyn_cast_or_null<RegisterSDNode>(Base_Reg.getNode()))
- return RegNode->getReg() == X86::RIP;
+ /// Return true if this addressing mode is already RIP-relative.
+ bool isRIPRelative() const {
+ if (BaseType != RegBase)
return false;
- }
+ if (RegisterSDNode *RegNode =
+ dyn_cast_or_null<RegisterSDNode>(Base_Reg.getNode()))
+ return RegNode->getReg() == X86::RIP;
+ return false;
+ }
- void setBaseReg(SDValue Reg) {
- BaseType = RegBase;
- Base_Reg = Reg;
- }
+ void setBaseReg(SDValue Reg) {
+ BaseType = RegBase;
+ Base_Reg = Reg;
+ }
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- void dump(SelectionDAG *DAG = nullptr) {
- dbgs() << "X86ISelAddressMode " << this << '\n';
- dbgs() << "Base_Reg ";
- if (Base_Reg.getNode())
- Base_Reg.getNode()->dump(DAG);
- else
- dbgs() << "nul\n";
- if (BaseType == FrameIndexBase)
- dbgs() << " Base.FrameIndex " << Base_FrameIndex << '\n';
- dbgs() << " Scale " << Scale << '\n'
- << "IndexReg ";
- if (NegateIndex)
- dbgs() << "negate ";
- if (IndexReg.getNode())
- IndexReg.getNode()->dump(DAG);
- else
- dbgs() << "nul\n";
- dbgs() << " Disp " << Disp << '\n'
- << "GV ";
- if (GV)
- GV->dump();
- else
- dbgs() << "nul";
- dbgs() << " CP ";
- if (CP)
- CP->dump();
- else
- dbgs() << "nul";
- dbgs() << '\n'
- << "ES ";
- if (ES)
- dbgs() << ES;
- else
- dbgs() << "nul";
- dbgs() << " MCSym ";
- if (MCSym)
- dbgs() << MCSym;
- else
- dbgs() << "nul";
- dbgs() << " JT" << JT << " Align" << Alignment.value() << '\n';
- }
+ void dump(SelectionDAG *DAG = nullptr) {
+ dbgs() << "X86ISelAddressMode " << this << '\n';
+ dbgs() << "Base_Reg ";
+ if (Base_Reg.getNode())
+ Base_Reg.getNode()->dump(DAG);
+ else
+ dbgs() << "nul\n";
+ if (BaseType == FrameIndexBase)
+ dbgs() << " Base.FrameIndex " << Base_FrameIndex << '\n';
+ dbgs() << " Scale " << Scale << '\n' << "IndexReg ";
+ if (NegateIndex)
+ dbgs() << "negate ";
+ if (IndexReg.getNode())
+ IndexReg.getNode()->dump(DAG);
+ else
+ dbgs() << "nul\n";
+ dbgs() << " Disp " << Disp << '\n' << "GV ";
+ if (GV)
+ GV->dump();
+ else
+ dbgs() << "nul";
+ dbgs() << " CP ";
+ if (CP)
+ CP->dump();
+ else
+ dbgs() << "nul";
+ dbgs() << '\n' << "ES ";
+ if (ES)
+ dbgs() << ES;
+ else
+ dbgs() << "nul";
+ dbgs() << " MCSym ";
+ if (MCSym)
+ dbgs() << MCSym;
+ else
+ dbgs() << "nul";
+ dbgs() << " JT" << JT << " Align" << Alignment.value() << '\n';
+ }
#endif
- };
-}
+};
+} // namespace
namespace {
- //===--------------------------------------------------------------------===//
- /// ISel - X86-specific code to select X86 machine instructions for
- /// SelectionDAG operations.
- ///
- class X86DAGToDAGISel final : public SelectionDAGISel {
- /// Keep a pointer to the X86Subtarget around so that we can
- /// make the right decision when generating code for different targets.
- const X86Subtarget *Subtarget;
-
- /// If true, selector should try to optimize for minimum code size.
- bool OptForMinSize;
-
- /// Disable direct TLS access through segment registers.
- bool IndirectTlsSegRefs;
-
- public:
- X86DAGToDAGISel() = delete;
-
- explicit X86DAGToDAGISel(X86TargetMachine &tm, CodeGenOptLevel OptLevel)
- : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr),
- OptForMinSize(false), IndirectTlsSegRefs(false) {}
-
- bool runOnMachineFunction(MachineFunction &MF) override {
- // Reset the subtarget each time through.
- Subtarget = &MF.getSubtarget<X86Subtarget>();
- IndirectTlsSegRefs = MF.getFunction().hasFnAttribute(
- "indirect-tls-seg-refs");
-
- // OptFor[Min]Size are used in pattern predicates that isel is matching.
- OptForMinSize = MF.getFunction().hasMinSize();
- return SelectionDAGISel::runOnMachineFunction(MF);
- }
+//===--------------------------------------------------------------------===//
+/// ISel - X86-specific code to select X86 machine instructions for
+/// SelectionDAG operations.
+///
+class X86DAGToDAGISel final : public SelectionDAGISel {
+ /// Keep a pointer to the X86Subtarget around so that we can
+ /// make the right decision when generating code for different targets.
+ const X86Subtarget *Subtarget;
+
+ /// If true, selector should try to optimize for minimum code size.
+ bool OptForMinSize;
+
+ /// Disable direct TLS access through segment registers.
+ bool IndirectTlsSegRefs;
+
+public:
+ X86DAGToDAGISel() = delete;
+
+ explicit X86DAGToDAGISel(X86TargetMachine &tm, CodeGenOptLevel OptLevel)
+ : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr),
+ OptForMinSize(false), IndirectTlsSegRefs(false) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ // Reset the subtarget each time through.
+ Subtarget = &MF.getSubtarget<X86Subtarget>();
+ IndirectTlsSegRefs =
+ MF.getFunction().hasFnAttribute("indirect-tls-seg-refs");
+
+ // OptFor[Min]Size are used in pattern predicates that isel is matching.
+ OptForMinSize = MF.getFunction().hasMinSize();
+ return SelectionDAGISel::runOnMachineFunction(MF);
+ }
- void emitFunctionEntryCode() override;
+ void emitFunctionEntryCode() override;
- bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
+ bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
- void PreprocessISelDAG() override;
- void PostprocessISelDAG() override;
+ void PreprocessISelDAG() override;
+ void PostprocessISelDAG() override;
// Include the pieces autogenerated from the target description.
#include "X86GenDAGISel.inc"
- private:
- void Select(SDNode *N) override;
-
- bool foldOffsetIntoAddress(uint64_t Offset, X86ISelAddressMode &AM);
- bool matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM,
- bool AllowSegmentRegForX32 = false);
- bool matchWrapper(SDValue N, X86ISelAddressMode &AM);
- bool matchAddress(SDValue N, X86ISelAddressMode &AM);
- bool matchVectorAddress(SDValue N, X86ISelAddressMode &AM);
- bool matchAdd(SDValue &N, X86ISelAddressMode &AM, unsigned Depth);
- bool hasMaterializingUse(SDValue V) const;
- SDValue matchIndexRecursively(SDValue N, X86ISelAddressMode &AM,
- unsigned Depth);
- bool matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
- unsigned Depth);
- bool matchVectorAddressRecursively(SDValue N, X86ISelAddressMode &AM,
- unsigned Depth);
- bool matchAddressBase(SDValue N, X86ISelAddressMode &AM);
- bool selectAddr(SDNode *Parent, SDValue N, SDValue &Base, SDValue &Scale,
- SDValue &Index, SDValue &Disp, SDValue &Segment,
- bool HasNDDM = true);
- bool selectNDDAddr(SDNode *Parent, SDValue N, SDValue &Base, SDValue &Scale,
- SDValue &Index, SDValue &Disp, SDValue &Segment);
- bool selectVectorAddr(MemSDNode *Parent, SDValue BasePtr, SDValue IndexOp,
- SDValue ScaleOp, SDValue &Base, SDValue &Scale,
- SDValue &Index, SDValue &Disp, SDValue &Segment);
- bool selectMOV64Imm32(SDValue N, SDValue &Imm);
- bool selectLEAAddr(SDValue N, SDValue &Base,
- SDValue &Scale, SDValue &Index, SDValue &Disp,
- SDValue &Segment);
- bool selectLEA64_Addr(SDValue N, SDValue &Base, SDValue &Scale,
- SDValue &Index, SDValue &Disp, SDValue &Segment);
- bool selectTLSADDRAddr(SDValue N, SDValue &Base,
- SDValue &Scale, SDValue &Index, SDValue &Disp,
- SDValue &Segment);
- bool selectRelocImm(SDValue N, SDValue &Op);
-
- bool tryFoldLoad(SDNode *Root, SDNode *P, SDValue N,
- SDValue &Base, SDValue &Scale,
- SDValue &Index, SDValue &Disp,
- SDValue &Segment);
-
- // Convenience method where P is also root.
- bool tryFoldLoad(SDNode *P, SDValue N,
- SDValue &Base, SDValue &Scale,
- SDValue &Index, SDValue &Disp,
- SDValue &Segment) {
- return tryFoldLoad(P, P, N, Base, Scale, Index, Disp, Segment);
- }
+private:
+ void Select(SDNode *N) override;
+
+ bool foldOffsetIntoAddress(uint64_t Offset, X86ISelAddressMode &AM);
+ bool matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM,
+ bool AllowSegmentRegForX32 = false);
+ bool matchWrapper(SDValue N, X86ISelAddressMode &AM);
+ bool matchAddress(SDValue N, X86ISelAddressMode &AM);
+ bool matchVectorAddress(SDValue N, X86ISelAddressMode &AM);
+ bool matchAdd(SDValue &N, X86ISelAddressMode &AM, unsigned Depth);
+ bool hasMaterializingUse(SDValue V) const;
+ SDValue matchIndexRecursively(SDValue N, X86ISelAddressMode &AM,
+ unsigned Depth);
+ bool matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
+ unsigned Depth);
+ bool matchVectorAddressRecursively(SDValue N, X86ISelAddressMode &AM,
+ unsigned Depth);
+ bool matchAddressBase(SDValue N, X86ISelAddressMode &AM);
+ bool selectAddr(SDNode *Parent, SDValue N, SDValue &Base, SDValue &Scale,
+ SDValue &Index, SDValue &Disp, SDValue &Segment,
+ bool HasNDDM = true);
+ bool selectNDDAddr(SDNode *Parent, SDValue N, SDValue &Base, SDValue &Scale,
+ SDValue &Index, SDValue &Disp, SDValue &Segment);
+ bool selectVectorAddr(MemSDNode *Parent, SDValue BasePtr, SDValue IndexOp,
+ SDValue ScaleOp, SDValue &Base, SDValue &Scale,
+ SDValue &Index, SDValue &Disp, SDValue &Segment);
+ bool selectMOV64Imm32(SDValue N, SDValue &Imm);
+ bool selectLEAAddr(SDValue N, SDValue &Base, SDValue &Scale, SDValue &Index,
+ SDValue &Disp, SDValue &Segment);
+ bool selectLEA64_Addr(SDValue N, SDValue &Base, SDValue &Scale,
+ SDValue &Index, SDValue &Disp, SDValue &Segment);
+ bool selectTLSADDRAddr(SDValue N, SDValue &Base, SDValue &Scale,
+ SDValue &Index, SDValue &Disp, SDValue &Segment);
+ bool selectRelocImm(SDValue N, SDValue &Op);
+
+ bool tryFoldLoad(SDNode *Root, SDNode *P, SDValue N, SDValue &Base,
+ SDValue &Scale, SDValue &Index, SDValue &Disp,
+ SDValue &Segment);
+
+ // Convenience method where P is also root.
+ bool tryFoldLoad(SDNode *P, SDValue N, SDValue &Base, SDValue &Scale,
+ SDValue &Index, SDValue &Disp, SDValue &Segment) {
+ return tryFoldLoad(P, P, N, Base, Scale, Index, Disp, Segment);
+ }
- bool tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N,
- SDValue &Base, SDValue &Scale,
- SDValue &Index, SDValue &Disp,
- SDValue &Segment);
-
- bool isProfitableToFormMaskedOp(SDNode *N) const;
-
- /// Implement addressing mode selection for inline asm expressions.
- bool SelectInlineAsmMemoryOperand(const SDValue &Op,
- InlineAsm::ConstraintCode ConstraintID,
- std::vector<SDValue> &OutOps) override;
-
- void emitSpecialCodeForMain();
-
- inline void getAddressOperands(X86ISelAddressMode &AM, const SDLoc &DL,
- MVT VT, SDValue &Base, SDValue &Scale,
- SDValue &Index, SDValue &Disp,
- SDValue &Segment) {
- if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
- Base = CurDAG->getTargetFrameIndex(
- AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout()));
- else if (AM.Base_Reg.getNode())
- Base = AM.Base_Reg;
- else
- Base = CurDAG->getRegister(0, VT);
+ bool tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N, SDValue &Base,
+ SDValue &Scale, SDValue &Index, SDValue &Disp,
+ SDValue &Segment);
- Scale = getI8Imm(AM.Scale, DL);
+ bool isProfitableToFormMaskedOp(SDNode *N) const;
-#define GET_ND_IF_ENABLED(OPC) (Subtarget->hasNDD() ? OPC##_ND : OPC)
-#define GET_NDM_IF_ENABLED(OPC) \
- (Subtarget->hasNDD() && Subtarget->hasNDDM() ? OPC##_ND : OPC)
- // Negate the index if needed.
- if (AM.NegateIndex) {
- unsigned NegOpc;
- switch (VT.SimpleTy) {
- default:
- llvm_unreachable("Unsupported VT!");
- case MVT::i64:
- NegOpc = GET_ND_IF_ENABLED(X86::NEG64r);
- break;
- case MVT::i32:
- NegOpc = GET_ND_IF_ENABLED(X86::NEG32r);
- break;
- case MVT::i16:
- NegOpc = GET_ND_IF_ENABLED(X86::NEG16r);
- break;
- case MVT::i8:
- NegOpc = GET_ND_IF_ENABLED(X86::NEG8r);
- break;
- }
- SDValue Neg = SDValue(CurDAG->getMachineNode(NegOpc, DL, VT, MVT::i32,
- AM.IndexReg), 0);
- AM.IndexReg = Neg;
- }
+ /// Implement addressing mode selection for inline asm expressions.
+ bool SelectInlineAsmMemoryOperand(const SDValue &Op,
+ InlineAsm::ConstraintCode ConstraintID,
+ std::vector<SDValue> &OutOps) override;
- if (AM.IndexReg.getNode())
- Index = AM.IndexReg;
- else
- Index = CurDAG->getRegister(0, VT);
-
- // These are 32-bit even in 64-bit mode since RIP-relative offset
- // is 32-bit.
- if (AM.GV)
- Disp = CurDAG->getTargetGlobalAddress(AM.GV, SDLoc(),
- MVT::i32, AM.Disp,
- AM.SymbolFlags);
- else if (AM.CP)
- Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment,
- AM.Disp, AM.SymbolFlags);
- else if (AM.ES) {
- assert(!AM.Disp && "Non-zero displacement is ignored with ES.");
- Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
- } else if (AM.MCSym) {
- assert(!AM.Disp && "Non-zero displacement is ignored with MCSym.");
- assert(AM.SymbolFlags == 0 && "oo");
- Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32);
- } else if (AM.JT != -1) {
- assert(!AM.Disp && "Non-zero displacement is ignored with JT.");
- Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
- } else if (AM.BlockAddr)
- Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
- AM.SymbolFlags);
- else
- Disp = CurDAG->getSignedTargetConstant(AM.Disp, DL, MVT::i32);
+ void emitSpecialCodeForMain();
- if (AM.Segment.getNode())
- Segment = AM.Segment;
- else
- Segment = CurDAG->getRegister(0, MVT::i16);
- }
+ inline void getAddressOperands(X86ISelAddressMode &AM, const SDLoc &DL,
+ MVT VT, SDValue &Base, SDValue &Scale,
+ SDValue &Index, SDValue &Disp,
+ SDValue &Segment) {
+ if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
+ Base = CurDAG->getTargetFrameIndex(
+ AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout()));
+ else if (AM.Base_Reg.getNode())
+ Base = AM.Base_Reg;
+ else
+ Base = CurDAG->getRegister(0, VT);
- // Utility function to determine whether it is AMX SDNode right after
- // lowering but before ISEL.
- bool isAMXSDNode(SDNode *N) const {
- // Check if N is AMX SDNode:
- // 1. check result type;
- // 2. check operand type;
- for (unsigned Idx = 0, E = N->getNumValues(); Idx != E; ++Idx) {
- if (N->getValueType(Idx) == MVT::x86amx)
- return true;
- }
- for (unsigned Idx = 0, E = N->getNumOperands(); Idx != E; ++Idx) {
- SDValue Op = N->getOperand(Idx);
- if (Op.getValueType() == MVT::x86amx)
- return true;
+ Scale = getI8Imm(AM.Scale, DL);
+
+#define GET_ND_IF_ENABLED(OPC) (Subtarget->hasNDD() ? OPC##_ND : OPC)
+#define GET_NDM_IF_ENABLED(OPC) \
+ (Subtarget->hasNDD() && Subtarget->hasNDDM() ? OPC##_ND : OPC)
+ // Negate the index if needed.
+ if (AM.NegateIndex) {
+ unsigned NegOpc;
+ switch (VT.SimpleTy) {
+ default:
+ llvm_unreachable("Unsupported VT!");
+ case MVT::i64:
+ NegOpc = GET_ND_IF_ENABLED(X86::NEG64r);
+ break;
+ case MVT::i32:
+ NegOpc = GET_ND_IF_ENABLED(X86::NEG32r);
+ break;
+ case MVT::i16:
+ NegOpc = GET_ND_IF_ENABLED(X86::NEG16r);
+ break;
+ case MVT::i8:
+ NegOpc = GET_ND_IF_ENABLED(X86::NEG8r);
+ break;
}
- return false;
+ SDValue Neg = SDValue(
+ CurDAG->getMachineNode(NegOpc, DL, VT, MVT::i32, AM.IndexReg), 0);
+ AM.IndexReg = Neg;
}
- // Utility function to determine whether we should avoid selecting
- // immediate forms of instructions for better code size or not.
- // At a high level, we'd like to avoid such instructions when
- // we have similar constants used within the same basic block
- // that can be kept in a register.
- //
- bool shouldAvoidImmediateInstFormsForSize(SDNode *N) const {
- uint32_t UseCount = 0;
-
- // Do not want to hoist if we're not optimizing for size.
- // TODO: We'd like to remove this restriction.
- // See the comment in X86InstrInfo.td for more info.
- if (!CurDAG->shouldOptForSize())
- return false;
+ if (AM.IndexReg.getNode())
+ Index = AM.IndexReg;
+ else
+ Index = CurDAG->getRegister(0, VT);
+
+ // These are 32-bit even in 64-bit mode since RIP-relative offset
+ // is 32-bit.
+ if (AM.GV)
+ Disp = CurDAG->getTargetGlobalAddress(AM.GV, SDLoc(), MVT::i32, AM.Disp,
+ AM.SymbolFlags);
+ else if (AM.CP)
+ Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment,
+ AM.Disp, AM.SymbolFlags);
+ else if (AM.ES) {
+ assert(!AM.Disp && "Non-zero displacement is ignored with ES.");
+ Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
+ } else if (AM.MCSym) {
+ assert(!AM.Disp && "Non-zero displacement is ignored with MCSym.");
+ assert(AM.SymbolFlags == 0 && "oo");
+ Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32);
+ } else if (AM.JT != -1) {
+ assert(!AM.Disp && "Non-zero displacement is ignored with JT.");
+ Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
+ } else if (AM.BlockAddr)
+ Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
+ AM.SymbolFlags);
+ else
+ Disp = CurDAG->getSignedTargetConstant(AM.Disp, DL, MVT::i32);
- // Walk all the users of the immediate.
- for (const SDNode *User : N->users()) {
- if (UseCount >= 2)
- break;
+ if (AM.Segment.getNode())
+ Segment = AM.Segment;
+ else
+ Segment = CurDAG->getRegister(0, MVT::i16);
+ }
- // This user is already selected. Count it as a legitimate use and
- // move on.
- if (User->isMachineOpcode()) {
- UseCount++;
- continue;
- }
+ // Utility function to determine whether it is AMX SDNode right after
+ // lowering but before ISEL.
+ bool isAMXSDNode(SDNode *N) const {
+ // Check if N is AMX SDNode:
+ // 1. check result type;
+ // 2. check operand type;
+ for (unsigned Idx = 0, E = N->getNumValues(); Idx != E; ++Idx) {
+ if (N->getValueType(Idx) == MVT::x86amx)
+ return true;
+ }
+ for (unsigned Idx = 0, E = N->getNumOperands(); Idx != E; ++Idx) {
+ SDValue Op = N->getOperand(Idx);
+ if (Op.getValueType() == MVT::x86amx)
+ return true;
+ }
+ return false;
+ }
- // We want to count stores of immediates as real uses.
- if (User->getOpcode() == ISD::STORE &&
- User->getOperand(1).getNode() == N) {
- UseCount++;
- continue;
- }
+ // Utility function to determine whether we should avoid selecting
+ // immediate forms of instructions for better code size or not.
+ // At a high level, we'd like to avoid such instructions when
+ // we have similar constants used within the same basic block
+ // that can be kept in a register.
+ //
+ bool shouldAvoidImmediateInstFormsForSize(SDNode *N) const {
+ uint32_t UseCount = 0;
- // We don't currently match users that have > 2 operands (except
- // for stores, which are handled above)
- // Those instruction won't match in ISEL, for now, and would
- // be counted incorrectly.
- // This may change in the future as we add additional instruction
- // types.
- if (User->getNumOperands() != 2)
- continue;
+ // Do not want to hoist if we're not optimizing for size.
+ // TODO: We'd like to remove this restriction.
+ // See the comment in X86InstrInfo.td for more info.
+ if (!CurDAG->shouldOptForSize())
+ return false;
- // If this is a sign-extended 8-bit integer immediate used in an ALU
- // instruction, there is probably an opcode encoding to save space.
- auto *C = dyn_cast<ConstantSDNode>(N);
- if (C && isInt<8>(C->getSExtValue()))
- continue;
+ // Walk all the users of the immediate.
+ for (const SDNode *User : N->users()) {
+ if (UseCount >= 2)
+ break;
- // Immediates that are used for offsets as part of stack
- // manipulation should be left alone. These are typically
- // used to indicate SP offsets for argument passing and
- // will get pulled into stores/pushes (implicitly).
- if (User->getOpcode() == X86ISD::ADD ||
- User->getOpcode() == ISD::ADD ||
- User->getOpcode() == X86ISD::SUB ||
- User->getOpcode() == ISD::SUB) {
-
- // Find the other operand of the add/sub.
- SDValue OtherOp = User->getOperand(0);
- if (OtherOp.getNode() == N)
- OtherOp = User->getOperand(1);
-
- // Don't count if the other operand is SP.
- RegisterSDNode *RegNode;
- if (OtherOp->getOpcode() == ISD::CopyFromReg &&
- (RegNode = dyn_cast_or_null<RegisterSDNode>(
- OtherOp->getOperand(1).getNode())))
- if ((RegNode->getReg() == X86::ESP) ||
- (RegNode->getReg() == X86::RSP))
- continue;
- }
+ // This user is already selected. Count it as a legitimate use and
+ // move on.
+ if (User->isMachineOpcode()) {
+ UseCount++;
+ continue;
+ }
- // ... otherwise, count this and move on.
+ // We want to count stores of immediates as real uses.
+ if (User->getOpcode() == ISD::STORE &&
+ User->getOperand(1).getNode() == N) {
UseCount++;
+ continue;
}
- // If we have more than 1 use, then recommend for hoisting.
- return (UseCount > 1);
- }
+ // We don't currently match users that have > 2 operands (except
+ // for stores, which are handled above)
+ // Those instruction won't match in ISEL, for now, and would
+ // be counted incorrectly.
+ // This may change in the future as we add additional instruction
+ // types.
+ if (User->getNumOperands() != 2)
+ continue;
- /// Return a target constant with the specified value of type i8.
- inline SDValue getI8Imm(unsigned Imm, const SDLoc &DL) {
- return CurDAG->getTargetConstant(Imm, DL, MVT::i8);
- }
+ // If this is a sign-extended 8-bit integer immediate used in an ALU
+ // instruction, there is probably an opcode encoding to save space.
+ auto *C = dyn_cast<ConstantSDNode>(N);
+ if (C && isInt<8>(C->getSExtValue()))
+ continue;
- /// Return a target constant with the specified value, of type i32.
- inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) {
- return CurDAG->getTargetConstant(Imm, DL, MVT::i32);
- }
+ // Immediates that are used for offsets as part of stack
+ // manipulation should be left alone. These are typically
+ // used to indicate SP offsets for argument passing and
+ // will get pulled into stores/pushes (implicitly).
+ if (User->getOpcode() == X86ISD::ADD || User->getOpcode() == ISD::ADD ||
+ User->getOpcode() == X86ISD::SUB || User->getOpcode() == ISD::SUB) {
+
+ // Find the other operand of the add/sub.
+ SDValue OtherOp = User->getOperand(0);
+ if (OtherOp.getNode() == N)
+ OtherOp = User->getOperand(1);
+
+ // Don't count if the other operand is SP.
+ RegisterSDNode *RegNode;
+ if (OtherOp->getOpcode() == ISD::CopyFromReg &&
+ (RegNode = dyn_cast_or_null<RegisterSDNode>(
+ OtherOp->getOperand(1).getNode())))
+ if ((RegNode->getReg() == X86::ESP) ||
+ (RegNode->getReg() == X86::RSP))
+ continue;
+ }
- /// Return a target constant with the specified value, of type i64.
- inline SDValue getI64Imm(uint64_t Imm, const SDLoc &DL) {
- return CurDAG->getTargetConstant(Imm, DL, MVT::i64);
+ // ... otherwise, count this and move on.
+ UseCount++;
}
- SDValue getExtractVEXTRACTImmediate(SDNode *N, unsigned VecWidth,
- const SDLoc &DL) {
- assert((VecWidth == 128 || VecWidth == 256) && "Unexpected vector width");
- uint64_t Index = N->getConstantOperandVal(1);
- MVT VecVT = N->getOperand(0).getSimpleValueType();
- return getI8Imm((Index * VecVT.getScalarSizeInBits()) / VecWidth, DL);
- }
+ // If we have more than 1 use, then recommend for hoisting.
+ return (UseCount > 1);
+ }
- SDValue getInsertVINSERTImmediate(SDNode *N, unsigned VecWidth,
- const SDLoc &DL) {
- assert((VecWidth == 128 || VecWidth == 256) && "Unexpected vector width");
- uint64_t Index = N->getConstantOperandVal(2);
- MVT VecVT = N->getSimpleValueType(0);
- return getI8Imm((Index * VecVT.getScalarSizeInBits()) / VecWidth, DL);
- }
+ /// Return a target constant with the specified value of type i8.
+ inline SDValue getI8Imm(unsigned Imm, const SDLoc &DL) {
+ return CurDAG->getTargetConstant(Imm, DL, MVT::i8);
+ }
- SDValue getPermuteVINSERTCommutedImmediate(SDNode *N, unsigned VecWidth,
- const SDLoc &DL) {
- assert(VecWidth == 128 && "Unexpected vector width");
- uint64_t Index = N->getConstantOperandVal(2);
- MVT VecVT = N->getSimpleValueType(0);
- uint64_t InsertIdx = (Index * VecVT.getScalarSizeInBits()) / VecWidth;
- assert((InsertIdx == 0 || InsertIdx == 1) && "Bad insertf128 index");
- // vinsert(0,sub,vec) -> [sub0][vec1] -> vperm2x128(0x30,vec,sub)
- // vinsert(1,sub,vec) -> [vec0][sub0] -> vperm2x128(0x02,vec,sub)
- return getI8Imm(InsertIdx ? 0x02 : 0x30, DL);
- }
+ /// Return a target constant with the specified value, of type i32.
+ inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) {
+ return CurDAG->getTargetConstant(Imm, DL, MVT::i32);
+ }
- SDValue getSBBZero(SDNode *N) {
- SDLoc dl(N);
- MVT VT = N->getSimpleValueType(0);
+ /// Return a target constant with the specified value, of type i64.
+ inline SDValue getI64Imm(uint64_t Imm, const SDLoc &DL) {
+ return CurDAG->getTargetConstant(Imm, DL, MVT::i64);
+ }
- // Create zero.
- SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
- SDValue Zero =
- SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, {}), 0);
- if (VT == MVT::i64) {
- Zero = SDValue(
- CurDAG->getMachineNode(
- TargetOpcode::SUBREG_TO_REG, dl, MVT::i64, Zero,
- CurDAG->getTargetConstant(X86::sub_32bit, dl, MVT::i32)),
- 0);
- }
+ SDValue getExtractVEXTRACTImmediate(SDNode *N, unsigned VecWidth,
+ const SDLoc &DL) {
+ assert((VecWidth == 128 || VecWidth == 256) && "Unexpected vector width");
+ uint64_t Index = N->getConstantOperandVal(1);
+ MVT VecVT = N->getOperand(0).getSimpleValueType();
+ return getI8Imm((Index * VecVT.getScalarSizeInBits()) / VecWidth, DL);
+ }
- // Copy flags to the EFLAGS register and glue it to next node.
- unsigned Opcode = N->getOpcode();
- assert((Opcode == X86ISD::SBB || Opcode == X86ISD::SETCC_CARRY) &&
- "Unexpected opcode for SBB materialization");
- unsigned FlagOpIndex = Opcode == X86ISD::SBB ? 2 : 1;
- SDValue EFLAGS =
- CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
- N->getOperand(FlagOpIndex), SDValue());
+ SDValue getInsertVINSERTImmediate(SDNode *N, unsigned VecWidth,
+ const SDLoc &DL) {
+ assert((VecWidth == 128 || VecWidth == 256) && "Unexpected vector width");
+ uint64_t Index = N->getConstantOperandVal(2);
+ MVT VecVT = N->getSimpleValueType(0);
+ return getI8Imm((Index * VecVT.getScalarSizeInBits()) / VecWidth, DL);
+ }
- // Create a 64-bit instruction if the result is 64-bits otherwise use the
- // 32-bit version.
- unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr;
- MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
- VTs = CurDAG->getVTList(SBBVT, MVT::i32);
- return SDValue(
- CurDAG->getMachineNode(Opc, dl, VTs,
- {Zero, Zero, EFLAGS, EFLAGS.getValue(1)}),
- 0);
+ SDValue getPermuteVINSERTCommutedImmediate(SDNode *N, unsigned VecWidth,
+ const SDLoc &DL) {
+ assert(VecWidth == 128 && "Unexpected vector width");
+ uint64_t Index = N->getConstantOperandVal(2);
+ MVT VecVT = N->getSimpleValueType(0);
+ uint64_t InsertIdx = (Index * VecVT.getScalarSizeInBits()) / VecWidth;
+ assert((InsertIdx == 0 || InsertIdx == 1) && "Bad insertf128 index");
+ // vinsert(0,sub,vec) -> [sub0][vec1] -> vperm2x128(0x30,vec,sub)
+ // vinsert(1,sub,vec) -> [vec0][sub0] -> vperm2x128(0x02,vec,sub)
+ return getI8Imm(InsertIdx ? 0x02 : 0x30, DL);
+ }
+
+ SDValue getSBBZero(SDNode *N) {
+ SDLoc dl(N);
+ MVT VT = N->getSimpleValueType(0);
+
+ // Create zero.
+ SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
+ SDValue Zero =
+ SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, {}), 0);
+ if (VT == MVT::i64) {
+ Zero =
+ SDValue(CurDAG->getMachineNode(
+ TargetOpcode::SUBREG_TO_REG, dl, MVT::i64, Zero,
+ CurDAG->getTargetConstant(X86::sub_32bit, dl, MVT::i32)),
+ 0);
}
- // Helper to detect unneeded and instructions on shift amounts. Called
- // from PatFrags in tablegen.
- bool isUnneededShiftMask(SDNode *N, unsigned Width) const {
- assert(N->getOpcode() == ISD::AND && "Unexpected opcode");
- const APInt &Val = N->getConstantOperandAPInt(1);
+ // Copy flags to the EFLAGS register and glue it to next node.
+ unsigned Opcode = N->getOpcode();
+ assert((Opcode == X86ISD::SBB || Opcode == X86ISD::SETCC_CARRY) &&
+ "Unexpected opcode for SBB materialization");
+ unsigned FlagOpIndex = Opcode == X86ISD::SBB ? 2 : 1;
+ SDValue EFLAGS =
+ CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
+ N->getOperand(FlagOpIndex), SDValue());
+
+ // Create a 64-bit instruction if the result is 64-bits otherwise use the
+ // 32-bit version.
+ unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr;
+ MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
+ VTs = CurDAG->getVTList(SBBVT, MVT::i32);
+ return SDValue(CurDAG->getMachineNode(
+ Opc, dl, VTs, {Zero, Zero, EFLAGS, EFLAGS.getValue(1)}),
+ 0);
+ }
- if (Val.countr_one() >= Width)
- return true;
+ // Helper to detect unneeded and instructions on shift amounts. Called
+ // from PatFrags in tablegen.
+ bool isUnneededShiftMask(SDNode *N, unsigned Width) const {
+ assert(N->getOpcode() == ISD::AND && "Unexpected opcode");
+ const APInt &Val = N->getConstantOperandAPInt(1);
- APInt Mask = Val | CurDAG->computeKnownBits(N->getOperand(0)).Zero;
- return Mask.countr_one() >= Width;
- }
+ if (Val.countr_one() >= Width)
+ return true;
- /// Return an SDNode that returns the value of the global base register.
- /// Output instructions required to initialize the global base register,
- /// if necessary.
- SDNode *getGlobalBaseReg();
+ APInt Mask = Val | CurDAG->computeKnownBits(N->getOperand(0)).Zero;
+ return Mask.countr_one() >= Width;
+ }
- /// Return a reference to the TargetMachine, casted to the target-specific
- /// type.
- const X86TargetMachine &getTargetMachine() const {
- return static_cast<const X86TargetMachine &>(TM);
- }
+ /// Return an SDNode that returns the value of the global base register.
+ /// Output instructions required to initialize the global base register,
+ /// if necessary.
+ SDNode *getGlobalBaseReg();
- /// Return a reference to the TargetInstrInfo, casted to the target-specific
- /// type.
- const X86InstrInfo *getInstrInfo() const {
- return Subtarget->getInstrInfo();
- }
+ /// Return a reference to the TargetMachine, casted to the target-specific
+ /// type.
+ const X86TargetMachine &getTargetMachine() const {
+ return static_cast<const X86TargetMachine &>(TM);
+ }
- /// Return a condition code of the given SDNode
- X86::CondCode getCondFromNode(SDNode *N) const;
+ /// Return a reference to the TargetInstrInfo, casted to the target-specific
+ /// type.
+ const X86InstrInfo *getInstrInfo() const { return Subtarget->getInstrInfo(); }
- /// Address-mode matching performs shift-of-and to and-of-shift
- /// reassociation in order to expose more scaled addressing
- /// opportunities.
- bool ComplexPatternFuncMutatesDAG() const override {
- return true;
- }
+ /// Return a condition code of the given SDNode
+ X86::CondCode getCondFromNode(SDNode *N) const;
- bool isSExtAbsoluteSymbolRef(unsigned Width, SDNode *N) const;
+ /// Address-mode matching performs shift-of-and to and-of-shift
+ /// reassociation in order to expose more scaled addressing
+ /// opportunities.
+ bool ComplexPatternFuncMutatesDAG() const override { return true; }
- // Indicates we should prefer to use a non-temporal load for this load.
- bool useNonTemporalLoad(LoadSDNode *N) const {
- if (!N->isNonTemporal())
- return false;
+ bool isSExtAbsoluteSymbolRef(unsigned Width, SDNode *N) const;
- unsigned StoreSize = N->getMemoryVT().getStoreSize();
+ // Indicates we should prefer to use a non-temporal load for this load.
+ bool useNonTemporalLoad(LoadSDNode *N) const {
+ if (!N->isNonTemporal())
+ return false;
- if (N->getAlign().value() < StoreSize)
- return false;
+ unsigned StoreSize = N->getMemoryVT().getStoreSize();
- switch (StoreSize) {
- default: llvm_unreachable("Unsupported store size");
- case 4:
- case 8:
- return false;
- case 16:
- return Subtarget->hasSSE41();
- case 32:
- return Subtarget->hasAVX2();
- case 64:
- return Subtarget->hasAVX512();
- }
- }
+ if (N->getAlign().value() < StoreSize)
+ return false;
- bool foldLoadStoreIntoMemOperand(SDNode *Node);
- MachineSDNode *matchBEXTRFromAndImm(SDNode *Node);
- bool matchBitExtract(SDNode *Node);
- bool shrinkAndImmediate(SDNode *N);
- bool isMaskZeroExtended(SDNode *N) const;
- bool tryShiftAmountMod(SDNode *N);
- bool tryShrinkShlLogicImm(SDNode *N);
- bool tryVPTERNLOG(SDNode *N);
- bool matchVPTERNLOG(SDNode *Root, SDNode *ParentA, SDNode *ParentB,
- SDNode *ParentC, SDValue A, SDValue B, SDValue C,
- uint8_t Imm);
- bool tryVPTESTM(SDNode *Root, SDValue Setcc, SDValue Mask);
- bool tryMatchBitSelect(SDNode *N);
-
- MachineSDNode *emitPCMPISTR(unsigned ROpc, unsigned MOpc, bool MayFoldLoad,
- const SDLoc &dl, MVT VT, SDNode *Node);
- MachineSDNode *emitPCMPESTR(unsigned ROpc, unsigned MOpc, bool MayFoldLoad,
- const SDLoc &dl, MVT VT, SDNode *Node,
- SDValue &InGlue);
-
- bool tryOptimizeRem8Extend(SDNode *N);
-
- bool onlyUsesZeroFlag(SDValue Flags) const;
- bool hasNoSignFlagUses(SDValue Flags) const;
- bool hasNoCarryFlagUses(SDValue Flags) const;
- bool checkTCRetEnoughRegs(SDNode *N) const;
- };
+ switch (StoreSize) {
+ default:
+ llvm_unreachable("Unsupported store size");
+ case 4:
+ case 8:
+ return false;
+ case 16:
+ return Subtarget->hasSSE41();
+ case 32:
+ return Subtarget->hasAVX2();
+ case 64:
+ return Subtarget->hasAVX512();
+ }
+ }
- class X86DAGToDAGISelLegacy : public SelectionDAGISelLegacy {
- public:
- static char ID;
- explicit X86DAGToDAGISelLegacy(X86TargetMachine &tm,
- CodeGenOptLevel OptLevel)
- : SelectionDAGISelLegacy(
- ID, std::make_unique<X86DAGToDAGISel>(tm, OptLevel)) {}
- };
-}
+ bool foldLoadStoreIntoMemOperand(SDNode *Node);
+ MachineSDNode *matchBEXTRFromAndImm(SDNode *Node);
+ bool matchBitExtract(SDNode *Node);
+ bool shrinkAndImmediate(SDNode *N);
+ bool isMaskZeroExtended(SDNode *N) const;
+ bool tryShiftAmountMod(SDNode *N);
+ bool tryShrinkShlLogicImm(SDNode *N);
+ bool tryVPTERNLOG(SDNode *N);
+ bool matchVPTERNLOG(SDNode *Root, SDNode *ParentA, SDNode *ParentB,
+ SDNode *ParentC, SDValue A, SDValue B, SDValue C,
+ uint8_t Imm);
+ bool tryVPTESTM(SDNode *Root, SDValue Setcc, SDValue Mask);
+ bool tryMatchBitSelect(SDNode *N);
+
+ MachineSDNode *emitPCMPISTR(unsigned ROpc, unsigned MOpc, bool MayFoldLoad,
+ const SDLoc &dl, MVT VT, SDNode *Node);
+ MachineSDNode *emitPCMPESTR(unsigned ROpc, unsigned MOpc, bool MayFoldLoad,
+ const SDLoc &dl, MVT VT, SDNode *Node,
+ SDValue &InGlue);
+
+ bool tryOptimizeRem8Extend(SDNode *N);
+
+ bool onlyUsesZeroFlag(SDValue Flags) const;
+ bool hasNoSignFlagUses(SDValue Flags) const;
+ bool hasNoCarryFlagUses(SDValue Flags) const;
+ bool checkTCRetEnoughRegs(SDNode *N) const;
+};
+
+class X86DAGToDAGISelLegacy : public SelectionDAGISelLegacy {
+public:
+ static char ID;
+ explicit X86DAGToDAGISelLegacy(X86TargetMachine &tm, CodeGenOptLevel OptLevel)
+ : SelectionDAGISelLegacy(
+ ID, std::make_unique<X86DAGToDAGISel>(tm, OptLevel)) {}
+};
+} // namespace
char X86DAGToDAGISelLegacy::ID = 0;
@@ -669,8 +651,8 @@ bool X86DAGToDAGISel::isMaskZeroExtended(SDNode *N) const {
return isLegalMaskCompare(N, Subtarget);
}
-bool
-X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const {
+bool X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
+ SDNode *Root) const {
if (OptLevel == CodeGenOptLevel::None)
return false;
@@ -687,7 +669,8 @@ X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const {
// If N is a load, do additional profitability checks.
if (U == Root) {
switch (U->getOpcode()) {
- default: break;
+ default:
+ break;
case X86ISD::ADD:
case X86ISD::ADC:
case X86ISD::SUB:
@@ -729,10 +712,9 @@ X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const {
// If this really a zext_inreg that can be represented with a movzx
// instruction, prefer that.
// TODO: We could shrink the load and fold if it is non-volatile.
- if (U->getOpcode() == ISD::AND &&
- (Imm->getAPIntValue() == UINT8_MAX ||
- Imm->getAPIntValue() == UINT16_MAX ||
- Imm->getAPIntValue() == UINT32_MAX))
+ if (U->getOpcode() == ISD::AND && (Imm->getAPIntValue() == UINT8_MAX ||
+ Imm->getAPIntValue() == UINT16_MAX ||
+ Imm->getAPIntValue() == UINT32_MAX))
return false;
// ADD/SUB with can negate the immediate and use the opposite operation
@@ -835,22 +817,21 @@ bool X86DAGToDAGISel::isProfitableToFormMaskedOp(SDNode *N) const {
/// Replace the original chain operand of the call with
/// load's chain operand and move load below the call's chain operand.
-static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load,
- SDValue Call, SDValue OrigChain) {
+static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, SDValue Call,
+ SDValue OrigChain) {
SmallVector<SDValue, 8> Ops;
SDValue Chain = OrigChain.getOperand(0);
if (Chain.getNode() == Load.getNode())
Ops.push_back(Load.getOperand(0));
else {
- assert(Chain.getOpcode() == ISD::TokenFactor &&
- "Unexpected chain operand");
+ assert(Chain.getOpcode() == ISD::TokenFactor && "Unexpected chain operand");
for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i)
if (Chain.getOperand(i).getNode() == Load.getNode())
Ops.push_back(Load.getOperand(0));
else
Ops.push_back(Chain.getOperand(i));
SDValue NewChain =
- CurDAG->getNode(ISD::TokenFactor, SDLoc(Load), MVT::Other, Ops);
+ CurDAG->getNode(ISD::TokenFactor, SDLoc(Load), MVT::Other, Ops);
Ops.clear();
Ops.push_back(NewChain);
}
@@ -878,9 +859,7 @@ static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) {
if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse())
return false;
auto *LD = dyn_cast<LoadSDNode>(Callee.getNode());
- if (!LD ||
- !LD->isSimple() ||
- LD->getAddressingMode() != ISD::UNINDEXED ||
+ if (!LD || !LD->isSimple() || LD->getAddressingMode() != ISD::UNINDEXED ||
LD->getExtensionType() != ISD::NON_EXTLOAD)
return false;
@@ -933,13 +912,13 @@ static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) {
}
static bool isEndbrImm64(uint64_t Imm) {
-// There may be some other prefix bytes between 0xF3 and 0x0F1EFA.
-// i.g: 0xF3660F1EFA, 0xF3670F1EFA
+ // There may be some other prefix bytes between 0xF3 and 0x0F1EFA.
+ // i.g: 0xF3660F1EFA, 0xF3670F1EFA
if ((Imm & 0x00FFFFFF) != 0x0F1EFA)
return false;
- uint8_t OptionalPrefixBytes [] = {0x26, 0x2e, 0x36, 0x3e, 0x64,
- 0x65, 0x66, 0x67, 0xf0, 0xf2};
+ uint8_t OptionalPrefixBytes[] = {0x26, 0x2e, 0x36, 0x3e, 0x64,
+ 0x65, 0x66, 0x67, 0xf0, 0xf2};
int i = 24; // 24bit 0x0F1EFA has matched
while (i < 64) {
uint8_t Byte = (Imm >> i) & 0xFF;
@@ -960,7 +939,8 @@ static bool needBWI(MVT VT) {
void X86DAGToDAGISel::PreprocessISelDAG() {
bool MadeChange = false;
for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
- E = CurDAG->allnodes_end(); I != E; ) {
+ E = CurDAG->allnodes_end();
+ I != E;) {
SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues.
// This is for CET enhancement.
@@ -1172,7 +1152,8 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
if (Subtarget->hasVLX() && CurDAG->ComputeNumSignBits(N->getOperand(0)) ==
EleVT.getSizeInBits()) {
R = CurDAG->getNode(X86ISD::VPTERNLOG, SDLoc(N), N->getValueType(0),
- N->getOperand(0), N->getOperand(1), N->getOperand(2),
+ N->getOperand(0), N->getOperand(1),
+ N->getOperand(2),
CurDAG->getTargetConstant(0xCA, SDLoc(N), MVT::i8));
} else {
R = CurDAG->getNode(X86ISD::BLENDV, SDLoc(N), N->getValueType(0),
@@ -1198,13 +1179,26 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
unsigned NewOpc;
switch (N->getOpcode()) {
- default: llvm_unreachable("Unexpected opcode!");
- case ISD::FP_ROUND: NewOpc = X86ISD::VFPROUND; break;
- case ISD::STRICT_FP_ROUND: NewOpc = X86ISD::STRICT_VFPROUND; break;
- case ISD::STRICT_FP_TO_SINT: NewOpc = X86ISD::STRICT_CVTTP2SI; break;
- case ISD::FP_TO_SINT: NewOpc = X86ISD::CVTTP2SI; break;
- case ISD::STRICT_FP_TO_UINT: NewOpc = X86ISD::STRICT_CVTTP2UI; break;
- case ISD::FP_TO_UINT: NewOpc = X86ISD::CVTTP2UI; break;
+ default:
+ llvm_unreachable("Unexpected opcode!");
+ case ISD::FP_ROUND:
+ NewOpc = X86ISD::VFPROUND;
+ break;
+ case ISD::STRICT_FP_ROUND:
+ NewOpc = X86ISD::STRICT_VFPROUND;
+ break;
+ case ISD::STRICT_FP_TO_SINT:
+ NewOpc = X86ISD::STRICT_CVTTP2SI;
+ break;
+ case ISD::FP_TO_SINT:
+ NewOpc = X86ISD::CVTTP2SI;
+ break;
+ case ISD::STRICT_FP_TO_UINT:
+ NewOpc = X86ISD::STRICT_CVTTP2UI;
+ break;
+ case ISD::FP_TO_UINT:
+ NewOpc = X86ISD::CVTTP2UI;
+ break;
}
SDValue Res;
if (N->isStrictFPOpcode())
@@ -1212,9 +1206,8 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
CurDAG->getNode(NewOpc, SDLoc(N), {N->getValueType(0), MVT::Other},
{N->getOperand(0), N->getOperand(1)});
else
- Res =
- CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0),
- N->getOperand(0));
+ Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0),
+ N->getOperand(0));
--I;
CurDAG->ReplaceAllUsesWith(N, Res.getNode());
++I;
@@ -1231,10 +1224,17 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
unsigned NewOpc;
switch (N->getOpcode()) {
- default: llvm_unreachable("Unexpected opcode!");
- case ISD::SHL: NewOpc = X86ISD::VSHLV; break;
- case ISD::SRA: NewOpc = X86ISD::VSRAV; break;
- case ISD::SRL: NewOpc = X86ISD::VSRLV; break;
+ default:
+ llvm_unreachable("Unexpected opcode!");
+ case ISD::SHL:
+ NewOpc = X86ISD::VSHLV;
+ break;
+ case ISD::SRA:
+ NewOpc = X86ISD::VSRAV;
+ break;
+ case ISD::SRL:
+ NewOpc = X86ISD::VSRLV;
+ break;
}
SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0),
N->getOperand(0), N->getOperand(1));
@@ -1258,8 +1258,8 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
NewOpc = ISD::SIGN_EXTEND;
} else {
NewOpc = N->getOpcode() == ISD::ANY_EXTEND
- ? ISD::ZERO_EXTEND
- : ISD::ZERO_EXTEND_VECTOR_INREG;
+ ? ISD::ZERO_EXTEND
+ : ISD::ZERO_EXTEND_VECTOR_INREG;
}
SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0),
@@ -1286,19 +1286,32 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
// need 2 sets of patterns.
unsigned Imm;
switch (N->getOpcode()) {
- default: llvm_unreachable("Unexpected opcode!");
+ default:
+ llvm_unreachable("Unexpected opcode!");
case ISD::STRICT_FCEIL:
- case ISD::FCEIL: Imm = 0xA; break;
+ case ISD::FCEIL:
+ Imm = 0xA;
+ break;
case ISD::STRICT_FFLOOR:
- case ISD::FFLOOR: Imm = 0x9; break;
+ case ISD::FFLOOR:
+ Imm = 0x9;
+ break;
case ISD::STRICT_FTRUNC:
- case ISD::FTRUNC: Imm = 0xB; break;
+ case ISD::FTRUNC:
+ Imm = 0xB;
+ break;
case ISD::STRICT_FROUNDEVEN:
- case ISD::FROUNDEVEN: Imm = 0x8; break;
+ case ISD::FROUNDEVEN:
+ Imm = 0x8;
+ break;
case ISD::STRICT_FNEARBYINT:
- case ISD::FNEARBYINT: Imm = 0xC; break;
+ case ISD::FNEARBYINT:
+ Imm = 0xC;
+ break;
case ISD::STRICT_FRINT:
- case ISD::FRINT: Imm = 0x4; break;
+ case ISD::FRINT:
+ Imm = 0x4;
+ break;
}
SDLoc dl(N);
bool IsStrict = N->isStrictFPOpcode();
@@ -1333,10 +1346,10 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
: MVT::v8f16;
SDLoc dl(N);
- SDValue Op0 = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT,
- N->getOperand(0));
- SDValue Op1 = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT,
- N->getOperand(1));
+ SDValue Op0 =
+ CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, N->getOperand(0));
+ SDValue Op1 =
+ CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, N->getOperand(1));
SDValue Res;
if (Subtarget->hasSSE2()) {
@@ -1345,11 +1358,20 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
Op1 = CurDAG->getNode(ISD::BITCAST, dl, IntVT, Op1);
unsigned Opc;
switch (N->getOpcode()) {
- default: llvm_unreachable("Unexpected opcode!");
- case X86ISD::FANDN: Opc = X86ISD::ANDNP; break;
- case X86ISD::FAND: Opc = ISD::AND; break;
- case X86ISD::FOR: Opc = ISD::OR; break;
- case X86ISD::FXOR: Opc = ISD::XOR; break;
+ default:
+ llvm_unreachable("Unexpected opcode!");
+ case X86ISD::FANDN:
+ Opc = X86ISD::ANDNP;
+ break;
+ case X86ISD::FAND:
+ Opc = ISD::AND;
+ break;
+ case X86ISD::FOR:
+ Opc = ISD::OR;
+ break;
+ case X86ISD::FXOR:
+ Opc = ISD::XOR;
+ break;
}
Res = CurDAG->getNode(Opc, dl, IntVT, Op0, Op1);
Res = CurDAG->getNode(ISD::BITCAST, dl, VecVT, Res);
@@ -1396,7 +1418,7 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
/// [CALL]
bool HasCallSeq = N->getOpcode() == X86ISD::CALL;
SDValue Chain = N->getOperand(0);
- SDValue Load = N->getOperand(1);
+ SDValue Load = N->getOperand(1);
if (!isCalleeLoad(Load, Chain, HasCallSeq))
continue;
if (N->getOpcode() == X86ISD::TC_RETURN && !checkTCRetEnoughRegs(N))
@@ -1416,10 +1438,10 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
// late" legalization of these inline with the X86 isel pass.
// FIXME: This should only happen when not compiled with -O0.
switch (N->getOpcode()) {
- default: continue;
+ default:
+ continue;
case ISD::FP_ROUND:
- case ISD::FP_EXTEND:
- {
+ case ISD::FP_EXTEND: {
MVT SrcVT = N->getOperand(0).getSimpleValueType();
MVT DstVT = N->getSimpleValueType(0);
@@ -1445,9 +1467,9 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
continue;
}
- // Here we could have an FP stack truncation or an FPStack <-> SSE convert.
- // FPStack has extload and truncstore. SSE can fold direct loads into other
- // operations. Based on this, decide what we want to do.
+ // Here we could have an FP stack truncation or an FPStack <-> SSE
+ // convert. FPStack has extload and truncstore. SSE can fold direct loads
+ // into other operations. Based on this, decide what we want to do.
MVT MemVT = (N->getOpcode() == ISD::FP_ROUND) ? DstVT : SrcVT;
SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
@@ -1457,25 +1479,36 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
// FIXME: optimize the case where the src/dest is a load or store?
- SDValue Store = CurDAG->getTruncStore(
- CurDAG->getEntryNode(), dl, N->getOperand(0), MemTmp, MPI, MemVT);
+ // Mark the start of the stack temporary's lifetime.
+ SDValue LifetimeStart =
+ CurDAG->getLifetimeNode(true, dl, CurDAG->getEntryNode(), SPFI);
+
+ SDValue Store = CurDAG->getTruncStore(LifetimeStart, dl, N->getOperand(0),
+ MemTmp, MPI, MemVT);
SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store,
MemTmp, MPI, MemVT);
+ // Mark the end of the stack temporary's lifetime. Anchor it to the
+ // DAG root to prevent dead code elimination.
+ SDValue LifetimeEnd =
+ CurDAG->getLifetimeNode(false, dl, Result.getValue(1), SPFI);
+ CurDAG->setRoot(CurDAG->getNode(ISD::TokenFactor, dl, MVT::Other,
+ CurDAG->getRoot(), LifetimeEnd));
+
// We're about to replace all uses of the FP_ROUND/FP_EXTEND with the
// extload we created. This will cause general havok on the dag because
- // anything below the conversion could be folded into other existing nodes.
- // To avoid invalidating 'I', back it up to the convert node.
+ // anything below the conversion could be folded into other existing
+ // nodes. To avoid invalidating 'I', back it up to the convert node.
--I;
CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
break;
}
- //The sequence of events for lowering STRICT_FP versions of these nodes requires
- //dealing with the chain differently, as there is already a preexisting chain.
+ // The sequence of events for lowering STRICT_FP versions of these nodes
+ // requires dealing with the chain differently, as there is already a
+ // preexisting chain.
case ISD::STRICT_FP_ROUND:
- case ISD::STRICT_FP_EXTEND:
- {
+ case ISD::STRICT_FP_EXTEND: {
MVT SrcVT = N->getOperand(1).getSimpleValueType();
MVT DstVT = N->getSimpleValueType(0);
@@ -1501,9 +1534,9 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
continue;
}
- // Here we could have an FP stack truncation or an FPStack <-> SSE convert.
- // FPStack has extload and truncstore. SSE can fold direct loads into other
- // operations. Based on this, decide what we want to do.
+ // Here we could have an FP stack truncation or an FPStack <-> SSE
+ // convert. FPStack has extload and truncstore. SSE can fold direct loads
+ // into other operations. Based on this, decide what we want to do.
MVT MemVT = (N->getOpcode() == ISD::STRICT_FP_ROUND) ? DstVT : SrcVT;
SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex();
@@ -1513,11 +1546,15 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
// FIXME: optimize the case where the src/dest is a load or store?
- //Since the operation is StrictFP, use the preexisting chain.
+ // Mark the start of the stack temporary's lifetime.
+ SDValue LifetimeStart =
+ CurDAG->getLifetimeNode(true, dl, N->getOperand(0), SPFI);
+
+ // Since the operation is StrictFP, use the preexisting chain.
SDValue Store, Result;
if (!SrcIsSSE) {
SDVTList VTs = CurDAG->getVTList(MVT::Other);
- SDValue Ops[] = {N->getOperand(0), N->getOperand(1), MemTmp};
+ SDValue Ops[] = {LifetimeStart, N->getOperand(1), MemTmp};
Store = CurDAG->getMemIntrinsicNode(X86ISD::FST, dl, VTs, Ops, MemVT,
MPI, /*Align*/ std::nullopt,
MachineMemOperand::MOStore);
@@ -1528,8 +1565,8 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
}
} else {
assert(SrcVT == MemVT && "Unexpected VT!");
- Store = CurDAG->getStore(N->getOperand(0), dl, N->getOperand(1), MemTmp,
- MPI);
+ Store =
+ CurDAG->getStore(LifetimeStart, dl, N->getOperand(1), MemTmp, MPI);
}
if (!DstIsSSE) {
@@ -1548,17 +1585,21 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
Result = CurDAG->getLoad(DstVT, dl, Store, MemTmp, MPI);
}
+ // Mark the end of the stack temporary's lifetime.
+ SDValue LifetimeEnd =
+ CurDAG->getLifetimeNode(false, dl, Result.getValue(1), SPFI);
+
// We're about to replace all uses of the FP_ROUND/FP_EXTEND with the
// extload we created. This will cause general havok on the dag because
- // anything below the conversion could be folded into other existing nodes.
- // To avoid invalidating 'I', back it up to the convert node.
+ // anything below the conversion could be folded into other existing
+ // nodes. To avoid invalidating 'I', back it up to the convert node.
--I;
- CurDAG->ReplaceAllUsesWith(N, Result.getNode());
+ CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
+ CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), LifetimeEnd);
break;
}
}
-
// Now that we did that, the node is dead. Increment the iterator to the
// next node to process, then delete N.
++I;
@@ -1586,8 +1627,8 @@ bool X86DAGToDAGISel::tryOptimizeRem8Extend(SDNode *N) {
return false;
// We're looking for either a movsx or movzx to match the original opcode.
- unsigned ExpectedOpc = Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX
- : X86::MOVSX32rr8_NOREX;
+ unsigned ExpectedOpc =
+ Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX : X86::MOVSX32rr8_NOREX;
SDValue N00 = N0.getOperand(0);
if (!N00.isMachineOpcode() || N00.getMachineOpcode() != ExpectedOpc)
return false;
@@ -1595,8 +1636,8 @@ bool X86DAGToDAGISel::tryOptimizeRem8Extend(SDNode *N) {
if (Opc == X86::MOVSX64rr8) {
// If we had a sign extend from 8 to 64 bits. We still need to go from 32
// to 64.
- MachineSDNode *Extend = CurDAG->getMachineNode(X86::MOVSX64rr32, SDLoc(N),
- MVT::i64, N00);
+ MachineSDNode *Extend =
+ CurDAG->getMachineNode(X86::MOVSX64rr32, SDLoc(N), MVT::i64, N00);
ReplaceUses(N, Extend);
} else {
// Ok we can drop this extend and just use the original extend.
@@ -1769,40 +1810,40 @@ void X86DAGToDAGISel::PostprocessISelDAG() {
switch (Move.getMachineOpcode()) {
default:
continue;
- CASE(VMOVAPDrr) CASE(VMOVUPDrr)
- CASE(VMOVAPSrr) CASE(VMOVUPSrr)
- CASE(VMOVDQArr) CASE(VMOVDQUrr)
- CASE(VMOVAPDYrr) CASE(VMOVUPDYrr)
- CASE(VMOVAPSYrr) CASE(VMOVUPSYrr)
- CASE(VMOVDQAYrr) CASE(VMOVDQUYrr)
- CASE(VMOVAPDZ128rr) CASE(VMOVUPDZ128rr)
- CASE(VMOVAPSZ128rr) CASE(VMOVUPSZ128rr)
- CASE(VMOVDQA32Z128rr) CASE(VMOVDQU32Z128rr)
- CASE(VMOVDQA64Z128rr) CASE(VMOVDQU64Z128rr)
- CASE(VMOVAPDZ256rr) CASE(VMOVUPDZ256rr)
- CASE(VMOVAPSZ256rr) CASE(VMOVUPSZ256rr)
- CASE(VMOVDQA32Z256rr) CASE(VMOVDQU32Z256rr)
- CASE(VMOVDQA64Z256rr) CASE(VMOVDQU64Z256rr)
+ CASE(VMOVAPDrr)
+ CASE(VMOVUPDrr) CASE(VMOVAPSrr) CASE(VMOVUPSrr) CASE(VMOVDQArr)
+ CASE(VMOVDQUrr) CASE(VMOVAPDYrr) CASE(VMOVUPDYrr) CASE(VMOVAPSYrr)
+ CASE(VMOVUPSYrr) CASE(VMOVDQAYrr) CASE(VMOVDQUYrr)
+ CASE(VMOVAPDZ128rr) CASE(VMOVUPDZ128rr) CASE(VMOVAPSZ128rr)
+ CASE(VMOVUPSZ128rr) CASE(VMOVDQA32Z128rr)
+ CASE(VMOVDQU32Z128rr) CASE(VMOVDQA64Z128rr)
+ CASE(VMOVDQU64Z128rr) CASE(VMOVAPDZ256rr)
+ CASE(VMOVUPDZ256rr) CASE(VMOVAPSZ256rr)
+ CASE(VMOVUPSZ256rr)
+ CASE(VMOVDQA32Z256rr)
+ CASE(VMOVDQU32Z256rr)
+ CASE(VMOVDQA64Z256rr)
+ CASE(VMOVDQU64Z256rr)
}
#undef CASE
- SDValue In = Move.getOperand(0);
- if (!In.isMachineOpcode() ||
- In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END)
- continue;
+ SDValue In = Move.getOperand(0);
+ if (!In.isMachineOpcode() ||
+ In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END)
+ continue;
- // Make sure the instruction has a VEX, XOP, or EVEX prefix. This covers
- // the SHA instructions which use a legacy encoding.
- uint64_t TSFlags = getInstrInfo()->get(In.getMachineOpcode()).TSFlags;
- if ((TSFlags & X86II::EncodingMask) != X86II::VEX &&
- (TSFlags & X86II::EncodingMask) != X86II::EVEX &&
- (TSFlags & X86II::EncodingMask) != X86II::XOP)
- continue;
+ // Make sure the instruction has a VEX, XOP, or EVEX prefix. This covers
+ // the SHA instructions which use a legacy encoding.
+ uint64_t TSFlags = getInstrInfo()->get(In.getMachineOpcode()).TSFlags;
+ if ((TSFlags & X86II::EncodingMask) != X86II::VEX &&
+ (TSFlags & X86II::EncodingMask) != X86II::EVEX &&
+ (TSFlags & X86II::EncodingMask) != X86II::XOP)
+ continue;
- // Producing instruction is another vector instruction. We can drop the
- // move.
- CurDAG->UpdateNodeOperands(N, In, N->getOperand(1));
- MadeChange = true;
+ // Producing instruction is another vector instruction. We can drop the
+ // move.
+ CurDAG->UpdateNodeOperands(N, In, N->getOperand(1));
+ MadeChange = true;
}
}
}
@@ -1811,7 +1852,6 @@ void X86DAGToDAGISel::PostprocessISelDAG() {
CurDAG->RemoveDeadNodes();
}
-
/// Emit any code that needs to be executed only in the main function.
void X86DAGToDAGISel::emitSpecialCodeForMain() {
if (Subtarget->isTargetCygMing()) {
@@ -1860,9 +1900,8 @@ bool X86DAGToDAGISel::foldOffsetIntoAddress(uint64_t Offset,
CodeModel::Model M = TM.getCodeModel();
if (Subtarget->is64Bit()) {
- if (Val != 0 &&
- !X86::isOffsetSuitableForCodeModel(Val, M,
- AM.hasSymbolicDisplacement()))
+ if (Val != 0 && !X86::isOffsetSuitableForCodeModel(
+ Val, M, AM.hasSymbolicDisplacement()))
return true;
// In addition to the checks required for a register base, check that
// we do not try to use an unsafe Disp with a frame index.
@@ -2037,8 +2076,7 @@ bool X86DAGToDAGISel::matchAddress(SDValue N, X86ISelAddressMode &AM) {
// Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has
// a smaller encoding and avoids a scaled-index.
- if (AM.Scale == 2 &&
- AM.BaseType == X86ISelAddressMode::RegBase &&
+ if (AM.Scale == 2 && AM.BaseType == X86ISelAddressMode::RegBase &&
AM.Base_Reg.getNode() == nullptr) {
AM.Base_Reg = AM.IndexReg;
AM.Scale = 1;
@@ -2187,8 +2225,7 @@ bool X86DAGToDAGISel::matchAdd(SDValue &N, X86ISelAddressMode &AM,
// If we couldn't fold both operands into the address at the same time,
// see if we can just put each operand into a register and fold at least
// the add.
- if (AM.BaseType == X86ISelAddressMode::RegBase &&
- !AM.Base_Reg.getNode() &&
+ if (AM.BaseType == X86ISelAddressMode::RegBase && !AM.Base_Reg.getNode() &&
!AM.IndexReg.getNode()) {
N = Handle.getValue();
AM.Base_Reg = N.getOperand(0);
@@ -2224,17 +2261,14 @@ static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N) {
// extract and a scaled index. Returns false if the simplification is
// performed.
static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N,
- uint64_t Mask,
- SDValue Shift, SDValue X,
+ uint64_t Mask, SDValue Shift, SDValue X,
X86ISelAddressMode &AM) {
if (Shift.getOpcode() != ISD::SRL ||
- !isa<ConstantSDNode>(Shift.getOperand(1)) ||
- !Shift.hasOneUse())
+ !isa<ConstantSDNode>(Shift.getOperand(1)) || !Shift.hasOneUse())
return true;
int ScaleLog = 8 - Shift.getConstantOperandVal(1);
- if (ScaleLog <= 0 || ScaleLog >= 4 ||
- Mask != (0xffu << ScaleLog))
+ if (ScaleLog <= 0 || ScaleLog >= 4 || Mask != (0xffu << ScaleLog))
return true;
MVT XVT = X.getSimpleValueType();
@@ -2362,8 +2396,7 @@ static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N,
// Note that this function assumes the mask is provided as a mask *after* the
// value is shifted. The input chain may or may not match that, but computing
// such a mask is trivial.
-static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N,
- uint64_t Mask,
+static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, uint64_t Mask,
SDValue Shift, SDValue X,
X86ISelAddressMode &AM) {
if (Shift.getOpcode() != ISD::SRL || !Shift.hasOneUse() ||
@@ -2384,7 +2417,8 @@ static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N,
// There is nothing we can do here unless the mask is removing some bits.
// Also, the addressing mode can only represent shifts of 1, 2, or 3 bits.
- if (AMShiftAmt == 0 || AMShiftAmt > 3) return true;
+ if (AMShiftAmt == 0 || AMShiftAmt > 3)
+ return true;
// Scale the leading zero count down based on the actual size of the value.
// Also scale it down based on the size of the shift.
@@ -2410,7 +2444,7 @@ static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N,
ReplacingAnyExtend = true;
}
APInt MaskedHighBits =
- APInt::getHighBitsSet(X.getSimpleValueType().getSizeInBits(), MaskLZ);
+ APInt::getHighBitsSet(X.getSimpleValueType().getSizeInBits(), MaskLZ);
if (!DAG.MaskedValueIsZero(X, MaskedHighBits))
return true;
@@ -2454,19 +2488,17 @@ static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N,
// Transform "(X >> SHIFT) & (MASK << C1)" to
// "((X >> (SHIFT + C1)) & (MASK)) << C1". Everything before the SHL will be
// matched to a BEXTR later. Returns false if the simplification is performed.
-static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N,
- uint64_t Mask,
+static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N, uint64_t Mask,
SDValue Shift, SDValue X,
X86ISelAddressMode &AM,
const X86Subtarget &Subtarget) {
if (Shift.getOpcode() != ISD::SRL ||
- !isa<ConstantSDNode>(Shift.getOperand(1)) ||
- !Shift.hasOneUse() || !N.hasOneUse())
+ !isa<ConstantSDNode>(Shift.getOperand(1)) || !Shift.hasOneUse() ||
+ !N.hasOneUse())
return true;
// Only do this if BEXTR will be matched by matchBEXTRFromAndImm.
- if (!Subtarget.hasTBM() &&
- !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR()))
+ if (!Subtarget.hasTBM() && !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR()))
return true;
// We need to ensure that mask is a continuous run of bits.
@@ -2482,7 +2514,8 @@ static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N,
// There is nothing we can do here unless the mask is removing some bits.
// Also, the addressing mode can only represent shifts of 1, 2, or 3 bits.
- if (AMShiftAmt == 0 || AMShiftAmt > 3) return true;
+ if (AMShiftAmt == 0 || AMShiftAmt > 3)
+ return true;
MVT XVT = X.getSimpleValueType();
MVT VT = N.getSimpleValueType();
@@ -2664,7 +2697,8 @@ bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
}
switch (N.getOpcode()) {
- default: break;
+ default:
+ break;
case ISD::LOCAL_RECOVER: {
if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
if (const auto *ESNode = dyn_cast<MCSymbolSDNode>(N.getOperand(0))) {
@@ -2723,7 +2757,8 @@ bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
case ISD::SRL: {
// Scale must not be used already.
- if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break;
+ if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1)
+ break;
// We only handle up to 64-bit values here as those are what matter for
// addressing mode optimizations.
@@ -2731,7 +2766,8 @@ bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
"Unexpected value size!");
SDValue And = N.getOperand(0);
- if (And.getOpcode() != ISD::AND) break;
+ if (And.getOpcode() != ISD::AND)
+ break;
SDValue X = And.getOperand(0);
// The mask used for the transform is expected to be post-shift, but we
@@ -2752,18 +2788,18 @@ bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
case ISD::SMUL_LOHI:
case ISD::UMUL_LOHI:
// A mul_lohi where we need the low part can be folded as a plain multiply.
- if (N.getResNo() != 0) break;
+ if (N.getResNo() != 0)
+ break;
[[fallthrough]];
case ISD::MUL:
case X86ISD::MUL_IMM:
// X*[3,5,9] -> X+X*[2,4,8]
if (AM.BaseType == X86ISelAddressMode::RegBase &&
- AM.Base_Reg.getNode() == nullptr &&
- AM.IndexReg.getNode() == nullptr) {
+ AM.Base_Reg.getNode() == nullptr && AM.IndexReg.getNode() == nullptr) {
if (auto *CN = dyn_cast<ConstantSDNode>(N.getOperand(1)))
if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
CN->getZExtValue() == 9) {
- AM.Scale = unsigned(CN->getZExtValue())-1;
+ AM.Scale = unsigned(CN->getZExtValue()) - 1;
SDValue MulVal = N.getOperand(0);
SDValue Reg;
@@ -2802,7 +2838,7 @@ bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
// Test if the LHS of the sub can be folded.
X86ISelAddressMode Backup = AM;
- if (matchAddressRecursively(N.getOperand(0), AM, Depth+1)) {
+ if (matchAddressRecursively(N.getOperand(0), AM, Depth + 1)) {
N = Handle.getValue();
AM = Backup;
break;
@@ -2835,8 +2871,9 @@ bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
// If the folded LHS was interesting, this transformation saves
// address arithmetic.
if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
- ((AM.Disp != 0) && (Backup.Disp == 0)) +
- (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
+ ((AM.Disp != 0) && (Backup.Disp == 0)) +
+ (AM.Segment.getNode() && !Backup.Segment.getNode()) >=
+ 2)
--Cost;
// If it doesn't look like it may be an overall win, don't do it.
if (Cost >= 0) {
@@ -2869,7 +2906,8 @@ bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
// with a constant to enable use of the scaled offset field.
// Scale must not be used already.
- if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break;
+ if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1)
+ break;
// We only handle up to 64-bit values here as those are what matter for
// addressing mode optimizations.
@@ -3134,14 +3172,14 @@ bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base,
// This list of opcodes are all the nodes that have an "addr:$ptr" operand
// that are not a MemSDNode, and thus don't have proper addrspace info.
Parent->getOpcode() != ISD::INTRINSIC_W_CHAIN && // unaligned loads, fixme
- Parent->getOpcode() != ISD::INTRINSIC_VOID && // nontemporal stores
- Parent->getOpcode() != X86ISD::TLSCALL && // Fixme
- Parent->getOpcode() != X86ISD::ENQCMD && // Fixme
- Parent->getOpcode() != X86ISD::ENQCMDS && // Fixme
+ Parent->getOpcode() != ISD::INTRINSIC_VOID && // nontemporal stores
+ Parent->getOpcode() != X86ISD::TLSCALL && // Fixme
+ Parent->getOpcode() != X86ISD::ENQCMD && // Fixme
+ Parent->getOpcode() != X86ISD::ENQCMDS && // Fixme
Parent->getOpcode() != X86ISD::EH_SJLJ_SETJMP && // setjmp
Parent->getOpcode() != X86ISD::EH_SJLJ_LONGJMP) { // longjmp
unsigned AddrSpace =
- cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace();
+ cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace();
if (AddrSpace == X86AS::GS)
AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
if (AddrSpace == X86AS::FS)
@@ -3230,8 +3268,8 @@ bool X86DAGToDAGISel::selectLEA64_Addr(SDValue N, SDValue &Base, SDValue &Scale,
BaseType == MVT::i32) &&
!isa<FrameIndexSDNode>(Base)) {
// Base could already be %rip, particularly in the x32 ABI.
- SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL,
- MVT::i64), 0);
+ SDValue ImplDef =
+ SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL, MVT::i64), 0);
Base = CurDAG->getTargetInsertSubreg(SubReg, DL, MVT::i64, ImplDef, Base);
}
@@ -3242,8 +3280,8 @@ bool X86DAGToDAGISel::selectLEA64_Addr(SDValue N, SDValue &Base, SDValue &Scale,
else {
assert((IndexType == BaseType) &&
"Expect to be extending 8/16/32-bit registers for use in LEA");
- SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL,
- MVT::i64), 0);
+ SDValue ImplDef =
+ SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL, MVT::i64), 0);
Index = CurDAG->getTargetInsertSubreg(SubReg, DL, MVT::i64, ImplDef, Index);
}
@@ -3252,8 +3290,7 @@ bool X86DAGToDAGISel::selectLEA64_Addr(SDValue N, SDValue &Base, SDValue &Scale,
/// Calls SelectAddr and determines if the maximal addressing
/// mode it matches can be cost effectively emitted as an LEA instruction.
-bool X86DAGToDAGISel::selectLEAAddr(SDValue N,
- SDValue &Base, SDValue &Scale,
+bool X86DAGToDAGISel::selectLEAAddr(SDValue N, SDValue &Base, SDValue &Scale,
SDValue &Index, SDValue &Disp,
SDValue &Segment) {
X86ISelAddressMode AM;
@@ -3270,7 +3307,7 @@ bool X86DAGToDAGISel::selectLEAAddr(SDValue N,
AM.Segment = T;
if (matchAddress(N, AM))
return false;
- assert (T == AM.Segment);
+ assert(T == AM.Segment);
AM.Segment = Copy;
unsigned Complexity = 0;
@@ -3312,12 +3349,12 @@ bool X86DAGToDAGISel::selectLEAAddr(SDValue N,
case X86ISD::SBB:
case X86ISD::SMUL:
case X86ISD::UMUL:
- /* TODO: These opcodes can be added safely, but we may want to justify
- their inclusion for different reasons (better for reg-alloc).
- case X86ISD::OR:
- case X86ISD::XOR:
- case X86ISD::AND:
- */
+ /* TODO: These opcodes can be added safely, but we may want to justify
+ their inclusion for different reasons (better for reg-alloc).
+ case X86ISD::OR:
+ case X86ISD::XOR:
+ case X86ISD::AND:
+ */
// Value 1 is the flag output of the node - verify it's not dead.
return !SDValue(V.getNode(), 1).use_empty();
default:
@@ -3407,17 +3444,15 @@ bool X86DAGToDAGISel::selectRelocImm(SDValue N, SDValue &Op) {
}
bool X86DAGToDAGISel::tryFoldLoad(SDNode *Root, SDNode *P, SDValue N,
- SDValue &Base, SDValue &Scale,
- SDValue &Index, SDValue &Disp,
- SDValue &Segment) {
+ SDValue &Base, SDValue &Scale, SDValue &Index,
+ SDValue &Disp, SDValue &Segment) {
assert(Root && P && "Unknown root/parent nodes");
- if (!ISD::isNON_EXTLoad(N.getNode()) ||
- !IsProfitableToFold(N, P, Root) ||
+ if (!ISD::isNON_EXTLoad(N.getNode()) || !IsProfitableToFold(N, P, Root) ||
!IsLegalToFold(N, P, Root, OptLevel))
return false;
- return selectAddr(N.getNode(),
- N.getOperand(1), Base, Scale, Index, Disp, Segment);
+ return selectAddr(N.getNode(), N.getOperand(1), Base, Scale, Index, Disp,
+ Segment);
}
bool X86DAGToDAGISel::tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N,
@@ -3426,12 +3461,11 @@ bool X86DAGToDAGISel::tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N,
SDValue &Segment) {
assert(Root && P && "Unknown root/parent nodes");
if (N->getOpcode() != X86ISD::VBROADCAST_LOAD ||
- !IsProfitableToFold(N, P, Root) ||
- !IsLegalToFold(N, P, Root, OptLevel))
+ !IsProfitableToFold(N, P, Root) || !IsLegalToFold(N, P, Root, OptLevel))
return false;
- return selectAddr(N.getNode(),
- N.getOperand(1), Base, Scale, Index, Disp, Segment);
+ return selectAddr(N.getNode(), N.getOperand(1), Base, Scale, Index, Disp,
+ Segment);
}
/// Return an SDNode that returns the value of the global base register.
@@ -3503,7 +3537,8 @@ bool X86DAGToDAGISel::onlyUsesZeroFlag(SDValue Flags) const {
switch (CC) {
// Comparisons which only use the zero flag.
- case X86::COND_E: case X86::COND_NE:
+ case X86::COND_E:
+ case X86::COND_NE:
continue;
// Anything else: assume conservatively.
default:
@@ -3540,11 +3575,16 @@ bool X86DAGToDAGISel::hasNoSignFlagUses(SDValue Flags) const {
switch (CC) {
// Comparisons which don't examine the SF flag.
- case X86::COND_A: case X86::COND_AE:
- case X86::COND_B: case X86::COND_BE:
- case X86::COND_E: case X86::COND_NE:
- case X86::COND_O: case X86::COND_NO:
- case X86::COND_P: case X86::COND_NP:
+ case X86::COND_A:
+ case X86::COND_AE:
+ case X86::COND_B:
+ case X86::COND_BE:
+ case X86::COND_E:
+ case X86::COND_NE:
+ case X86::COND_O:
+ case X86::COND_NO:
+ case X86::COND_P:
+ case X86::COND_NP:
continue;
// Anything else: assume conservatively.
default:
@@ -3558,12 +3598,18 @@ bool X86DAGToDAGISel::hasNoSignFlagUses(SDValue Flags) const {
static bool mayUseCarryFlag(X86::CondCode CC) {
switch (CC) {
// Comparisons which don't examine the CF flag.
- case X86::COND_O: case X86::COND_NO:
- case X86::COND_E: case X86::COND_NE:
- case X86::COND_S: case X86::COND_NS:
- case X86::COND_P: case X86::COND_NP:
- case X86::COND_L: case X86::COND_GE:
- case X86::COND_G: case X86::COND_LE:
+ case X86::COND_O:
+ case X86::COND_NO:
+ case X86::COND_E:
+ case X86::COND_NE:
+ case X86::COND_S:
+ case X86::COND_NS:
+ case X86::COND_P:
+ case X86::COND_NP:
+ case X86::COND_L:
+ case X86::COND_GE:
+ case X86::COND_G:
+ case X86::COND_LE:
return false;
// Anything else: assume conservatively.
default:
@@ -3573,7 +3619,7 @@ static bool mayUseCarryFlag(X86::CondCode CC) {
/// Test whether the given node which sets flags has any uses which require the
/// CF flag to be accurate.
- bool X86DAGToDAGISel::hasNoCarryFlagUses(SDValue Flags) const {
+bool X86DAGToDAGISel::hasNoCarryFlagUses(SDValue Flags) const {
// Examine each user of the node.
for (SDUse &Use : Flags->uses()) {
// Only check things that use the flags.
@@ -3613,10 +3659,18 @@ static bool mayUseCarryFlag(X86::CondCode CC) {
default:
// Something unusual. Be conservative.
return false;
- case X86ISD::SETCC: CCOpNo = 0; break;
- case X86ISD::SETCC_CARRY: CCOpNo = 0; break;
- case X86ISD::CMOV: CCOpNo = 2; break;
- case X86ISD::BRCOND: CCOpNo = 2; break;
+ case X86ISD::SETCC:
+ CCOpNo = 0;
+ break;
+ case X86ISD::SETCC_CARRY:
+ CCOpNo = 0;
+ break;
+ case X86ISD::CMOV:
+ CCOpNo = 2;
+ break;
+ case X86ISD::BRCOND:
+ CCOpNo = 2;
+ break;
}
X86::CondCode CC = (X86::CondCode)User->getConstantOperandVal(CCOpNo);
@@ -3694,10 +3748,12 @@ static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode,
LoadSDNode *&LoadNode,
SDValue &InputChain) {
// Is the stored value result 0 of the operation?
- if (StoredVal.getResNo() != 0) return false;
+ if (StoredVal.getResNo() != 0)
+ return false;
// Are there other uses of the operation other than the store?
- if (!StoredVal.getNode()->hasNUsesOfValue(1, 0)) return false;
+ if (!StoredVal.getNode()->hasNUsesOfValue(1, 0))
+ return false;
// Is the store non-extending and non-indexed?
if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal())
@@ -3705,7 +3761,8 @@ static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode,
SDValue Load = StoredVal->getOperand(LoadOpNo);
// Is the stored value a non-extending and non-indexed load?
- if (!ISD::isNormalLoad(Load.getNode())) return false;
+ if (!ISD::isNormalLoad(Load.getNode()))
+ return false;
// Return LoadNode by reference.
LoadNode = cast<LoadSDNode>(Load);
@@ -3892,14 +3949,14 @@ bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(SDNode *Node) {
case X86ISD::SUB:
// Handle negate.
if (IsNegate) {
- unsigned NewOpc = SelectOpcode(X86::NEG64m, X86::NEG32m, X86::NEG16m,
- X86::NEG8m);
+ unsigned NewOpc =
+ SelectOpcode(X86::NEG64m, X86::NEG32m, X86::NEG16m, X86::NEG8m);
const SDValue Ops[] = {Base, Scale, Index, Disp, Segment, InputChain};
- Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32,
- MVT::Other, Ops);
+ Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32, MVT::Other,
+ Ops);
break;
}
- [[fallthrough]];
+ [[fallthrough]];
case X86ISD::ADD:
// Try to match inc/dec.
if (!Subtarget->slowIncDec() || CurDAG->shouldOptForSize()) {
@@ -3907,10 +3964,11 @@ bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(SDNode *Node) {
bool IsNegOne = isAllOnesConstant(StoredVal.getOperand(1));
// ADD/SUB with 1/-1 and carry flag isn't used can use inc/dec.
if ((IsOne || IsNegOne) && hasNoCarryFlagUses(StoredVal.getValue(1))) {
- unsigned NewOpc =
- ((Opc == X86ISD::ADD) == IsOne)
- ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m)
- : SelectOpcode(X86::DEC64m, X86::DEC32m, X86::DEC16m, X86::DEC8m);
+ unsigned NewOpc = ((Opc == X86ISD::ADD) == IsOne)
+ ? SelectOpcode(X86::INC64m, X86::INC32m,
+ X86::INC16m, X86::INC8m)
+ : SelectOpcode(X86::DEC64m, X86::DEC32m,
+ X86::DEC16m, X86::DEC8m);
const SDValue Ops[] = {Base, Scale, Index, Disp, Segment, InputChain};
Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32,
MVT::Other, Ops);
@@ -3978,7 +4036,7 @@ bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(SDNode *Node) {
};
unsigned NewOpc = SelectRegOpcode(Opc);
- SDValue Operand = StoredVal->getOperand(1-LoadOpNo);
+ SDValue Operand = StoredVal->getOperand(1 - LoadOpNo);
// See if the operand is a constant that we can fold into an immediate
// operand.
@@ -4065,12 +4123,12 @@ bool X86DAGToDAGISel::matchBitExtract(SDNode *Node) {
// If we have BMI2's BZHI, we are ok with muti-use patterns.
// Else, if we only have BMI1's BEXTR, we require one-use.
const bool AllowExtraUsesByDefault = Subtarget->hasBMI2();
- auto checkUses = [AllowExtraUsesByDefault](
- SDValue Op, unsigned NUses,
- std::optional<bool> AllowExtraUses) {
- return AllowExtraUses.value_or(AllowExtraUsesByDefault) ||
- Op.getNode()->hasNUsesOfValue(NUses, Op.getResNo());
- };
+ auto checkUses =
+ [AllowExtraUsesByDefault](SDValue Op, unsigned NUses,
+ std::optional<bool> AllowExtraUses) {
+ return AllowExtraUses.value_or(AllowExtraUsesByDefault) ||
+ Op.getNode()->hasNUsesOfValue(NUses, Op.getResNo());
+ };
auto checkOneUse = [checkUses](SDValue Op,
std::optional<bool> AllowExtraUses =
std::nullopt) {
@@ -4506,8 +4564,7 @@ MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(unsigned ROpc, unsigned MOpc,
// Try to fold a load. No need to check alignment.
SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
if (MayFoldLoad && tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
- SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
- N1.getOperand(0) };
+ SDValue Ops[] = {N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm, N1.getOperand(0)};
SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other);
MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
// Update the chain.
@@ -4517,7 +4574,7 @@ MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(unsigned ROpc, unsigned MOpc,
return CNode;
}
- SDValue Ops[] = { N0, N1, Imm };
+ SDValue Ops[] = {N0, N1, Imm};
SDVTList VTs = CurDAG->getVTList(VT, MVT::i32);
MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
return CNode;
@@ -4539,8 +4596,8 @@ MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(unsigned ROpc, unsigned MOpc,
// Try to fold a load. No need to check alignment.
SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
if (MayFoldLoad && tryFoldLoad(Node, N2, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
- SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
- N2.getOperand(0), InGlue };
+ SDValue Ops[] = {N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm, N2.getOperand(0),
+ InGlue};
SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other, MVT::Glue);
MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
InGlue = SDValue(CNode, 3);
@@ -4551,7 +4608,7 @@ MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(unsigned ROpc, unsigned MOpc,
return CNode;
}
- SDValue Ops[] = { N0, N2, Imm, InGlue };
+ SDValue Ops[] = {N0, N2, Imm, InGlue};
SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Glue);
MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
InGlue = SDValue(CNode, 2);
@@ -4665,8 +4722,8 @@ bool X86DAGToDAGISel::tryShiftAmountMod(SDNode *N) {
// Place in a correct topological ordering.
insertDAGNode(*CurDAG, OrigShiftAmt, NewShiftAmt);
- SDNode *UpdatedNode = CurDAG->UpdateNodeOperands(N, N->getOperand(0),
- NewShiftAmt);
+ SDNode *UpdatedNode =
+ CurDAG->UpdateNodeOperands(N, N->getOperand(0), NewShiftAmt);
if (UpdatedNode != N) {
// If we found an existing node, we should replace ourselves with that node
// and wait for it to be selected after its other users.
@@ -4706,8 +4763,7 @@ bool X86DAGToDAGISel::tryShrinkShlLogicImm(SDNode *N) {
// FIXME: Generalize this to other ANY_EXTEND than i32 to i64?
bool FoundAnyExtend = false;
if (Shift.getOpcode() == ISD::ANY_EXTEND && Shift.hasOneUse() &&
- Shift.getOperand(0).getSimpleValueType() == MVT::i32 &&
- isUInt<32>(Val)) {
+ Shift.getOperand(0).getSimpleValueType() == MVT::i32 && isUInt<32>(Val)) {
FoundAnyExtend = true;
Shift = Shift.getOperand(0);
}
@@ -4772,8 +4828,7 @@ bool X86DAGToDAGISel::tryShrinkShlLogicImm(SDNode *N) {
ZExtWidth = llvm::bit_ceil(std::max(ZExtWidth, 8U));
// Figure out which bits need to be zero to achieve that mask.
- APInt NeededMask = APInt::getLowBitsSet(NVT.getSizeInBits(),
- ZExtWidth);
+ APInt NeededMask = APInt::getLowBitsSet(NVT.getSizeInBits(), ZExtWidth);
NeededMask &= ~Cst->getAPIntValue();
if (CurDAG->MaskedValueIsZero(N->getOperand(0), NeededMask))
@@ -4791,8 +4846,8 @@ bool X86DAGToDAGISel::tryShrinkShlLogicImm(SDNode *N) {
insertDAGNode(*CurDAG, SDValue(N, 0), NewCst);
SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT, X, NewCst);
insertDAGNode(*CurDAG, SDValue(N, 0), NewBinOp);
- SDValue NewSHL = CurDAG->getNode(ISD::SHL, dl, NVT, NewBinOp,
- Shift.getOperand(1));
+ SDValue NewSHL =
+ CurDAG->getNode(ISD::SHL, dl, NVT, NewBinOp, Shift.getOperand(1));
ReplaceNode(N, NewSHL.getNode());
SelectCode(NewSHL.getNode());
return true;
@@ -4840,10 +4895,14 @@ bool X86DAGToDAGISel::matchVPTERNLOG(SDNode *Root, SDNode *ParentA,
// Swap bits 1/4 and 3/6.
uint8_t OldImm = Imm;
Imm = OldImm & 0xa5;
- if (OldImm & 0x02) Imm |= 0x10;
- if (OldImm & 0x10) Imm |= 0x02;
- if (OldImm & 0x08) Imm |= 0x40;
- if (OldImm & 0x40) Imm |= 0x08;
+ if (OldImm & 0x02)
+ Imm |= 0x10;
+ if (OldImm & 0x10)
+ Imm |= 0x02;
+ if (OldImm & 0x08)
+ Imm |= 0x40;
+ if (OldImm & 0x40)
+ Imm |= 0x08;
} else if (tryFoldLoadOrBCast(Root, ParentB, B, Tmp0, Tmp1, Tmp2, Tmp3,
Tmp4)) {
FoldedLoad = true;
@@ -4851,10 +4910,14 @@ bool X86DAGToDAGISel::matchVPTERNLOG(SDNode *Root, SDNode *ParentA,
// Swap bits 1/2 and 5/6.
uint8_t OldImm = Imm;
Imm = OldImm & 0x99;
- if (OldImm & 0x02) Imm |= 0x04;
- if (OldImm & 0x04) Imm |= 0x02;
- if (OldImm & 0x20) Imm |= 0x40;
- if (OldImm & 0x40) Imm |= 0x20;
+ if (OldImm & 0x02)
+ Imm |= 0x04;
+ if (OldImm & 0x04)
+ Imm |= 0x02;
+ if (OldImm & 0x20)
+ Imm |= 0x40;
+ if (OldImm & 0x40)
+ Imm |= 0x20;
}
SDLoc DL(Root);
@@ -5023,24 +5086,40 @@ bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
uint8_t Imm;
switch (FoldableOp.getOpcode()) {
- default: llvm_unreachable("Unexpected opcode!");
- case ISD::AND: Imm = TernlogMagicB & TernlogMagicC; break;
- case ISD::OR: Imm = TernlogMagicB | TernlogMagicC; break;
- case ISD::XOR: Imm = TernlogMagicB ^ TernlogMagicC; break;
- case X86ISD::ANDNP: Imm = ~(TernlogMagicB) & TernlogMagicC; break;
+ default:
+ llvm_unreachable("Unexpected opcode!");
+ case ISD::AND:
+ Imm = TernlogMagicB & TernlogMagicC;
+ break;
+ case ISD::OR:
+ Imm = TernlogMagicB | TernlogMagicC;
+ break;
+ case ISD::XOR:
+ Imm = TernlogMagicB ^ TernlogMagicC;
+ break;
+ case X86ISD::ANDNP:
+ Imm = ~(TernlogMagicB)&TernlogMagicC;
+ break;
}
switch (N->getOpcode()) {
- default: llvm_unreachable("Unexpected opcode!");
+ default:
+ llvm_unreachable("Unexpected opcode!");
case X86ISD::ANDNP:
if (A == N0)
Imm &= ~TernlogMagicA;
else
- Imm = ~(Imm) & TernlogMagicA;
+ Imm = ~(Imm)&TernlogMagicA;
+ break;
+ case ISD::AND:
+ Imm &= TernlogMagicA;
+ break;
+ case ISD::OR:
+ Imm |= TernlogMagicA;
+ break;
+ case ISD::XOR:
+ Imm ^= TernlogMagicA;
break;
- case ISD::AND: Imm &= TernlogMagicA; break;
- case ISD::OR: Imm |= TernlogMagicA; break;
- case ISD::XOR: Imm ^= TernlogMagicA; break;
}
if (PeeledOuterNot)
@@ -5124,46 +5203,40 @@ bool X86DAGToDAGISel::shrinkAndImmediate(SDNode *And) {
static unsigned getVPTESTMOpc(MVT TestVT, bool IsTestN, bool FoldedLoad,
bool FoldedBCast, bool Masked) {
-#define VPTESTM_CASE(VT, SUFFIX) \
-case MVT::VT: \
- if (Masked) \
- return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \
- return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;
-
-
-#define VPTESTM_BROADCAST_CASES(SUFFIX) \
-default: llvm_unreachable("Unexpected VT!"); \
-VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
-VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
-VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
-VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
-VPTESTM_CASE(v16i32, DZ##SUFFIX) \
-VPTESTM_CASE(v8i64, QZ##SUFFIX)
-
-#define VPTESTM_FULL_CASES(SUFFIX) \
-VPTESTM_BROADCAST_CASES(SUFFIX) \
-VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
-VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
-VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
-VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
-VPTESTM_CASE(v64i8, BZ##SUFFIX) \
-VPTESTM_CASE(v32i16, WZ##SUFFIX)
+#define VPTESTM_CASE(VT, SUFFIX) \
+ case MVT::VT: \
+ if (Masked) \
+ return IsTestN ? X86::VPTESTNM##SUFFIX##k : X86::VPTESTM##SUFFIX##k; \
+ return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;
+
+#define VPTESTM_BROADCAST_CASES(SUFFIX) \
+ default: \
+ llvm_unreachable("Unexpected VT!"); \
+ VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
+ VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
+ VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
+ VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
+ VPTESTM_CASE(v16i32, DZ##SUFFIX) \
+ VPTESTM_CASE(v8i64, QZ##SUFFIX)
+
+#define VPTESTM_FULL_CASES(SUFFIX) \
+ VPTESTM_BROADCAST_CASES(SUFFIX) \
+ VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
+ VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
+ VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
+ VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
+ VPTESTM_CASE(v64i8, BZ##SUFFIX) \
+ VPTESTM_CASE(v32i16, WZ##SUFFIX)
if (FoldedBCast) {
- switch (TestVT.SimpleTy) {
- VPTESTM_BROADCAST_CASES(rmb)
- }
+ switch (TestVT.SimpleTy) { VPTESTM_BROADCAST_CASES(rmb) }
}
if (FoldedLoad) {
- switch (TestVT.SimpleTy) {
- VPTESTM_FULL_CASES(rm)
- }
+ switch (TestVT.SimpleTy) { VPTESTM_FULL_CASES(rm) }
}
- switch (TestVT.SimpleTy) {
- VPTESTM_FULL_CASES(rr)
- }
+ switch (TestVT.SimpleTy) { VPTESTM_FULL_CASES(rr) }
#undef VPTESTM_FULL_CASES
#undef VPTESTM_BROADCAST_CASES
@@ -5187,8 +5260,7 @@ static void orderRegForMul(SDValue &N0, SDValue &N1, const unsigned LoReg,
// Try to create VPTESTM instruction. If InMask is not null, it will be used
// to form a masked operation.
-bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
- SDValue InMask) {
+bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc, SDValue InMask) {
assert(Subtarget->hasAVX512() && "Expected AVX512!");
assert(Setcc.getSimpleValueType().getVectorElementType() == MVT::i1 &&
"Unexpected VT!");
@@ -5224,7 +5296,7 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
if (N0Temp.getOpcode() == ISD::BITCAST && N0Temp.hasOneUse())
N0Temp = N0.getOperand(0);
- // Look for single use AND.
+ // Look for single use AND.
if (N0Temp.getOpcode() == ISD::AND && N0Temp.hasOneUse()) {
Src0 = N0Temp.getOperand(0);
Src1 = N0Temp.getOperand(1);
@@ -5295,8 +5367,8 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
unsigned NumElts = CmpVT.getVectorNumElements() * Scale;
CmpVT = MVT::getVectorVT(CmpSVT, NumElts);
MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
- SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl,
- CmpVT), 0);
+ SDValue ImplDef =
+ SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl, CmpVT), 0);
Src0 = CurDAG->getTargetInsertSubreg(SubReg, dl, CmpVT, ImplDef, Src0);
if (!FoldedBCast)
@@ -5307,25 +5379,25 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
unsigned RegClass = TLI->getRegClassFor(MaskVT)->getID();
SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
InMask = SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
- dl, MaskVT, InMask, RC), 0);
+ dl, MaskVT, InMask, RC),
+ 0);
}
}
bool IsTestN = CC == ISD::SETEQ;
- unsigned Opc = getVPTESTMOpc(CmpVT, IsTestN, FoldedLoad, FoldedBCast,
- IsMasked);
+ unsigned Opc =
+ getVPTESTMOpc(CmpVT, IsTestN, FoldedLoad, FoldedBCast, IsMasked);
MachineSDNode *CNode;
if (FoldedLoad) {
SDVTList VTs = CurDAG->getVTList(MaskVT, MVT::Other);
if (IsMasked) {
- SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
- Src1.getOperand(0) };
+ SDValue Ops[] = {InMask, Src0, Tmp0, Tmp1,
+ Tmp2, Tmp3, Tmp4, Src1.getOperand(0)};
CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
} else {
- SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
- Src1.getOperand(0) };
+ SDValue Ops[] = {Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Src1.getOperand(0)};
CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
}
@@ -5344,8 +5416,8 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
if (Widen) {
unsigned RegClass = TLI->getRegClassFor(ResVT)->getID();
SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
- CNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
- dl, ResVT, SDValue(CNode, 0), RC);
+ CNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, dl, ResVT,
+ SDValue(CNode, 0), RC);
}
ReplaceUses(SDValue(Root, 0), SDValue(CNode, 0));
@@ -5375,8 +5447,7 @@ bool X86DAGToDAGISel::tryMatchBitSelect(SDNode *N) {
if (N1.getOpcode() == ISD::AND)
std::swap(N0, N1);
- if (N0.getOpcode() != ISD::AND ||
- N1.getOpcode() != X86ISD::ANDNP ||
+ if (N0.getOpcode() != ISD::AND || N1.getOpcode() != X86ISD::ANDNP ||
!N0.hasOneUse() || !N1.hasOneUse())
return false;
@@ -5411,15 +5482,17 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
if (Node->isMachineOpcode()) {
LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << '\n');
Node->setNodeId(-1);
- return; // Already selected.
+ return; // Already selected.
}
switch (Opcode) {
- default: break;
+ default:
+ break;
case ISD::INTRINSIC_W_CHAIN: {
unsigned IntNo = Node->getConstantOperandVal(1);
switch (IntNo) {
- default: break;
+ default:
+ break;
case Intrinsic::x86_encodekey128:
case Intrinsic::x86_encodekey256: {
if (!Subtarget->hasKL())
@@ -5427,7 +5500,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
unsigned Opcode;
switch (IntNo) {
- default: llvm_unreachable("Impossible intrinsic");
+ default:
+ llvm_unreachable("Impossible intrinsic");
case Intrinsic::x86_encodekey128:
Opcode = X86::ENCODEKEY128;
break;
@@ -5504,7 +5578,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
case ISD::INTRINSIC_VOID: {
unsigned IntNo = Node->getConstantOperandVal(1);
switch (IntNo) {
- default: break;
+ default:
+ break;
case Intrinsic::x86_sse3_monitor:
case Intrinsic::x86_monitorx:
case Intrinsic::x86_clzero: {
@@ -5512,7 +5587,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
unsigned Opc = 0;
switch (IntNo) {
- default: llvm_unreachable("Unexpected intrinsic!");
+ default:
+ llvm_unreachable("Unexpected intrinsic!");
case Intrinsic::x86_sse3_monitor:
if (!Subtarget->hasSSE3())
break;
@@ -5547,8 +5623,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
InGlue = Chain.getValue(1);
}
- MachineSDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
- { Chain, InGlue});
+ MachineSDNode *CNode =
+ CurDAG->getMachineNode(Opc, dl, MVT::Other, {Chain, InGlue});
ReplaceNode(Node, CNode);
return;
}
@@ -5596,16 +5672,23 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
MFI->setAMXProgModel(AMXProgModelEnum::DirectReg);
unsigned Opc;
switch (IntNo) {
- default: llvm_unreachable("Unexpected intrinsic!");
- case Intrinsic::x86_tileloadd64: Opc = X86::PTILELOADD; break;
+ default:
+ llvm_unreachable("Unexpected intrinsic!");
+ case Intrinsic::x86_tileloadd64:
+ Opc = X86::PTILELOADD;
+ break;
case Intrinsic::x86_tileloaddrs64:
Opc = X86::PTILELOADDRS;
break;
- case Intrinsic::x86_tileloaddt164: Opc = X86::PTILELOADDT1; break;
+ case Intrinsic::x86_tileloaddt164:
+ Opc = X86::PTILELOADDT1;
+ break;
case Intrinsic::x86_tileloaddrst164:
Opc = X86::PTILELOADDRST1;
break;
- case Intrinsic::x86_tilestored64: Opc = X86::PTILESTORED; break;
+ case Intrinsic::x86_tilestored64:
+ Opc = X86::PTILESTORED;
+ break;
}
// FIXME: Match displacement and scale.
unsigned TIndex = Node->getConstantOperandVal(2);
@@ -5618,10 +5701,10 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
SDValue Chain = Node->getOperand(0);
MachineSDNode *CNode;
if (Opc == X86::PTILESTORED) {
- SDValue Ops[] = { Base, Scale, Index, Disp, Segment, TReg, Chain };
+ SDValue Ops[] = {Base, Scale, Index, Disp, Segment, TReg, Chain};
CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
} else {
- SDValue Ops[] = { TReg, Base, Scale, Index, Disp, Segment, Chain };
+ SDValue Ops[] = {TReg, Base, Scale, Index, Disp, Segment, Chain};
CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
}
ReplaceNode(Node, CNode);
@@ -5762,10 +5845,12 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
// We should not fold the immediate. So we need a register form instead.
unsigned ROpc, MOpc;
switch (NVT.SimpleTy) {
- default: llvm_unreachable("Unexpected VT!");
+ default:
+ llvm_unreachable("Unexpected VT!");
case MVT::i8:
switch (Opcode) {
- default: llvm_unreachable("Unexpected opcode!");
+ default:
+ llvm_unreachable("Unexpected opcode!");
case ISD::ADD:
ROpc = GET_ND_IF_ENABLED(X86::ADD8rr);
MOpc = GET_NDM_IF_ENABLED(X86::ADD8rm);
@@ -5790,7 +5875,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
break;
case MVT::i16:
switch (Opcode) {
- default: llvm_unreachable("Unexpected opcode!");
+ default:
+ llvm_unreachable("Unexpected opcode!");
case ISD::ADD:
ROpc = GET_ND_IF_ENABLED(X86::ADD16rr);
MOpc = GET_NDM_IF_ENABLED(X86::ADD16rm);
@@ -5815,7 +5901,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
break;
case MVT::i32:
switch (Opcode) {
- default: llvm_unreachable("Unexpected opcode!");
+ default:
+ llvm_unreachable("Unexpected opcode!");
case ISD::ADD:
ROpc = GET_ND_IF_ENABLED(X86::ADD32rr);
MOpc = GET_NDM_IF_ENABLED(X86::ADD32rm);
@@ -5840,7 +5927,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
break;
case MVT::i64:
switch (Opcode) {
- default: llvm_unreachable("Unexpected opcode!");
+ default:
+ llvm_unreachable("Unexpected opcode!");
case ISD::ADD:
ROpc = GET_ND_IF_ENABLED(X86::ADD64rr);
MOpc = GET_NDM_IF_ENABLED(X86::ADD64rm);
@@ -5871,7 +5959,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
if (Opcode != ISD::SUB) {
SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
if (tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
- SDValue Ops[] = { N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) };
+ SDValue Ops[] = {N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0)};
SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
// Update the chain.
@@ -5899,7 +5987,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
unsigned LoReg, ROpc, MOpc;
switch (NVT.SimpleTy) {
- default: llvm_unreachable("Unsupported VT!");
+ default:
+ llvm_unreachable("Unsupported VT!");
case MVT::i8:
LoReg = X86::AL;
ROpc = Opcode == X86ISD::SMUL ? X86::IMUL8r : X86::MUL8r;
@@ -5936,8 +6025,9 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
if (!FoldedLoad)
orderRegForMul(N0, N1, LoReg, CurDAG->getMachineFunction().getRegInfo());
- SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
- N0, SDValue()).getValue(1);
+ SDValue InGlue =
+ CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, N0, SDValue())
+ .getValue(1);
MachineSDNode *CNode;
if (FoldedLoad) {
@@ -5949,8 +6039,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
else
VTs = CurDAG->getVTList(NVT, NVT, MVT::i32, MVT::Other);
- SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
- InGlue };
+ SDValue Ops[] = {Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), InGlue};
CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
// Update the chain.
@@ -5986,7 +6075,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
bool UseMULX = !IsSigned && Subtarget->hasBMI2();
bool UseMULXHi = UseMULX && SDValue(Node, 0).use_empty();
switch (NVT.SimpleTy) {
- default: llvm_unreachable("Unsupported VT!");
+ default:
+ llvm_unreachable("Unsupported VT!");
case MVT::i32:
Opc = UseMULXHi ? X86::MULX32Hrr
: UseMULX ? GET_EGPR_IF_ENABLED(X86::MULX32rr)
@@ -6027,14 +6117,14 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
if (!foldedLoad)
orderRegForMul(N0, N1, LoReg, CurDAG->getMachineFunction().getRegInfo());
- SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
- N0, SDValue()).getValue(1);
+ SDValue InGlue =
+ CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, N0, SDValue())
+ .getValue(1);
SDValue ResHi, ResLo;
if (foldedLoad) {
SDValue Chain;
MachineSDNode *CNode = nullptr;
- SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
- InGlue };
+ SDValue Ops[] = {Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), InGlue};
if (UseMULXHi) {
SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
@@ -6058,7 +6148,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
// Record the mem-refs
CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
} else {
- SDValue Ops[] = { N1, InGlue };
+ SDValue Ops[] = {N1, InGlue};
if (UseMULXHi) {
SDVTList VTs = CurDAG->getVTList(NVT);
SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
@@ -6079,8 +6169,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
if (!SDValue(Node, 0).use_empty()) {
if (!ResLo) {
assert(LoReg && "Register for low half is not defined!");
- ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg,
- NVT, InGlue);
+ ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg, NVT,
+ InGlue);
InGlue = ResLo.getValue(2);
}
ReplaceUses(SDValue(Node, 0), ResLo);
@@ -6091,8 +6181,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
if (!SDValue(Node, 1).use_empty()) {
if (!ResHi) {
assert(HiReg && "Register for high half is not defined!");
- ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg,
- NVT, InGlue);
+ ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg, NVT,
+ InGlue);
InGlue = ResHi.getValue(2);
}
ReplaceUses(SDValue(Node, 1), ResHi);
@@ -6113,41 +6203,72 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
bool isSigned = Opcode == ISD::SDIVREM;
if (!isSigned) {
switch (NVT.SimpleTy) {
- default: llvm_unreachable("Unsupported VT!");
- case MVT::i8: ROpc = X86::DIV8r; MOpc = X86::DIV8m; break;
- case MVT::i16: ROpc = X86::DIV16r; MOpc = X86::DIV16m; break;
- case MVT::i32: ROpc = X86::DIV32r; MOpc = X86::DIV32m; break;
- case MVT::i64: ROpc = X86::DIV64r; MOpc = X86::DIV64m; break;
+ default:
+ llvm_unreachable("Unsupported VT!");
+ case MVT::i8:
+ ROpc = X86::DIV8r;
+ MOpc = X86::DIV8m;
+ break;
+ case MVT::i16:
+ ROpc = X86::DIV16r;
+ MOpc = X86::DIV16m;
+ break;
+ case MVT::i32:
+ ROpc = X86::DIV32r;
+ MOpc = X86::DIV32m;
+ break;
+ case MVT::i64:
+ ROpc = X86::DIV64r;
+ MOpc = X86::DIV64m;
+ break;
}
} else {
switch (NVT.SimpleTy) {
- default: llvm_unreachable("Unsupported VT!");
- case MVT::i8: ROpc = X86::IDIV8r; MOpc = X86::IDIV8m; break;
- case MVT::i16: ROpc = X86::IDIV16r; MOpc = X86::IDIV16m; break;
- case MVT::i32: ROpc = X86::IDIV32r; MOpc = X86::IDIV32m; break;
- case MVT::i64: ROpc = X86::IDIV64r; MOpc = X86::IDIV64m; break;
+ default:
+ llvm_unreachable("Unsupported VT!");
+ case MVT::i8:
+ ROpc = X86::IDIV8r;
+ MOpc = X86::IDIV8m;
+ break;
+ case MVT::i16:
+ ROpc = X86::IDIV16r;
+ MOpc = X86::IDIV16m;
+ break;
+ case MVT::i32:
+ ROpc = X86::IDIV32r;
+ MOpc = X86::IDIV32m;
+ break;
+ case MVT::i64:
+ ROpc = X86::IDIV64r;
+ MOpc = X86::IDIV64m;
+ break;
}
}
unsigned LoReg, HiReg, ClrReg;
unsigned SExtOpcode;
switch (NVT.SimpleTy) {
- default: llvm_unreachable("Unsupported VT!");
+ default:
+ llvm_unreachable("Unsupported VT!");
case MVT::i8:
- LoReg = X86::AL; ClrReg = HiReg = X86::AH;
+ LoReg = X86::AL;
+ ClrReg = HiReg = X86::AH;
SExtOpcode = 0; // Not used.
break;
case MVT::i16:
- LoReg = X86::AX; HiReg = X86::DX;
+ LoReg = X86::AX;
+ HiReg = X86::DX;
ClrReg = X86::DX;
SExtOpcode = X86::CWD;
break;
case MVT::i32:
- LoReg = X86::EAX; ClrReg = HiReg = X86::EDX;
+ LoReg = X86::EAX;
+ ClrReg = HiReg = X86::EDX;
SExtOpcode = X86::CDQ;
break;
case MVT::i64:
- LoReg = X86::RAX; ClrReg = HiReg = X86::RDX;
+ LoReg = X86::RAX;
+ ClrReg = HiReg = X86::RDX;
SExtOpcode = X86::CQO;
break;
}
@@ -6163,31 +6284,31 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain;
MachineSDNode *Move;
if (tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
- SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) };
- unsigned Opc = (isSigned && !signBitIsZero) ? X86::MOVSX16rm8
- : X86::MOVZX16rm8;
+ SDValue Ops[] = {Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0)};
+ unsigned Opc =
+ (isSigned && !signBitIsZero) ? X86::MOVSX16rm8 : X86::MOVZX16rm8;
Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, MVT::Other, Ops);
Chain = SDValue(Move, 1);
ReplaceUses(N0.getValue(1), Chain);
// Record the mem-refs
CurDAG->setNodeMemRefs(Move, {cast<LoadSDNode>(N0)->getMemOperand()});
} else {
- unsigned Opc = (isSigned && !signBitIsZero) ? X86::MOVSX16rr8
- : X86::MOVZX16rr8;
+ unsigned Opc =
+ (isSigned && !signBitIsZero) ? X86::MOVSX16rr8 : X86::MOVZX16rr8;
Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, N0);
Chain = CurDAG->getEntryNode();
}
- Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX, SDValue(Move, 0),
- SDValue());
+ Chain =
+ CurDAG->getCopyToReg(Chain, dl, X86::AX, SDValue(Move, 0), SDValue());
InGlue = Chain.getValue(1);
} else {
InGlue =
- CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl,
- LoReg, N0, SDValue()).getValue(1);
+ CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, N0, SDValue())
+ .getValue(1);
if (isSigned && !signBitIsZero) {
// Sign extend the low part into the high part.
- InGlue =
- SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InGlue),0);
+ InGlue = SDValue(
+ CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InGlue), 0);
} else {
// Zero out the high part, effectively zero extending the input.
SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
@@ -6195,12 +6316,11 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, {}), 0);
switch (NVT.SimpleTy) {
case MVT::i16:
- ClrNode =
- SDValue(CurDAG->getMachineNode(
- TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode,
- CurDAG->getTargetConstant(X86::sub_16bit, dl,
- MVT::i32)),
- 0);
+ ClrNode = SDValue(
+ CurDAG->getMachineNode(
+ TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode,
+ CurDAG->getTargetConstant(X86::sub_16bit, dl, MVT::i32)),
+ 0);
break;
case MVT::i32:
break;
@@ -6215,16 +6335,17 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
llvm_unreachable("Unexpected division source");
}
- InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg,
- ClrNode, InGlue).getValue(1);
+ InGlue = CurDAG
+ ->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg, ClrNode,
+ InGlue)
+ .getValue(1);
}
}
if (foldedLoad) {
- SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
- InGlue };
+ SDValue Ops[] = {Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), InGlue};
MachineSDNode *CNode =
- CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops);
+ CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops);
InGlue = SDValue(CNode, 1);
// Update the chain.
ReplaceUses(N1.getValue(1), SDValue(CNode, 0));
@@ -6232,7 +6353,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
} else {
InGlue =
- SDValue(CurDAG->getMachineNode(ROpc, dl, MVT::Glue, N1, InGlue), 0);
+ SDValue(CurDAG->getMachineNode(ROpc, dl, MVT::Glue, N1, InGlue), 0);
}
// Prevent use of AH in a REX instruction by explicitly copying it to
@@ -6261,8 +6382,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
}
// Copy the division (low) result, if it is needed.
if (!SDValue(Node, 0).use_empty()) {
- SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
- LoReg, NVT, InGlue);
+ SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg,
+ NVT, InGlue);
InGlue = Result.getValue(2);
ReplaceUses(SDValue(Node, 0), Result);
LLVM_DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG);
@@ -6270,8 +6391,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
}
// Copy the remainder (high) result, if it is needed.
if (!SDValue(Node, 1).use_empty()) {
- SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
- HiReg, NVT, InGlue);
+ SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg,
+ NVT, InGlue);
InGlue = Result.getValue(2);
ReplaceUses(SDValue(Node, 1), Result);
LLVM_DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG);
@@ -6300,7 +6421,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
unsigned Opc;
switch (CmpVT.SimpleTy) {
- default: llvm_unreachable("Unexpected type!");
+ default:
+ llvm_unreachable("Unexpected type!");
case MVT::f32:
Opc = IsSignaling ? X86::COM_Fpr32 : X86::UCOM_Fpr32;
break;
@@ -6312,8 +6434,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
break;
}
- SDValue Chain =
- IsStrictCmp ? Node->getOperand(0) : CurDAG->getEntryNode();
+ SDValue Chain = IsStrictCmp ? Node->getOperand(0) : CurDAG->getEntryNode();
SDValue Glue;
if (IsStrictCmp) {
SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
@@ -6365,8 +6486,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
// TODO: Should we check the users and use the BEXTR flags directly?
if (N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
if (MachineSDNode *NewNode = matchBEXTRFromAndImm(N0.getNode())) {
- unsigned TestOpc = CmpVT == MVT::i64 ? X86::TEST64rr
- : X86::TEST32rr;
+ unsigned TestOpc = CmpVT == MVT::i64 ? X86::TEST64rr : X86::TEST32rr;
SDValue BEXTR = SDValue(NewNode, 0);
NewNode = CurDAG->getMachineNode(TestOpc, dl, MVT::i32, BEXTR, BEXTR);
ReplaceUses(SDValue(Node, 0), SDValue(NewNode, 0));
@@ -6476,9 +6596,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
// either the sign bit in the shrunken mask is zero or the final test
// size is equal to the original compare size.
- if (isUInt<8>(Mask) &&
- (!(Mask & 0x80) || CmpVT == MVT::i8 ||
- hasNoSignFlagUses(SDValue(Node, 0)))) {
+ if (isUInt<8>(Mask) && (!(Mask & 0x80) || CmpVT == MVT::i8 ||
+ hasNoSignFlagUses(SDValue(Node, 0)))) {
// For example, convert "testl %eax, $8" to "testb %al, $8"
VT = MVT::i8;
SubRegOp = X86::sub_8bit;
@@ -6500,8 +6619,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
// Without minsize 16-bit Cmps can get here so we need to
// be sure we calculate the correct sign flag if needed.
(CmpVT != MVT::i16 || !(Mask & 0x8000))) ||
- CmpVT == MVT::i32 ||
- hasNoSignFlagUses(SDValue(Node, 0)))) {
+ CmpVT == MVT::i32 || hasNoSignFlagUses(SDValue(Node, 0)))) {
// For example, "testq %rax, $268468232" to "testl %eax, $268468232".
// NOTE: We only want to run that transform if N0 is 32 or 64 bits.
// Otherwize, we find ourselves in a position where we have to do
@@ -6532,8 +6650,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
break;
}
}
- SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
- Reg.getOperand(0) };
+ SDValue Ops[] = {Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm, Reg.getOperand(0)};
NewNode = CurDAG->getMachineNode(MOpc, dl, MVT::i32, MVT::Other, Ops);
// Update the chain.
ReplaceUses(Reg.getValue(1), SDValue(NewNode, 1));
@@ -6590,11 +6707,14 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
break;
// Copy the two implicit register inputs.
- SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EAX,
- Node->getOperand(1),
- SDValue()).getValue(1);
- InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EDX,
- Node->getOperand(3), InGlue).getValue(1);
+ SDValue InGlue = CurDAG
+ ->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EAX,
+ Node->getOperand(1), SDValue())
+ .getValue(1);
+ InGlue = CurDAG
+ ->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EDX,
+ Node->getOperand(3), InGlue)
+ .getValue(1);
bool NeedIndex = !SDValue(Node, 0).use_empty();
bool NeedMask = !SDValue(Node, 1).use_empty();
@@ -6840,8 +6960,8 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
break;
SDValue Base, Scale, Index, Disp, Segment;
- if (!selectVectorAddr(Sc, Sc->getBasePtr(), IndexOp, Sc->getScale(),
- Base, Scale, Index, Disp, Segment))
+ if (!selectVectorAddr(Sc, Sc->getBasePtr(), IndexOp, Sc->getScale(), Base,
+ Scale, Index, Disp, Segment))
break;
SDValue Mask = Sc->getMask();
diff --git a/llvm/test/CodeGen/X86/avx512-regcall-Mask.ll b/llvm/test/CodeGen/X86/avx512-regcall-Mask.ll
index 162f5efd78f6d..e772e1bf3fcf5 100644
--- a/llvm/test/CodeGen/X86/avx512-regcall-Mask.ll
+++ b/llvm/test/CodeGen/X86/avx512-regcall-Mask.ll
@@ -323,9 +323,9 @@ define dso_local x86_regcallcc i32 @test_argv32i1(<32 x i1> %x0, <32 x i1> %x1,
; WIN64-NEXT: .seh_setframe %rbp, 128
; WIN64-NEXT: .seh_endprologue
; WIN64-NEXT: andq $-32, %rsp
-; WIN64-NEXT: kmovd %edx, %k0
-; WIN64-NEXT: kmovd %eax, %k1
-; WIN64-NEXT: kmovd %ecx, %k2
+; WIN64-NEXT: kmovd %eax, %k0
+; WIN64-NEXT: kmovd %ecx, %k1
+; WIN64-NEXT: kmovd %edx, %k2
; WIN64-NEXT: vpmovm2b %k2, %zmm0
; WIN64-NEXT: vmovdqa %ymm0, {{[0-9]+}}(%rsp)
; WIN64-NEXT: vpmovm2b %k1, %zmm0
@@ -549,9 +549,9 @@ define dso_local x86_regcallcc i16 @test_argv16i1(<16 x i1> %x0, <16 x i1> %x1,
; WIN64-NEXT: subq $88, %rsp
; WIN64-NEXT: .seh_stackalloc 88
; WIN64-NEXT: .seh_endprologue
-; WIN64-NEXT: kmovd %edx, %k0
-; WIN64-NEXT: kmovd %eax, %k1
-; WIN64-NEXT: kmovd %ecx, %k2
+; WIN64-NEXT: kmovd %eax, %k0
+; WIN64-NEXT: kmovd %ecx, %k1
+; WIN64-NEXT: kmovd %edx, %k2
; WIN64-NEXT: vpmovm2b %k2, %zmm0
; WIN64-NEXT: vmovdqa %xmm0, {{[0-9]+}}(%rsp)
; WIN64-NEXT: vpmovm2b %k1, %zmm0
@@ -779,9 +779,9 @@ define dso_local x86_regcallcc i8 @test_argv8i1(<8 x i1> %x0, <8 x i1> %x1, <8 x
; WIN64-NEXT: subq $88, %rsp
; WIN64-NEXT: .seh_stackalloc 88
; WIN64-NEXT: .seh_endprologue
-; WIN64-NEXT: kmovd %edx, %k0
-; WIN64-NEXT: kmovd %eax, %k1
-; WIN64-NEXT: kmovd %ecx, %k2
+; WIN64-NEXT: kmovd %eax, %k0
+; WIN64-NEXT: kmovd %ecx, %k1
+; WIN64-NEXT: kmovd %edx, %k2
; WIN64-NEXT: vpmovm2w %k2, %zmm0
; WIN64-NEXT: vmovdqa %xmm0, {{[0-9]+}}(%rsp)
; WIN64-NEXT: vpmovm2w %k1, %zmm0
diff --git a/llvm/test/CodeGen/X86/dag-stlf-mismatch.ll b/llvm/test/CodeGen/X86/dag-stlf-mismatch.ll
index 309df51e303d8..b94e8f9c8b448 100644
--- a/llvm/test/CodeGen/X86/dag-stlf-mismatch.ll
+++ b/llvm/test/CodeGen/X86/dag-stlf-mismatch.ll
@@ -79,7 +79,8 @@ define void @test_stlf_late_byval(ptr %ptr) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: subq $40, %rsp
; AVX512-NEXT: movl $0, (%rcx)
-; AVX512-NEXT: movl $0, {{[0-9]+}}(%rsp)
+; AVX512-NEXT: movl (%rcx), %eax
+; AVX512-NEXT: movl %eax, {{[0-9]+}}(%rsp)
; AVX512-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
; AVX512-NEXT: callq ext_func
; AVX512-NEXT: addq $40, %rsp
diff --git a/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll b/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
index cae271f7a2ed4..9c80bdab640b6 100644
--- a/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
+++ b/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
@@ -599,9 +599,9 @@ define fp128 @fma(fp128 %x, fp128 %y, fp128 %z) nounwind strictfp {
; WIN-NEXT: pushq %rsi
; WIN-NEXT: subq $96, %rsp
; WIN-NEXT: movq %rcx, %rsi
-; WIN-NEXT: movaps (%r9), %xmm0
-; WIN-NEXT: movaps (%rdx), %xmm1
-; WIN-NEXT: movaps (%r8), %xmm2
+; WIN-NEXT: movaps (%rdx), %xmm0
+; WIN-NEXT: movaps (%r8), %xmm1
+; WIN-NEXT: movaps (%r9), %xmm2
; WIN-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp)
; WIN-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; WIN-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
diff --git a/llvm/test/CodeGen/X86/fp128-libcalls.ll b/llvm/test/CodeGen/X86/fp128-libcalls.ll
index c594b15ef1cbe..4c71a44e95757 100644
--- a/llvm/test/CodeGen/X86/fp128-libcalls.ll
+++ b/llvm/test/CodeGen/X86/fp128-libcalls.ll
@@ -212,8 +212,8 @@ define dso_local void @Test128_1Add(fp128 %d1) nounwind {
; WIN-NEXT: subq $88, %rsp
; WIN-NEXT: movaps (%rcx), %xmm0
; WIN-NEXT: movaps vf128(%rip), %xmm1
-; WIN-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; WIN-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
+; WIN-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %r8
@@ -469,8 +469,8 @@ define dso_local void @Test128_1Sub(fp128 %d1) nounwind {
; WIN-NEXT: subq $88, %rsp
; WIN-NEXT: movaps (%rcx), %xmm0
; WIN-NEXT: movaps vf128(%rip), %xmm1
-; WIN-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; WIN-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
+; WIN-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %r8
@@ -726,8 +726,8 @@ define dso_local void @Test128_1Mul(fp128 %d1) nounwind {
; WIN-NEXT: subq $88, %rsp
; WIN-NEXT: movaps (%rcx), %xmm0
; WIN-NEXT: movaps vf128(%rip), %xmm1
-; WIN-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; WIN-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
+; WIN-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %r8
@@ -983,8 +983,8 @@ define dso_local void @Test128_1Div(fp128 %d1) nounwind {
; WIN-NEXT: subq $88, %rsp
; WIN-NEXT: movaps (%rcx), %xmm0
; WIN-NEXT: movaps vf128(%rip), %xmm1
-; WIN-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; WIN-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
+; WIN-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %r8
@@ -1222,8 +1222,8 @@ define dso_local void @Test128_1Rem(fp128 %d1) nounwind {
; WIN-NEXT: subq $88, %rsp
; WIN-NEXT: movaps (%rcx), %xmm0
; WIN-NEXT: movaps vf128(%rip), %xmm1
-; WIN-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; WIN-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
+; WIN-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; WIN-NEXT: leaq {{[0-9]+}}(%rsp), %r8
@@ -2143,9 +2143,9 @@ define fp128 @Test128FMA(fp128 %a, fp128 %b, fp128 %c) nounwind {
; WIN-NEXT: pushq %rsi
; WIN-NEXT: subq $96, %rsp
; WIN-NEXT: movq %rcx, %rsi
-; WIN-NEXT: movaps (%r9), %xmm0
-; WIN-NEXT: movaps (%rdx), %xmm1
-; WIN-NEXT: movaps (%r8), %xmm2
+; WIN-NEXT: movaps (%rdx), %xmm0
+; WIN-NEXT: movaps (%r8), %xmm1
+; WIN-NEXT: movaps (%r9), %xmm2
; WIN-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp)
; WIN-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; WIN-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
diff --git a/llvm/test/CodeGen/X86/i128-fp128-abi.ll b/llvm/test/CodeGen/X86/i128-fp128-abi.ll
index 9f385ee2faf4e..4454e4fa5e848 100644
--- a/llvm/test/CodeGen/X86/i128-fp128-abi.ll
+++ b/llvm/test/CodeGen/X86/i128-fp128-abi.ll
@@ -706,9 +706,9 @@ define void @call_leading_args(PrimTy %x) nounwind {
; CHECK-MSVC64-F128: # %bb.0:
; CHECK-MSVC64-F128-NEXT: subq $88, %rsp
; CHECK-MSVC64-F128-NEXT: movaps (%rcx), %xmm0
-; CHECK-MSVC64-F128-NEXT: movaps %xmm0, 48(%rsp)
; CHECK-MSVC64-F128-NEXT: leaq 48(%rsp), %rax
; CHECK-MSVC64-F128-NEXT: movq %rax, 40(%rsp)
+; CHECK-MSVC64-F128-NEXT: movaps %xmm0, 48(%rsp)
; CHECK-MSVC64-F128-NEXT: movq $0, 32(%rsp)
; CHECK-MSVC64-F128-NEXT: leaq 64(%rsp), %rcx
; CHECK-MSVC64-F128-NEXT: xorl %edx, %edx
@@ -735,9 +735,9 @@ define void @call_leading_args(PrimTy %x) nounwind {
; CHECK-MINGW-F128: # %bb.0:
; CHECK-MINGW-F128-NEXT: subq $88, %rsp
; CHECK-MINGW-F128-NEXT: movaps (%rcx), %xmm0
-; CHECK-MINGW-F128-NEXT: movaps %xmm0, 48(%rsp)
; CHECK-MINGW-F128-NEXT: leaq 48(%rsp), %rax
; CHECK-MINGW-F128-NEXT: movq %rax, 40(%rsp)
+; CHECK-MINGW-F128-NEXT: movaps %xmm0, 48(%rsp)
; CHECK-MINGW-F128-NEXT: movq $0, 32(%rsp)
; CHECK-MINGW-F128-NEXT: leaq 64(%rsp), %rcx
; CHECK-MINGW-F128-NEXT: xorl %edx, %edx
@@ -857,13 +857,13 @@ define void @call_many_leading_args(PrimTy %x) nounwind {
; CHECK-MSVC64-F128: # %bb.0:
; CHECK-MSVC64-F128-NEXT: subq $120, %rsp
; CHECK-MSVC64-F128-NEXT: movaps (%rcx), %xmm0
-; CHECK-MSVC64-F128-NEXT: xorps %xmm1, %xmm1
-; CHECK-MSVC64-F128-NEXT: movaps %xmm1, 80(%rsp)
+; CHECK-MSVC64-F128-NEXT: leaq 64(%rsp), %rax
+; CHECK-MSVC64-F128-NEXT: movq %rax, 48(%rsp)
; CHECK-MSVC64-F128-NEXT: leaq 80(%rsp), %rax
; CHECK-MSVC64-F128-NEXT: movq %rax, 40(%rsp)
; CHECK-MSVC64-F128-NEXT: movaps %xmm0, 64(%rsp)
-; CHECK-MSVC64-F128-NEXT: leaq 64(%rsp), %rax
-; CHECK-MSVC64-F128-NEXT: movq %rax, 48(%rsp)
+; CHECK-MSVC64-F128-NEXT: xorps %xmm0, %xmm0
+; CHECK-MSVC64-F128-NEXT: movaps %xmm0, 80(%rsp)
; CHECK-MSVC64-F128-NEXT: movq $0, 32(%rsp)
; CHECK-MSVC64-F128-NEXT: leaq 96(%rsp), %rcx
; CHECK-MSVC64-F128-NEXT: xorl %edx, %edx
@@ -892,13 +892,13 @@ define void @call_many_leading_args(PrimTy %x) nounwind {
; CHECK-MINGW-F128: # %bb.0:
; CHECK-MINGW-F128-NEXT: subq $120, %rsp
; CHECK-MINGW-F128-NEXT: movaps (%rcx), %xmm0
-; CHECK-MINGW-F128-NEXT: xorps %xmm1, %xmm1
-; CHECK-MINGW-F128-NEXT: movaps %xmm1, 80(%rsp)
+; CHECK-MINGW-F128-NEXT: leaq 64(%rsp), %rax
+; CHECK-MINGW-F128-NEXT: movq %rax, 48(%rsp)
; CHECK-MINGW-F128-NEXT: leaq 80(%rsp), %rax
; CHECK-MINGW-F128-NEXT: movq %rax, 40(%rsp)
; CHECK-MINGW-F128-NEXT: movaps %xmm0, 64(%rsp)
-; CHECK-MINGW-F128-NEXT: leaq 64(%rsp), %rax
-; CHECK-MINGW-F128-NEXT: movq %rax, 48(%rsp)
+; CHECK-MINGW-F128-NEXT: xorps %xmm0, %xmm0
+; CHECK-MINGW-F128-NEXT: movaps %xmm0, 80(%rsp)
; CHECK-MINGW-F128-NEXT: movq $0, 32(%rsp)
; CHECK-MINGW-F128-NEXT: leaq 96(%rsp), %rcx
; CHECK-MINGW-F128-NEXT: xorl %edx, %edx
@@ -1021,9 +1021,9 @@ define void @call_trailing_arg(PrimTy %x) nounwind {
; CHECK-MSVC64-F128: # %bb.0:
; CHECK-MSVC64-F128-NEXT: subq $88, %rsp
; CHECK-MSVC64-F128-NEXT: movaps (%rcx), %xmm0
-; CHECK-MSVC64-F128-NEXT: movaps %xmm0, 48(%rsp)
; CHECK-MSVC64-F128-NEXT: leaq 48(%rsp), %rax
; CHECK-MSVC64-F128-NEXT: movq %rax, 40(%rsp)
+; CHECK-MSVC64-F128-NEXT: movaps %xmm0, 48(%rsp)
; CHECK-MSVC64-F128-NEXT: movq $0, 32(%rsp)
; CHECK-MSVC64-F128-NEXT: leaq 64(%rsp), %rcx
; CHECK-MSVC64-F128-NEXT: xorl %edx, %edx
@@ -1050,9 +1050,9 @@ define void @call_trailing_arg(PrimTy %x) nounwind {
; CHECK-MINGW-F128: # %bb.0:
; CHECK-MINGW-F128-NEXT: subq $88, %rsp
; CHECK-MINGW-F128-NEXT: movaps (%rcx), %xmm0
-; CHECK-MINGW-F128-NEXT: movaps %xmm0, 48(%rsp)
; CHECK-MINGW-F128-NEXT: leaq 48(%rsp), %rax
; CHECK-MINGW-F128-NEXT: movq %rax, 40(%rsp)
+; CHECK-MINGW-F128-NEXT: movaps %xmm0, 48(%rsp)
; CHECK-MINGW-F128-NEXT: movq $0, 32(%rsp)
; CHECK-MINGW-F128-NEXT: leaq 64(%rsp), %rcx
; CHECK-MINGW-F128-NEXT: xorl %edx, %edx
diff --git a/llvm/test/CodeGen/X86/sse-intel-ocl.ll b/llvm/test/CodeGen/X86/sse-intel-ocl.ll
index b2de7545ff5f5..001f3ff66de37 100644
--- a/llvm/test/CodeGen/X86/sse-intel-ocl.ll
+++ b/llvm/test/CodeGen/X86/sse-intel-ocl.ll
@@ -220,26 +220,26 @@ define intel_ocl_bicc <16 x float> @test_prolog_epilog(<16 x float> %a, <16 x fl
; WIN64-NEXT: subq $232, %rsp
; WIN64-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; WIN64-NEXT: movaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; WIN64-NEXT: movaps (%r9), %xmm4
+; WIN64-NEXT: movaps (%rcx), %xmm4
; WIN64-NEXT: movaps (%rdx), %xmm5
; WIN64-NEXT: movaps (%r8), %xmm6
-; WIN64-NEXT: movaps (%rcx), %xmm7
-; WIN64-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
-; WIN64-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
-; WIN64-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp)
-; WIN64-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp)
-; WIN64-NEXT: movaps %xmm7, {{[0-9]+}}(%rsp)
-; WIN64-NEXT: movaps %xmm6, {{[0-9]+}}(%rsp)
-; WIN64-NEXT: movaps %xmm5, {{[0-9]+}}(%rsp)
+; WIN64-NEXT: movaps (%r9), %xmm7
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rax
; WIN64-NEXT: movq %rax, {{[0-9]+}}(%rsp)
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rax
; WIN64-NEXT: movq %rax, {{[0-9]+}}(%rsp)
-; WIN64-NEXT: movaps %xmm4, {{[0-9]+}}(%rsp)
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rax
; WIN64-NEXT: movq %rax, {{[0-9]+}}(%rsp)
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rax
; WIN64-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; WIN64-NEXT: movaps %xmm7, {{[0-9]+}}(%rsp)
+; WIN64-NEXT: movaps %xmm6, {{[0-9]+}}(%rsp)
+; WIN64-NEXT: movaps %xmm5, {{[0-9]+}}(%rsp)
+; WIN64-NEXT: movaps %xmm4, {{[0-9]+}}(%rsp)
+; WIN64-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp)
+; WIN64-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp)
+; WIN64-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
+; WIN64-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %r8
diff --git a/llvm/test/CodeGen/X86/stack-coloring-x87.ll b/llvm/test/CodeGen/X86/stack-coloring-x87.ll
index 8990b810bd590..53dd483d3d5be 100644
--- a/llvm/test/CodeGen/X86/stack-coloring-x87.ll
+++ b/llvm/test/CodeGen/X86/stack-coloring-x87.ll
@@ -11,8 +11,8 @@ define void @test_byval(ptr %p1, ptr %p2, ptr %p3) {
; CHECK-NEXT: .seh_pushreg %rsi
; CHECK-NEXT: pushq %rdi
; CHECK-NEXT: .seh_pushreg %rdi
-; CHECK-NEXT: subq $136, %rsp
-; CHECK-NEXT: .seh_stackalloc 136
+; CHECK-NEXT: subq $72, %rsp
+; CHECK-NEXT: .seh_stackalloc 72
; CHECK-NEXT: .seh_endprologue
; CHECK-NEXT: movq %r8, %rsi
; CHECK-NEXT: movq %rdx, %rdi
@@ -48,7 +48,7 @@ define void @test_byval(ptr %p1, ptr %p2, ptr %p3) {
; CHECK-NEXT: callq use_byval
; CHECK-NEXT: nop
; CHECK-NEXT: .seh_startepilogue
-; CHECK-NEXT: addq $136, %rsp
+; CHECK-NEXT: addq $72, %rsp
; CHECK-NEXT: popq %rdi
; CHECK-NEXT: popq %rsi
; CHECK-NEXT: .seh_endepilogue
@@ -65,8 +65,8 @@ declare void @use_f80(x86_fp80)
define void @test_f80(x86_fp80 %p1, x86_fp80 %p2, x86_fp80 %p3) {
; CHECK-LABEL: test_f80:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: subq $104, %rsp
-; CHECK-NEXT: .seh_stackalloc 104
+; CHECK-NEXT: subq $72, %rsp
+; CHECK-NEXT: .seh_stackalloc 72
; CHECK-NEXT: .seh_endprologue
; CHECK-NEXT: fldt (%r8)
; CHECK-NEXT: fstpt {{[-0-9]+}}(%r{{[sb]}}p) # 10-byte Folded Spill
@@ -86,7 +86,7 @@ define void @test_f80(x86_fp80 %p1, x86_fp80 %p2, x86_fp80 %p3) {
; CHECK-NEXT: callq use_f80
; CHECK-NEXT: nop
; CHECK-NEXT: .seh_startepilogue
-; CHECK-NEXT: addq $104, %rsp
+; CHECK-NEXT: addq $72, %rsp
; CHECK-NEXT: .seh_endepilogue
; CHECK-NEXT: retq
; CHECK-NEXT: .seh_endproc
diff --git a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
index 6aec5278fa75a..51fd75e095c32 100644
--- a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics.ll
@@ -1556,10 +1556,10 @@ define <3 x double> @constrained_vector_sin_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq sin at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq sin at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -1567,9 +1567,9 @@ define <3 x double> @constrained_vector_sin_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -1781,10 +1781,10 @@ define <3 x double> @constrained_vector_cos_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq cos at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq cos at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -1792,9 +1792,9 @@ define <3 x double> @constrained_vector_cos_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -2006,10 +2006,10 @@ define <3 x double> @constrained_vector_exp_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq exp at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq exp at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -2017,9 +2017,9 @@ define <3 x double> @constrained_vector_exp_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -2231,10 +2231,10 @@ define <3 x double> @constrained_vector_exp2_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq exp2 at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq exp2 at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -2242,9 +2242,9 @@ define <3 x double> @constrained_vector_exp2_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -2456,10 +2456,10 @@ define <3 x double> @constrained_vector_log_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq log at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq log at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -2467,9 +2467,9 @@ define <3 x double> @constrained_vector_log_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -2681,10 +2681,10 @@ define <3 x double> @constrained_vector_log10_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq log10 at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq log10 at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -2692,9 +2692,9 @@ define <3 x double> @constrained_vector_log10_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -2906,10 +2906,10 @@ define <3 x double> @constrained_vector_log2_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq log2 at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq log2 at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -2917,9 +2917,9 @@ define <3 x double> @constrained_vector_log2_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -3145,10 +3145,10 @@ define <3 x double> @constrained_vector_rint_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq rint at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq rint at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -3156,9 +3156,9 @@ define <3 x double> @constrained_vector_rint_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -3188,10 +3188,10 @@ define <3 x double> @constrained_vector_rint_v3f64_var(ptr %a) #0 {
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps (%rdi), %xmm0
; CHECK-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: callq rint at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: callq rint at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
@@ -3200,9 +3200,9 @@ define <3 x double> @constrained_vector_rint_v3f64_var(ptr %a) #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $40, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -3441,10 +3441,10 @@ define <3 x double> @constrained_vector_nearby_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq nearbyint at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq nearbyint at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -3452,9 +3452,9 @@ define <3 x double> @constrained_vector_nearby_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -3484,10 +3484,10 @@ define <3 x double> @constrained_vector_nearbyint_v3f64_var(ptr %a) #0 {
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps (%rdi), %xmm0
; CHECK-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: callq nearbyint at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: callq nearbyint at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
@@ -3496,9 +3496,9 @@ define <3 x double> @constrained_vector_nearbyint_v3f64_var(ptr %a) #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $40, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -5977,10 +5977,10 @@ define <3 x double> @constrained_vector_ceil_v3f64_var(ptr %a) #0 {
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps (%rdi), %xmm0
; CHECK-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: callq ceil at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: callq ceil at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
@@ -5989,9 +5989,9 @@ define <3 x double> @constrained_vector_ceil_v3f64_var(ptr %a) #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $40, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -6120,10 +6120,10 @@ define <3 x double> @constrained_vector_floor_v3f64_var(ptr %a) #0 {
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps (%rdi), %xmm0
; CHECK-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: callq floor at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: callq floor at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
@@ -6132,9 +6132,9 @@ define <3 x double> @constrained_vector_floor_v3f64_var(ptr %a) #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $40, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -6297,10 +6297,10 @@ define <3 x double> @constrained_vector_round_v3f64_var(ptr %a) #0 {
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps (%rdi), %xmm0
; CHECK-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: callq round at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: callq round at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
@@ -6309,9 +6309,9 @@ define <3 x double> @constrained_vector_round_v3f64_var(ptr %a) #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $40, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -6461,10 +6461,10 @@ define <3 x double> @constrained_vector_trunc_v3f64_var(ptr %a) #0 {
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps (%rdi), %xmm0
; CHECK-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: callq trunc at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: callq trunc at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
@@ -6473,9 +6473,9 @@ define <3 x double> @constrained_vector_trunc_v3f64_var(ptr %a) #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $40, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -8221,10 +8221,10 @@ define <3 x double> @constrained_vector_tan_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq tan at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq tan at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -8232,9 +8232,9 @@ define <3 x double> @constrained_vector_tan_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -8446,10 +8446,10 @@ define <3 x double> @constrained_vector_acos_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq acos at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq acos at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -8457,9 +8457,9 @@ define <3 x double> @constrained_vector_acos_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -8671,10 +8671,10 @@ define <3 x double> @constrained_vector_asin_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq asin at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq asin at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -8682,9 +8682,9 @@ define <3 x double> @constrained_vector_asin_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -8896,10 +8896,10 @@ define <3 x double> @constrained_vector_atan_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq atan at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq atan at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -8907,9 +8907,9 @@ define <3 x double> @constrained_vector_atan_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -9378,10 +9378,10 @@ define <3 x double> @constrained_vector_cosh_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq cosh at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq cosh at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -9389,9 +9389,9 @@ define <3 x double> @constrained_vector_cosh_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -9603,10 +9603,10 @@ define <3 x double> @constrained_vector_sinh_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq sinh at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq sinh at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -9614,9 +9614,9 @@ define <3 x double> @constrained_vector_sinh_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
@@ -9828,10 +9828,10 @@ define <3 x double> @constrained_vector_tanh_v3f64() #0 {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: subq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
; CHECK-NEXT: callq tanh at PLT
; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2E+1,0.0E+0]
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2100000000000001E+1,0.0E+0]
; CHECK-NEXT: callq tanh at PLT
; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [4.2200000000000003E+1,0.0E+0]
@@ -9839,9 +9839,9 @@ define <3 x double> @constrained_vector_tanh_v3f64() #0 {
; CHECK-NEXT: movsd %xmm0, {{[0-9]+}}(%rsp)
; CHECK-NEXT: fldl {{[0-9]+}}(%rsp)
; CHECK-NEXT: wait
-; CHECK-NEXT: movsd (%rsp), %xmm0 # 8-byte Reload
+; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero
-; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload
+; CHECK-NEXT: movsd (%rsp), %xmm1 # 8-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero
; CHECK-NEXT: addq $24, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
diff --git a/llvm/test/CodeGen/X86/win64-byval.ll b/llvm/test/CodeGen/X86/win64-byval.ll
index 573a0016e8772..a4ed85cad6078 100644
--- a/llvm/test/CodeGen/X86/win64-byval.ll
+++ b/llvm/test/CodeGen/X86/win64-byval.ll
@@ -63,21 +63,21 @@ define void @test() {
; CHECK-NEXT: subq $136, %rsp
; CHECK-NEXT: .seh_stackalloc 136
; CHECK-NEXT: .seh_endprologue
+; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
; CHECK-NEXT: movq .refptr.G(%rip), %rax
; CHECK-NEXT: movq (%rax), %rcx
; CHECK-NEXT: movq 8(%rax), %rax
; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
; CHECK-NEXT: movq %rcx, {{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rcx, {{[0-9]+}}(%rsp)
; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
; CHECK-NEXT: movq %rcx, {{[0-9]+}}(%rsp)
; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
; CHECK-NEXT: movq %rcx, {{[0-9]+}}(%rsp)
; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
; CHECK-NEXT: movq %rcx, {{[0-9]+}}(%rsp)
-; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rax
; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rcx, {{[0-9]+}}(%rsp)
; CHECK-NEXT: movq $10, {{[0-9]+}}(%rsp)
; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
diff --git a/llvm/test/CodeGen/X86/x87-stack-coloring.ll b/llvm/test/CodeGen/X86/x87-stack-coloring.ll
index eeca372cf3ac5..17665f71c7e91 100644
--- a/llvm/test/CodeGen/X86/x87-stack-coloring.ll
+++ b/llvm/test/CodeGen/X86/x87-stack-coloring.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+sse2,+x87 | FileCheck %s
-; ModuleID = 'example_big.c'
-source_filename = "example_big.c"
+; ModuleID = 'user_cases.c'
+source_filename = "user_cases.c"
target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:128-f64:32:64-f80:32-n8:16:32-S128"
target triple = "i386-unknown-linux-gnu"
@@ -10,7 +10,7 @@ target triple = "i386-unknown-linux-gnu"
define dso_local float @returns_float() local_unnamed_addr #0 {
; CHECK-LABEL: returns_float:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: subl $44, %esp
+; CHECK-NEXT: subl $12, %esp
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: calll rand at PLT
; CHECK-NEXT: cvtsi2ss %eax, %xmm0
@@ -27,11 +27,29 @@ entry:
declare i32 @rand() local_unnamed_addr #1
+; Function Attrs: noinline nounwind uwtable
+define dso_local x86_fp80 @returns_long_double() local_unnamed_addr #0 {
+; CHECK-LABEL: returns_long_double:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: subl $12, %esp
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: calll rand at PLT
+; CHECK-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT: fildl {{[0-9]+}}(%esp)
+; CHECK-NEXT: addl $12, %esp
+; CHECK-NEXT: .cfi_def_cfa_offset 4
+; CHECK-NEXT: retl
+entry:
+ %call = tail call i32 @rand() #3
+ %conv = sitofp i32 %call to x86_fp80
+ ret x86_fp80 %conv
+}
+
; Function Attrs: nounwind uwtable
-define dso_local float @example() local_unnamed_addr #2 {
-; CHECK-LABEL: example:
+define dso_local float @example_float() local_unnamed_addr #2 {
+; CHECK-LABEL: example_float:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: subl $44, %esp
+; CHECK-NEXT: subl $12, %esp
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: calll returns_float
; CHECK-NEXT: fstps {{[0-9]+}}(%esp)
@@ -78,6 +96,50 @@ entry:
ret float %add7
}
+; Function Attrs: nounwind uwtable
+define dso_local float @example_long_double() local_unnamed_addr #2 {
+; CHECK-LABEL: example_long_double:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: subl $28, %esp
+; CHECK-NEXT: .cfi_def_cfa_offset 32
+; CHECK-NEXT: calll returns_long_double
+; CHECK-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; CHECK-NEXT: calll returns_long_double
+; CHECK-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; CHECK-NEXT: faddp %st, %st(1)
+; CHECK-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; CHECK-NEXT: calll returns_long_double
+; CHECK-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; CHECK-NEXT: faddp %st, %st(1)
+; CHECK-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; CHECK-NEXT: calll returns_long_double
+; CHECK-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; CHECK-NEXT: faddp %st, %st(1)
+; CHECK-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; CHECK-NEXT: calll returns_long_double
+; CHECK-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; CHECK-NEXT: faddp %st, %st(1)
+; CHECK-NEXT: fstps {{[0-9]+}}(%esp)
+; CHECK-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: movss %xmm0, {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: addl $28, %esp
+; CHECK-NEXT: .cfi_def_cfa_offset 4
+; CHECK-NEXT: retl
+entry:
+ %call = tail call x86_fp80 @returns_long_double()
+ %call1 = tail call x86_fp80 @returns_long_double()
+ %add = fadd x86_fp80 %call, %call1
+ %call2 = tail call x86_fp80 @returns_long_double()
+ %add3 = fadd x86_fp80 %add, %call2
+ %call4 = tail call x86_fp80 @returns_long_double()
+ %add5 = fadd x86_fp80 %add3, %call4
+ %call6 = tail call x86_fp80 @returns_long_double()
+ %add7 = fadd x86_fp80 %add5, %call6
+ %conv = fptrunc x86_fp80 %add7 to float
+ ret float %conv
+}
+
attributes #0 = { noinline nounwind uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
attributes #2 = { nounwind uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="pentium4" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
More information about the llvm-commits
mailing list