[clang] [llvm] [clang][X86] Return Win64 __int128 indirectly (PR #210358)
via cfe-commits
cfe-commits at lists.llvm.org
Wed Nov 4 09:54:07 PST 2026
https://github.com/oltolm updated https://github.com/llvm/llvm-project/pull/210358
>From ab7ae6021b8a084a2e812994406f738912c91191 Mon Sep 17 00:00:00 2001
From: Oleg Tolmatcev <oleg.tolmatcev at gmail.com>
Date: Wed, 4 Nov 2026 18:46:09 +0100
Subject: [PATCH] [clang][X86] Return Win64 __int128 indirectly
Return Win64 __int128 and unsigned __int128 values through a hidden sret pointer instead of treating them as v2i64 values returned in XMM0. For instance methods, pass the sret pointer after this to match Microsoft ABI argument ordering. -fclang-abi-compat=23 restores Clang's previous source-level convention.
Make the X86 backend apply the same return convention to i128 libcalls and pass 16-byte atomic libcall operands indirectly on Windows. This matches GCC's ABI while preserving inline i128 multiplication lowering.
Assisted-by: OpenAI Codex
---
clang/docs/ReleaseNotes.md | 7 +
clang/lib/CodeGen/Targets/X86.cpp | 56 +-
clang/test/CodeGen/win64-i128.c | 45 +-
clang/test/CodeGenCXX/win64-i128.cpp | 37 +
llvm/lib/CodeGen/AtomicExpandPass.cpp | 28 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 37 +-
llvm/lib/Target/X86/X86ISelLoweringCall.cpp | 16 +-
llvm/test/CodeGen/X86/atomic-nocx16-win.ll | 9 +-
llvm/test/CodeGen/X86/divmod128.ll | 894 ++++++++++--------
llvm/test/CodeGen/X86/fp128-cast.ll | 12 +-
.../CodeGen/X86/fp128-return-calling-conv.ll | 41 +-
llvm/test/CodeGen/X86/fp80-conv-libcalls.ll | 16 +-
llvm/test/CodeGen/X86/fptosi-f16-wide.ll | 44 +-
llvm/test/CodeGen/X86/fptoui-f16-wide.ll | 36 +-
llvm/test/CodeGen/X86/i128-fp128-abi.ll | 128 ++-
.../CodeGen/X86/i128-fpconv-win64-strict.ll | 46 +-
llvm/test/CodeGen/X86/i128-fpconv-win64.ll | 46 +-
llvm/test/CodeGen/X86/i128-ret.ll | 11 +-
llvm/test/CodeGen/X86/mul128.ll | 25 +
.../X86/expand-atomic-non-integer.ll | 393 +++++++-
20 files changed, 1330 insertions(+), 597 deletions(-)
create mode 100644 clang/test/CodeGenCXX/win64-i128.cpp
diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md
index 409d2459b0580b..f1c663c4349226 100644
--- a/clang/docs/ReleaseNotes.md
+++ b/clang/docs/ReleaseNotes.md
@@ -125,6 +125,13 @@ features cannot lower the translation-unit ABI level;
always passed the parts separately. `-fclang-abi-compat=23` restores the previous
behavior. (#GH212109)
+- On x86_64 Windows targets, Clang now returns `__int128` and `unsigned
+ __int128` indirectly through a hidden return pointer. This matches how Clang
+ already returns 16-byte structs on the same targets. This is ABI-incompatible
+ with objects and runtime libraries using Clang's previous direct XMM return
+ convention. `-fclang-abi-compat=23` restores the previous convention for
+ source-level function calls.
+
- On MIPS N32/N64, an `__int128` now correctly start in an even-numbered register
or 16-byte aligned stack slot, matching GCC.
diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 5324c8d366b829..e6dcbf1a9a6fe2 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -3673,7 +3673,7 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
case BuiltinType::Int128:
case BuiltinType::UInt128:
- case BuiltinType::Float128:
+ case BuiltinType::Float128: {
// If it's a parameter type, the normal ABI rule is that arguments larger
// than 8 bytes are passed indirectly. GCC follows it. We follow it too,
// even though it isn't particularly efficient.
@@ -3682,32 +3682,32 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
/*ByVal=*/false);
- // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
- // Clang matches them for compatibility.
+ // Win64 C returns i128 and f128 via sret. Other calling conventions,
+ // including vectorcall and regcall, return them in XMM registers.
+ auto IsWin64StackCC = [this](unsigned CC) -> bool {
+ switch (CC) {
+ case llvm::CallingConv::Win64:
+ return true;
+ case llvm::CallingConv::C:
+ return getTarget().getTriple().isOSWindowsOrUEFI();
+ default:
+ return false;
+ }
+ };
+
+ if (IsWin64StackCC(CC) &&
+ (BT->getKind() == BuiltinType::Float128 ||
+ getContext().getLangOpts().getClangABICompat() >
+ LangOptions::ClangABI::Ver23))
+ return getNaturalAlignIndirect(
+ Ty, getDataLayout().getAllocaAddrSpace(), /*ByVal=*/false);
+
if (BT->getKind() == BuiltinType::Int128 ||
BT->getKind() == BuiltinType::UInt128)
return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
llvm::Type::getInt64Ty(getVMContext()), 2));
-
- // Mingw64 GCC returns f128 via sret, and Clang matches that for
- // compatibility. This mirrors the X86 backend's CanLowerReturn logic.
- if (BT->getKind() == BuiltinType::Float128) {
- auto IsWin64F128StackCC = [this](unsigned CC) -> bool {
- switch (CC) {
- case llvm::CallingConv::Win64:
- return true;
- case llvm::CallingConv::C:
- return getTarget().getTriple().isOSWindowsOrUEFI();
- default:
- return false;
- }
- };
-
- if (IsWin64F128StackCC(CC))
- return getNaturalAlignIndirect(
- Ty, getDataLayout().getAllocaAddrSpace(), /*ByVal=*/false);
- }
break;
+ }
default:
break;
@@ -3766,6 +3766,18 @@ void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
FI.getReturnInfo() =
classify(FI.getReturnType(), FreeSSERegs, ClassifyKind::Return, CC);
+ const BuiltinType *BT = FI.getReturnType()->getAs<BuiltinType>();
+ bool IsI128OrF128 =
+ BT && (BT->getKind() == BuiltinType::Int128 ||
+ BT->getKind() == BuiltinType::UInt128 ||
+ BT->getKind() == BuiltinType::Float128);
+ if (IsI128OrF128 &&
+ getContext().getLangOpts().getClangABICompat() >
+ LangOptions::ClangABI::Ver23 &&
+ FI.getReturnInfo().isIndirect() && FI.isInstanceMethod() &&
+ getCXXABI().isSRetParameterAfterThis())
+ FI.getReturnInfo().setSRetAfterThis(true);
+
if (IsVectorCall) {
// We can use up to 6 SSE register parameters with vectorcall.
FreeSSERegs = 6;
diff --git a/clang/test/CodeGen/win64-i128.c b/clang/test/CodeGen/win64-i128.c
index ce338d451115c3..8c3e61c74040b8 100644
--- a/clang/test/CodeGen/win64-i128.c
+++ b/clang/test/CodeGen/win64-i128.c
@@ -2,6 +2,11 @@
// RUN: | FileCheck %s --check-prefixes=CHECK,X64
// RUN: %clang_cc1 -triple x86_64-windows-msvc -emit-llvm -o - %s \
// RUN: | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -target-feature -sse \
+// RUN: -emit-llvm -o - %s \
+// RUN: | FileCheck %s --check-prefixes=CHECK,NOSSE
+// RUN: %clang_cc1 -triple x86_64-windows-gnu -fclang-abi-compat=23 \
+// RUN: -emit-llvm -o - %s | FileCheck %s --check-prefix=COMPAT23
// RUN: %clang_cc1 -triple aarch64-windows-msvc -emit-llvm -o - %s \
// RUN: | FileCheck %s --check-prefixes=CHECK,ARM,ARM64
// RUN: %clang_cc1 -triple arm64ec-windows-msvc -emit-llvm -o - %s \
@@ -11,14 +16,50 @@ typedef int int128_t __attribute__((mode(TI)));
int128_t foo(void) { return 0; }
-// X64: define dso_local <2 x i64> @foo()
+// X64-LABEL: define dso_local void @foo(
+// X64-SAME: sret(i128) align 16
+// COMPAT23: define dso_local <2 x i64> @foo()
// ARM: define dso_local i128 @foo()
int128_t bar(int128_t a, int128_t b) { return a * b; }
-// X64: define dso_local <2 x i64> @bar(ptr nofreeobj noundef align 16 dead_on_return dereferenceable(16) %0, ptr nofreeobj noundef align 16 dead_on_return dereferenceable(16) %1)
+// X64-LABEL: define dso_local void @bar(
+// X64-SAME: sret(i128) align 16
+// X64-SAME: ptr nofreeobj noundef align 16 dead_on_return
+// X64-SAME: ptr nofreeobj noundef align 16 dead_on_return
+// NOSSE-LABEL: define dso_local void @bar(
+// NOSSE-SAME: sret(i128) align 16
+// NOSSE-SAME: ptr nofreeobj noundef align 16 dead_on_return
+// NOSSE-SAME: ptr nofreeobj noundef align 16 dead_on_return
+// COMPAT23: define dso_local <2 x i64> @bar(ptr nofreeobj noundef align 16 dead_on_return dereferenceable(16) %0, ptr nofreeobj noundef align 16 dead_on_return dereferenceable(16) %1)
// ARM: define dso_local i128 @bar(i128 noundef %a, i128 noundef %b)
+#if defined(__x86_64__) && !defined(__arm64ec__)
+int128_t __attribute__((vectorcall)) vectorcall_foo(void) { return 0; }
+
+// X64-LABEL: define dso_local x86_vectorcallcc <2 x i64> @"\01vectorcall_foo@@0"()
+
+int128_t __attribute__((vectorcall)) vectorcall_bar(int128_t a, int128_t b) {
+ return a * b;
+}
+
+// X64-LABEL: define dso_local x86_vectorcallcc <2 x i64> @"\01vectorcall_bar@@32"(
+// X64-SAME: ptr nofreeobj noundef align 16 dead_on_return
+// X64-SAME: ptr nofreeobj noundef align 16 dead_on_return
+
+int128_t __attribute__((regcall)) regcall_foo(void) { return 0; }
+
+// X64-LABEL: define dso_local x86_regcallcc <2 x i64> @__regcall3__regcall_foo()
+
+int128_t __attribute__((regcall)) regcall_bar(int128_t a, int128_t b) {
+ return a * b;
+}
+
+// X64-LABEL: define dso_local x86_regcallcc <2 x i64> @__regcall3__regcall_bar(
+// X64-SAME: ptr nofreeobj noundef align 16 dead_on_return
+// X64-SAME: ptr nofreeobj noundef align 16 dead_on_return
+#endif
+
void vararg(int a, ...) {
// CHECK: define{{.*}} void @vararg
__builtin_va_list ap;
diff --git a/clang/test/CodeGenCXX/win64-i128.cpp b/clang/test/CodeGenCXX/win64-i128.cpp
new file mode 100644
index 00000000000000..08ffd21bccefcd
--- /dev/null
+++ b/clang/test/CodeGenCXX/win64-i128.cpp
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -triple x86_64-windows-gnu -emit-llvm -o - %s \
+// RUN: | FileCheck %s --check-prefix=GNU
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -emit-llvm -o - %s \
+// RUN: | FileCheck %s --check-prefixes=MSVC,BITINT
+// RUN: %clang_cc1 -triple x86_64-windows-gnu -fclang-abi-compat=23 \
+// RUN: -emit-llvm -o - %s | FileCheck %s --check-prefix=COMPAT23-GNU
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -fclang-abi-compat=23 \
+// RUN: -emit-llvm -o - %s | FileCheck %s --check-prefixes=COMPAT23-MSVC,BITINT
+
+typedef int int128_t __attribute__((mode(TI)));
+
+struct S {
+ int128_t f();
+};
+
+int128_t S::f() { return 0; }
+
+// GNU-LABEL: define dso_local void @_ZN1S1fEv(
+// GNU-SAME: ptr dead_on_unwind noalias writable sret(i128) align 16 %agg.result,
+// GNU-SAME: ptr noundef nonnull align 1 dereferenceable(1) %this)
+// MSVC-LABEL: define dso_local void @"?f at S@@QEAA_LXZ"(
+// MSVC-SAME: ptr noundef nonnull align 1 dereferenceable(1) %this,
+// MSVC-SAME: ptr dead_on_unwind noalias writable sret(i128) align 16 %agg.result)
+// COMPAT23-GNU-LABEL: define dso_local{{.*}} <2 x i64> @_ZN1S1fEv(
+// COMPAT23-GNU-SAME: ptr noundef nonnull align 1 dereferenceable(1) %this)
+// COMPAT23-MSVC-LABEL: define dso_local{{.*}} <2 x i64> @"?f at S@@QEAA_LXZ"(
+// COMPAT23-MSVC-SAME: ptr noundef nonnull align 1 dereferenceable(1) %this)
+
+struct BitInt {
+ _BitInt(128) f();
+};
+
+_BitInt(128) BitInt::f() { return 0; }
+
+// BITINT-LABEL: define dso_local void @"?f at BitInt
+// BITINT-SAME: ptr dead_on_unwind noalias writable sret(i128) align 8 %agg.result,
+// BITINT-SAME: ptr noundef nonnull align 1 dereferenceable(1) %this)
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 0134c3f672d499..8dc0e6910f3b62 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -2255,16 +2255,9 @@ bool AtomicExpandImpl::expandAtomicOpToLibcall(
bool UseSizedLibcall = canUseSizedAtomicCall(Size, Alignment, DL);
Type *SizedIntTy = Type::getIntNTy(Ctx, Size * 8);
-
- if (M->getTargetTriple().isOSWindows() && M->getTargetTriple().isX86_64() &&
- Size == 16) {
- // x86_64 Windows passes i128 as an XMM vector; on return, it is in
- // XMM0, and as a parameter, it is passed indirectly. The generic lowering
- // rules handles this correctly if we pass it as a v2i64 rather than
- // i128. This is what Clang does in the frontend for such types as well
- // (see WinX86_64ABIInfo::classify in Clang).
- SizedIntTy = FixedVectorType::get(Type::getInt64Ty(Ctx), 2);
- }
+ bool PassI128Indirectly =
+ UseSizedLibcall && Size == 16 && M->getTargetTriple().isX86_64() &&
+ M->getTargetTriple().isOSWindowsOrUEFI();
const Align AllocaAlignment = DL.getPrefTypeAlign(SizedIntTy);
@@ -2377,7 +2370,18 @@ bool AtomicExpandImpl::expandAtomicOpToLibcall(
if (UseSizedLibcall) {
Value *IntValue =
Builder.CreateBitPreservingCastChain(DL, ValueOperand, SizedIntTy);
- Args.push_back(IntValue);
+ if (PassI128Indirectly) {
+ // Win64 passes i128 arguments indirectly. Materialize that ABI here
+ // because SelectionDAG cannot recover the original i128 after it has
+ // been split for calling convention analysis.
+ AllocaValue = AllocaBuilder.CreateAlloca(SizedIntTy);
+ AllocaValue->setAlignment(AllocaAlignment);
+ Builder.CreateLifetimeStart(AllocaValue);
+ Builder.CreateAlignedStore(IntValue, AllocaValue, AllocaAlignment);
+ Args.push_back(AllocaValue);
+ } else {
+ Args.push_back(IntValue);
+ }
} else {
AllocaValue = AllocaBuilder.CreateAlloca(ValueOperand->getType());
AllocaValue->setAlignment(AllocaAlignment);
@@ -2433,7 +2437,7 @@ bool AtomicExpandImpl::expandAtomicOpToLibcall(
Value *Result = Call;
// And then, extract the results...
- if (ValueOperand && !UseSizedLibcall)
+ if (AllocaValue)
Builder.CreateLifetimeEnd(AllocaValue);
if (CASExpected) {
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 637e96308270d0..0ac423c98ddef4 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2741,7 +2741,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
- // The Win64 C ABI passes i128 by reference and returns it in XMM0. The
+ // The Win64 C ABI passes i128 by reference and returns it via sret. The
// libcalls for i128 division, remainder and floating point conversions are
// compiled with that ABI wherever it is the default C convention (Windows and
// UEFI), so lower them accordingly there.
@@ -22478,12 +22478,14 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
// fp128 needs to use a libcall.
if (SrcVT == MVT::f128) {
+ if (Subtarget.isCallingConvWin64(CallingConv::C) && VT == MVT::i128)
+ return LowerWin64_FP_TO_INT128(Op, DAG, Chain);
+
RTLIB::Libcall LC;
if (IsSigned)
LC = RTLIB::getFPTOSINT(SrcVT, VT);
else
LC = RTLIB::getFPTOUINT(SrcVT, VT);
-
MakeLibCallOptions CallOptions;
std::pair<SDValue, SDValue> Tmp =
makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
@@ -30977,7 +30979,8 @@ static SDValue LowerMULO(SDValue Op, const X86Subtarget &Subtarget,
return DAG.getMergeValues({Low, Ovf}, dl);
}
-SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op,
+ SelectionDAG &DAG) const {
assert(Subtarget.isCallingConvWin64(CallingConv::C) && "Unexpected target");
EVT VT = Op.getValueType();
assert(VT.isInteger() && VT.getSizeInBits() == 128 &&
@@ -30990,14 +30993,13 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) cons
}
RTLIB::Libcall LC;
- bool isSigned;
switch (Op->getOpcode()) {
// clang-format off
default: llvm_unreachable("Unexpected request for libcall!");
- case ISD::SDIV: isSigned = true; LC = RTLIB::SDIV_I128; break;
- case ISD::UDIV: isSigned = false; LC = RTLIB::UDIV_I128; break;
- case ISD::SREM: isSigned = true; LC = RTLIB::SREM_I128; break;
- case ISD::UREM: isSigned = false; LC = RTLIB::UREM_I128; break;
+ case ISD::SDIV: LC = RTLIB::SDIV_I128; break;
+ case ISD::UDIV: LC = RTLIB::UDIV_I128; break;
+ case ISD::SREM: LC = RTLIB::SREM_I128; break;
+ case ISD::UREM: LC = RTLIB::UREM_I128; break;
// clang-format on
}
@@ -31015,7 +31017,10 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) cons
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
InChain =
DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, MPI, Align(16));
- Args.emplace_back(StackPtr, PointerType::get(*DAG.getContext(), 0));
+ Args.emplace_back(
+ StackPtr,
+ PointerType::get(*DAG.getContext(),
+ DAG.getDataLayout().getAllocaAddrSpace()));
}
RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
@@ -31029,14 +31034,11 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) cons
CLI.setDebugLoc(dl)
.setChain(InChain)
.setLibCallee(DAG.getLibcalls().getLibcallImplCallingConv(LCImpl),
- EVT(MVT::v2i64).getTypeForEVT(*DAG.getContext()), Callee,
- std::move(Args))
- .setInRegister()
- .setSExtResult(isSigned)
- .setZExtResult(!isSigned);
+ VT.getTypeForEVT(*DAG.getContext()), Callee,
+ std::move(Args));
std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
- return DAG.getBitcast(VT, CallInfo.first);
+ return CallInfo.first;
}
SDValue X86TargetLowering::LowerWin64_FP_TO_INT128(SDValue Op,
@@ -31065,11 +31067,8 @@ SDValue X86TargetLowering::LowerWin64_FP_TO_INT128(SDValue Op,
Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
SDValue Result;
- // Expect the i128 argument returned as a v2i64 in xmm0, cast back to the
- // expected VT (i128).
std::tie(Result, Chain) =
- makeLibCall(DAG, LC, MVT::v2i64, Arg, CallOptions, dl, Chain);
- Result = DAG.getBitcast(VT, Result);
+ makeLibCall(DAG, LC, VT, Arg, CallOptions, dl, Chain);
return Result;
}
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index a78621fd897b35..df60644caec790 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -671,9 +671,9 @@ bool X86TargetLowering::CanLowerReturn(
CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
const Type *RetTy) const {
- // Mingw64 GCC returns f128 via sret, and LLVM matches it for compatibility.
- // This logic exists for libcalls, a frontend should explicitly use sret
- // rather than rely on the sret demotion here.
+ // Mingw64 GCC returns f128 and i128 via sret, and LLVM matches it for
+ // compatibility. This logic exists for libcalls, a frontend should
+ // explicitly use sret rather than rely on the sret demotion here.
//
// Using sret is a reasonable implementation of the Windows x64 calling
// convention:
@@ -692,7 +692,7 @@ bool X86TargetLowering::CanLowerReturn(
// published, this can be revisited.
//
// Return false, which will perform sret demotion.
- auto IsWin64F128StackCC = [this](CallingConv::ID CC) -> bool {
+ auto IsWin64StackCC = [this](CallingConv::ID CC) -> bool {
switch (CC) {
case CallingConv::Win64:
return true;
@@ -703,9 +703,11 @@ bool X86TargetLowering::CanLowerReturn(
}
};
- if (IsWin64F128StackCC(CallConv) &&
- llvm::any_of(
- Outs, [](const ISD::OutputArg &Out) { return Out.VT == MVT::f128; }))
+ // i128 is legalized into i64 parts, so inspect ArgVT rather than VT.
+ if (IsWin64StackCC(CallConv) &&
+ llvm::any_of(Outs, [](const ISD::OutputArg &Out) {
+ return Out.VT == MVT::f128 || Out.ArgVT == MVT::i128;
+ }))
return false;
SmallVector<CCValAssign, 16> RVLocs;
diff --git a/llvm/test/CodeGen/X86/atomic-nocx16-win.ll b/llvm/test/CodeGen/X86/atomic-nocx16-win.ll
index 3f0649cd9ae07e..c2c8cec224966f 100644
--- a/llvm/test/CodeGen/X86/atomic-nocx16-win.ll
+++ b/llvm/test/CodeGen/X86/atomic-nocx16-win.ll
@@ -5,13 +5,14 @@ define void @call_load(ptr %dst, ptr %src) nounwind {
; CHECK-LABEL: call_load:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: pushq %rsi
-; CHECK-NEXT: subq $32, %rsp
+; CHECK-NEXT: subq $48, %rsp
; CHECK-NEXT: movq %rcx, %rsi
-; CHECK-NEXT: movq %rdx, %rcx
-; CHECK-NEXT: movl $5, %edx
+; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT: movl $5, %r8d
; CHECK-NEXT: callq __atomic_load_16
+; CHECK-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
; CHECK-NEXT: movaps %xmm0, (%rsi)
-; CHECK-NEXT: addq $32, %rsp
+; CHECK-NEXT: addq $48, %rsp
; CHECK-NEXT: popq %rsi
; CHECK-NEXT: retq
entry:
diff --git a/llvm/test/CodeGen/X86/divmod128.ll b/llvm/test/CodeGen/X86/divmod128.ll
index e298e221af2312..c8c2229a37a314 100644
--- a/llvm/test/CodeGen/X86/divmod128.ll
+++ b/llvm/test/CodeGen/X86/divmod128.ll
@@ -17,16 +17,17 @@ define i64 @mod128(i128 %x) nounwind {
;
; WIN64-LABEL: mod128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $72, %rsp
+; WIN64-NEXT: subq $88, %rsp
; WIN64-NEXT: movq %rdx, {{[0-9]+}}(%rsp)
; WIN64-NEXT: movq %rcx, {{[0-9]+}}(%rsp)
; WIN64-NEXT: movq $3, {{[0-9]+}}(%rsp)
; WIN64-NEXT: movq $0, {{[0-9]+}}(%rsp)
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %r8
; WIN64-NEXT: callq __modti3
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $72, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $88, %rsp
; WIN64-NEXT: retq
@@ -47,16 +48,17 @@ define i64 @div128(i128 %x) nounwind {
;
; WIN64-LABEL: div128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $72, %rsp
+; WIN64-NEXT: subq $88, %rsp
; WIN64-NEXT: movq %rdx, {{[0-9]+}}(%rsp)
; WIN64-NEXT: movq %rcx, {{[0-9]+}}(%rsp)
; WIN64-NEXT: movq $3, {{[0-9]+}}(%rsp)
; WIN64-NEXT: movq $0, {{[0-9]+}}(%rsp)
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %r8
; WIN64-NEXT: callq __divti3
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $72, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $88, %rsp
; WIN64-NEXT: retq
@@ -165,16 +167,18 @@ define i128 @urem_i128_3(i128 %x) nounwind {
;
; WIN64-LABEL: urem_i128_3:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: addq %rdx, %rcx
-; WIN64-NEXT: adcq $0, %rcx
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: addq %r8, %r9
+; WIN64-NEXT: adcq $0, %r9
; WIN64-NEXT: movabsq $-6148914691236517205, %rdx # imm = 0xAAAAAAAAAAAAAAAB
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: movq %r9, %rax
; WIN64-NEXT: mulq %rdx
; WIN64-NEXT: shrq %rdx
; WIN64-NEXT: leaq (%rdx,%rdx,2), %rax
-; WIN64-NEXT: subq %rax, %rcx
+; WIN64-NEXT: subq %rax, %r9
+; WIN64-NEXT: movq %r9, (%rcx)
+; WIN64-NEXT: movq $0, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: xorl %edx, %edx
; WIN64-NEXT: retq
entry:
%rem = urem i128 %x, 3
@@ -198,16 +202,18 @@ define i128 @urem_i128_5(i128 %x) nounwind {
;
; WIN64-LABEL: urem_i128_5:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: addq %rdx, %rcx
-; WIN64-NEXT: adcq $0, %rcx
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: addq %r8, %r9
+; WIN64-NEXT: adcq $0, %r9
; WIN64-NEXT: movabsq $-3689348814741910323, %rdx # imm = 0xCCCCCCCCCCCCCCCD
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: movq %r9, %rax
; WIN64-NEXT: mulq %rdx
; WIN64-NEXT: shrq $2, %rdx
; WIN64-NEXT: leaq (%rdx,%rdx,4), %rax
-; WIN64-NEXT: subq %rax, %rcx
+; WIN64-NEXT: subq %rax, %r9
+; WIN64-NEXT: movq %r9, (%rcx)
+; WIN64-NEXT: movq $0, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: xorl %edx, %edx
; WIN64-NEXT: retq
entry:
%rem = urem i128 %x, 5
@@ -232,17 +238,19 @@ define i128 @urem_i128_15(i128 %x) nounwind {
;
; WIN64-LABEL: urem_i128_15:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: addq %rdx, %rcx
-; WIN64-NEXT: adcq $0, %rcx
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: addq %r8, %r9
+; WIN64-NEXT: adcq $0, %r9
; WIN64-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: movq %r9, %rax
; WIN64-NEXT: mulq %rdx
; WIN64-NEXT: shrq $3, %rdx
; WIN64-NEXT: leaq (%rdx,%rdx,4), %rax
; WIN64-NEXT: leaq (%rax,%rax,2), %rax
-; WIN64-NEXT: subq %rax, %rcx
+; WIN64-NEXT: subq %rax, %r9
+; WIN64-NEXT: movq %r9, (%rcx)
+; WIN64-NEXT: movq $0, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: xorl %edx, %edx
; WIN64-NEXT: retq
entry:
%rem = urem i128 %x, 15
@@ -268,18 +276,20 @@ define i128 @urem_i128_17(i128 %x) nounwind {
;
; WIN64-LABEL: urem_i128_17:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: addq %rdx, %rcx
-; WIN64-NEXT: adcq $0, %rcx
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: addq %r8, %r9
+; WIN64-NEXT: adcq $0, %r9
; WIN64-NEXT: movabsq $-1085102592571150095, %rdx # imm = 0xF0F0F0F0F0F0F0F1
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: movq %r9, %rax
; WIN64-NEXT: mulq %rdx
; WIN64-NEXT: movq %rdx, %rax
; WIN64-NEXT: andq $-16, %rax
; WIN64-NEXT: shrq $4, %rdx
; WIN64-NEXT: addq %rax, %rdx
-; WIN64-NEXT: subq %rdx, %rcx
+; WIN64-NEXT: subq %rdx, %r9
+; WIN64-NEXT: movq %r9, (%rcx)
+; WIN64-NEXT: movq $0, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: xorl %edx, %edx
; WIN64-NEXT: retq
entry:
%rem = urem i128 %x, 17
@@ -306,9 +316,9 @@ define i128 @urem_i128_255(i128 %x) nounwind {
;
; WIN64-LABEL: urem_i128_255:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: addq %rdx, %rax
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq %rdx, %rax
+; WIN64-NEXT: addq %r8, %rax
; WIN64-NEXT: adcq $0, %rax
; WIN64-NEXT: movabsq $-9187201950435737471, %rdx # imm = 0x8080808080808081
; WIN64-NEXT: mulq %rdx
@@ -316,10 +326,11 @@ define i128 @urem_i128_255(i128 %x) nounwind {
; WIN64-NEXT: movq %rdx, %rax
; WIN64-NEXT: shlq $8, %rax
; WIN64-NEXT: subq %rax, %rdx
-; WIN64-NEXT: addq %rcx, %r8
-; WIN64-NEXT: adcq %rdx, %r8
-; WIN64-NEXT: movq %r8, %rax
-; WIN64-NEXT: xorl %edx, %edx
+; WIN64-NEXT: addq %r8, %r9
+; WIN64-NEXT: adcq %rdx, %r9
+; WIN64-NEXT: movq %r9, (%rcx)
+; WIN64-NEXT: movq $0, 8(%rcx)
+; WIN64-NEXT: movq %rcx, %rax
; WIN64-NEXT: retq
entry:
%rem = urem i128 %x, 255
@@ -345,18 +356,20 @@ define i128 @urem_i128_257(i128 %x) nounwind {
;
; WIN64-LABEL: urem_i128_257:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: addq %rdx, %rcx
-; WIN64-NEXT: adcq $0, %rcx
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: addq %r8, %r9
+; WIN64-NEXT: adcq $0, %r9
; WIN64-NEXT: movabsq $-71777214294589695, %rdx # imm = 0xFF00FF00FF00FF01
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: movq %r9, %rax
; WIN64-NEXT: mulq %rdx
; WIN64-NEXT: movq %rdx, %rax
; WIN64-NEXT: andq $-256, %rax
; WIN64-NEXT: shrq $8, %rdx
; WIN64-NEXT: addq %rax, %rdx
-; WIN64-NEXT: subq %rdx, %rcx
+; WIN64-NEXT: subq %rdx, %r9
+; WIN64-NEXT: movq %r9, (%rcx)
+; WIN64-NEXT: movq $0, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: xorl %edx, %edx
; WIN64-NEXT: retq
entry:
%rem = urem i128 %x, 257
@@ -383,9 +396,9 @@ define i128 @urem_i128_65535(i128 %x) nounwind {
;
; WIN64-LABEL: urem_i128_65535:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: addq %rdx, %rax
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq %rdx, %rax
+; WIN64-NEXT: addq %r8, %rax
; WIN64-NEXT: adcq $0, %rax
; WIN64-NEXT: movabsq $-9223231297218904063, %rdx # imm = 0x8000800080008001
; WIN64-NEXT: mulq %rdx
@@ -393,10 +406,11 @@ define i128 @urem_i128_65535(i128 %x) nounwind {
; WIN64-NEXT: movq %rdx, %rax
; WIN64-NEXT: shlq $16, %rax
; WIN64-NEXT: subq %rax, %rdx
-; WIN64-NEXT: addq %rcx, %r8
-; WIN64-NEXT: adcq %rdx, %r8
-; WIN64-NEXT: movq %r8, %rax
-; WIN64-NEXT: xorl %edx, %edx
+; WIN64-NEXT: addq %r8, %r9
+; WIN64-NEXT: adcq %rdx, %r9
+; WIN64-NEXT: movq %r9, (%rcx)
+; WIN64-NEXT: movq $0, 8(%rcx)
+; WIN64-NEXT: movq %rcx, %rax
; WIN64-NEXT: retq
entry:
%rem = urem i128 %x, 65535
@@ -422,18 +436,20 @@ define i128 @urem_i128_65537(i128 %x) nounwind {
;
; WIN64-LABEL: urem_i128_65537:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: addq %rdx, %rcx
-; WIN64-NEXT: adcq $0, %rcx
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: addq %r8, %r9
+; WIN64-NEXT: adcq $0, %r9
; WIN64-NEXT: movabsq $-281470681808895, %rdx # imm = 0xFFFF0000FFFF0001
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: movq %r9, %rax
; WIN64-NEXT: mulq %rdx
; WIN64-NEXT: movq %rdx, %rax
; WIN64-NEXT: andq $-65536, %rax # imm = 0xFFFF0000
; WIN64-NEXT: shrq $16, %rdx
; WIN64-NEXT: addq %rax, %rdx
-; WIN64-NEXT: subq %rdx, %rcx
+; WIN64-NEXT: subq %rdx, %r9
+; WIN64-NEXT: movq %r9, (%rcx)
+; WIN64-NEXT: movq $0, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: xorl %edx, %edx
; WIN64-NEXT: retq
entry:
%rem = urem i128 %x, 65537
@@ -461,20 +477,23 @@ define i128 @urem_i128_12(i128 %x) nounwind {
;
; WIN64-LABEL: urem_i128_12:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rcx, %r8
-; WIN64-NEXT: shrdq $2, %rdx, %r8
-; WIN64-NEXT: shrq $2, %rdx
-; WIN64-NEXT: addq %rdx, %r8
-; WIN64-NEXT: adcq $0, %r8
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: shrdq $2, %r8, %r10
+; WIN64-NEXT: shrq $2, %r8
+; WIN64-NEXT: addq %r8, %r10
+; WIN64-NEXT: adcq $0, %r10
; WIN64-NEXT: movabsq $-6148914691236517205, %rdx # imm = 0xAAAAAAAAAAAAAAAB
-; WIN64-NEXT: movq %r8, %rax
+; WIN64-NEXT: movq %r10, %rax
; WIN64-NEXT: mulq %rdx
; WIN64-NEXT: shrq %rdx
; WIN64-NEXT: leal (%rdx,%rdx,2), %eax
-; WIN64-NEXT: subl %eax, %r8d
-; WIN64-NEXT: andl $3, %ecx
-; WIN64-NEXT: leaq (%rcx,%r8,4), %rax
-; WIN64-NEXT: xorl %edx, %edx
+; WIN64-NEXT: subl %eax, %r10d
+; WIN64-NEXT: andl $3, %r9d
+; WIN64-NEXT: leaq (%r9,%r10,4), %rax
+; WIN64-NEXT: movq %rax, (%rcx)
+; WIN64-NEXT: movq $0, 8(%rcx)
+; WIN64-NEXT: movq %rcx, %rax
; WIN64-NEXT: retq
entry:
%rem = urem i128 %x, 12
@@ -506,25 +525,28 @@ define i128 @udiv_i128_3(i128 %x) nounwind {
;
; WIN64-LABEL: udiv_i128_3:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rcx, %r9
-; WIN64-NEXT: addq %rdx, %r9
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: movabsq $-6148914691236517205, %r10 # imm = 0xAAAAAAAAAAAAAAAB
-; WIN64-NEXT: movq %r9, %rax
-; WIN64-NEXT: mulq %r10
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: addq %r8, %r10
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: movabsq $-6148914691236517205, %r11 # imm = 0xAAAAAAAAAAAAAAAB
+; WIN64-NEXT: movq %r10, %rax
+; WIN64-NEXT: mulq %r11
; WIN64-NEXT: shrq %rdx
; WIN64-NEXT: leaq (%rdx,%rdx,2), %rax
-; WIN64-NEXT: subq %rax, %r9
-; WIN64-NEXT: subq %r9, %rcx
+; WIN64-NEXT: subq %rax, %r10
+; WIN64-NEXT: subq %r10, %r9
; WIN64-NEXT: sbbq $0, %r8
-; WIN64-NEXT: movabsq $-6148914691236517206, %r9 # imm = 0xAAAAAAAAAAAAAAAA
-; WIN64-NEXT: imulq %rcx, %r9
+; WIN64-NEXT: movabsq $-6148914691236517206, %r10 # imm = 0xAAAAAAAAAAAAAAAA
+; WIN64-NEXT: imulq %r9, %r10
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: addq %r10, %rdx
+; WIN64-NEXT: imulq %r11, %r8
+; WIN64-NEXT: addq %rdx, %r8
+; WIN64-NEXT: movq %rax, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %r10
-; WIN64-NEXT: addq %r9, %rdx
-; WIN64-NEXT: imulq %r10, %r8
-; WIN64-NEXT: addq %r8, %rdx
; WIN64-NEXT: retq
entry:
%rem = udiv i128 %x, 3
@@ -556,25 +578,28 @@ define i128 @udiv_i128_5(i128 %x) nounwind {
;
; WIN64-LABEL: udiv_i128_5:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rcx, %r9
-; WIN64-NEXT: addq %rdx, %r9
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: movabsq $-3689348814741910323, %r10 # imm = 0xCCCCCCCCCCCCCCCD
-; WIN64-NEXT: movq %r9, %rax
-; WIN64-NEXT: mulq %r10
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: addq %r8, %r10
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: movabsq $-3689348814741910323, %r11 # imm = 0xCCCCCCCCCCCCCCCD
+; WIN64-NEXT: movq %r10, %rax
+; WIN64-NEXT: mulq %r11
; WIN64-NEXT: shrq $2, %rdx
; WIN64-NEXT: leaq (%rdx,%rdx,4), %rax
-; WIN64-NEXT: subq %rax, %r9
-; WIN64-NEXT: subq %r9, %rcx
+; WIN64-NEXT: subq %rax, %r10
+; WIN64-NEXT: subq %r10, %r9
; WIN64-NEXT: sbbq $0, %r8
-; WIN64-NEXT: movabsq $-3689348814741910324, %r9 # imm = 0xCCCCCCCCCCCCCCCC
-; WIN64-NEXT: imulq %rcx, %r9
+; WIN64-NEXT: movabsq $-3689348814741910324, %r10 # imm = 0xCCCCCCCCCCCCCCCC
+; WIN64-NEXT: imulq %r9, %r10
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: addq %r10, %rdx
+; WIN64-NEXT: imulq %r11, %r8
+; WIN64-NEXT: addq %rdx, %r8
+; WIN64-NEXT: movq %rax, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %r10
-; WIN64-NEXT: addq %r9, %rdx
-; WIN64-NEXT: imulq %r10, %r8
-; WIN64-NEXT: addq %r8, %rdx
; WIN64-NEXT: retq
entry:
%rem = udiv i128 %x, 5
@@ -608,27 +633,30 @@ define i128 @udiv_i128_15(i128 %x) nounwind {
;
; WIN64-LABEL: udiv_i128_15:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rcx, %r9
-; WIN64-NEXT: addq %rdx, %r9
-; WIN64-NEXT: adcq $0, %r9
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: addq %r8, %r10
+; WIN64-NEXT: adcq $0, %r10
; WIN64-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889
-; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: movq %r10, %rax
; WIN64-NEXT: mulq %rdx
; WIN64-NEXT: shrq $3, %rdx
; WIN64-NEXT: leaq (%rdx,%rdx,4), %rax
; WIN64-NEXT: leaq (%rax,%rax,2), %rax
-; WIN64-NEXT: subq %rax, %r9
-; WIN64-NEXT: subq %r9, %rcx
+; WIN64-NEXT: subq %rax, %r10
+; WIN64-NEXT: subq %r10, %r9
; WIN64-NEXT: sbbq $0, %r8
-; WIN64-NEXT: movabsq $-1229782938247303442, %r9 # imm = 0xEEEEEEEEEEEEEEEE
-; WIN64-NEXT: imulq %rcx, %r9
-; WIN64-NEXT: movabsq $-1229782938247303441, %r10 # imm = 0xEEEEEEEEEEEEEEEF
+; WIN64-NEXT: movabsq $-1229782938247303442, %r10 # imm = 0xEEEEEEEEEEEEEEEE
+; WIN64-NEXT: imulq %r9, %r10
+; WIN64-NEXT: movabsq $-1229782938247303441, %r11 # imm = 0xEEEEEEEEEEEEEEEF
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: addq %r10, %rdx
+; WIN64-NEXT: imulq %r11, %r8
+; WIN64-NEXT: addq %rdx, %r8
+; WIN64-NEXT: movq %rax, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %r10
-; WIN64-NEXT: addq %r9, %rdx
-; WIN64-NEXT: imulq %r10, %r8
-; WIN64-NEXT: addq %r8, %rdx
; WIN64-NEXT: retq
entry:
%rem = udiv i128 %x, 15
@@ -662,27 +690,30 @@ define i128 @udiv_i128_17(i128 %x) nounwind {
;
; WIN64-LABEL: udiv_i128_17:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rcx, %r9
-; WIN64-NEXT: addq %rdx, %r9
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: movabsq $-1085102592571150095, %r10 # imm = 0xF0F0F0F0F0F0F0F1
-; WIN64-NEXT: movq %r9, %rax
-; WIN64-NEXT: mulq %r10
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: addq %r8, %r10
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: movabsq $-1085102592571150095, %r11 # imm = 0xF0F0F0F0F0F0F0F1
+; WIN64-NEXT: movq %r10, %rax
+; WIN64-NEXT: mulq %r11
; WIN64-NEXT: movq %rdx, %rax
; WIN64-NEXT: andq $-16, %rax
; WIN64-NEXT: shrq $4, %rdx
; WIN64-NEXT: addq %rax, %rdx
-; WIN64-NEXT: subq %rdx, %r9
-; WIN64-NEXT: subq %r9, %rcx
+; WIN64-NEXT: subq %rdx, %r10
+; WIN64-NEXT: subq %r10, %r9
; WIN64-NEXT: sbbq $0, %r8
-; WIN64-NEXT: movabsq $-1085102592571150096, %r9 # imm = 0xF0F0F0F0F0F0F0F0
-; WIN64-NEXT: imulq %rcx, %r9
+; WIN64-NEXT: movabsq $-1085102592571150096, %r10 # imm = 0xF0F0F0F0F0F0F0F0
+; WIN64-NEXT: imulq %r9, %r10
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: addq %r10, %rdx
+; WIN64-NEXT: imulq %r11, %r8
+; WIN64-NEXT: addq %rdx, %r8
+; WIN64-NEXT: movq %rax, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %r10
-; WIN64-NEXT: addq %r9, %rdx
-; WIN64-NEXT: imulq %r10, %r8
-; WIN64-NEXT: addq %r8, %rdx
; WIN64-NEXT: retq
entry:
%rem = udiv i128 %x, 17
@@ -718,9 +749,9 @@ define i128 @udiv_i128_255(i128 %x) nounwind {
;
; WIN64-LABEL: udiv_i128_255:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: addq %rdx, %rax
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq %rdx, %rax
+; WIN64-NEXT: addq %r8, %rax
; WIN64-NEXT: adcq $0, %rax
; WIN64-NEXT: movabsq $-9187201950435737471, %rdx # imm = 0x8080808080808081
; WIN64-NEXT: mulq %rdx
@@ -728,19 +759,22 @@ define i128 @udiv_i128_255(i128 %x) nounwind {
; WIN64-NEXT: movq %rdx, %rax
; WIN64-NEXT: shlq $8, %rax
; WIN64-NEXT: subq %rax, %rdx
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: movq %r9, %rax
; WIN64-NEXT: addq %r8, %rax
; WIN64-NEXT: adcq %rdx, %rax
-; WIN64-NEXT: subq %rax, %rcx
+; WIN64-NEXT: subq %rax, %r9
; WIN64-NEXT: sbbq $0, %r8
-; WIN64-NEXT: movabsq $-72340172838076674, %r9 # imm = 0xFEFEFEFEFEFEFEFE
-; WIN64-NEXT: imulq %rcx, %r9
-; WIN64-NEXT: movabsq $-72340172838076673, %r10 # imm = 0xFEFEFEFEFEFEFEFF
+; WIN64-NEXT: movabsq $-72340172838076674, %r10 # imm = 0xFEFEFEFEFEFEFEFE
+; WIN64-NEXT: imulq %r9, %r10
+; WIN64-NEXT: movabsq $-72340172838076673, %r11 # imm = 0xFEFEFEFEFEFEFEFF
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: addq %r10, %rdx
+; WIN64-NEXT: imulq %r11, %r8
+; WIN64-NEXT: addq %rdx, %r8
+; WIN64-NEXT: movq %rax, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %r10
-; WIN64-NEXT: addq %r9, %rdx
-; WIN64-NEXT: imulq %r10, %r8
-; WIN64-NEXT: addq %r8, %rdx
; WIN64-NEXT: retq
entry:
%rem = udiv i128 %x, 255
@@ -774,27 +808,30 @@ define i128 @udiv_i128_257(i128 %x) nounwind {
;
; WIN64-LABEL: udiv_i128_257:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rcx, %r9
-; WIN64-NEXT: addq %rdx, %r9
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: movabsq $-71777214294589695, %r10 # imm = 0xFF00FF00FF00FF01
-; WIN64-NEXT: movq %r9, %rax
-; WIN64-NEXT: mulq %r10
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: addq %r8, %r10
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: movabsq $-71777214294589695, %r11 # imm = 0xFF00FF00FF00FF01
+; WIN64-NEXT: movq %r10, %rax
+; WIN64-NEXT: mulq %r11
; WIN64-NEXT: movq %rdx, %rax
; WIN64-NEXT: andq $-256, %rax
; WIN64-NEXT: shrq $8, %rdx
; WIN64-NEXT: addq %rax, %rdx
-; WIN64-NEXT: subq %rdx, %r9
-; WIN64-NEXT: subq %r9, %rcx
+; WIN64-NEXT: subq %rdx, %r10
+; WIN64-NEXT: subq %r10, %r9
; WIN64-NEXT: sbbq $0, %r8
-; WIN64-NEXT: movabsq $-71777214294589696, %r9 # imm = 0xFF00FF00FF00FF00
-; WIN64-NEXT: imulq %rcx, %r9
+; WIN64-NEXT: movabsq $-71777214294589696, %r10 # imm = 0xFF00FF00FF00FF00
+; WIN64-NEXT: imulq %r9, %r10
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: addq %r10, %rdx
+; WIN64-NEXT: imulq %r11, %r8
+; WIN64-NEXT: addq %rdx, %r8
+; WIN64-NEXT: movq %rax, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %r10
-; WIN64-NEXT: addq %r9, %rdx
-; WIN64-NEXT: imulq %r10, %r8
-; WIN64-NEXT: addq %r8, %rdx
; WIN64-NEXT: retq
entry:
%rem = udiv i128 %x, 257
@@ -830,9 +867,9 @@ define i128 @udiv_i128_65535(i128 %x) nounwind {
;
; WIN64-LABEL: udiv_i128_65535:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: addq %rdx, %rax
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq %rdx, %rax
+; WIN64-NEXT: addq %r8, %rax
; WIN64-NEXT: adcq $0, %rax
; WIN64-NEXT: movabsq $-9223231297218904063, %rdx # imm = 0x8000800080008001
; WIN64-NEXT: mulq %rdx
@@ -840,19 +877,22 @@ define i128 @udiv_i128_65535(i128 %x) nounwind {
; WIN64-NEXT: movq %rdx, %rax
; WIN64-NEXT: shlq $16, %rax
; WIN64-NEXT: subq %rax, %rdx
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: movq %r9, %rax
; WIN64-NEXT: addq %r8, %rax
; WIN64-NEXT: adcq %rdx, %rax
-; WIN64-NEXT: subq %rax, %rcx
+; WIN64-NEXT: subq %rax, %r9
; WIN64-NEXT: sbbq $0, %r8
-; WIN64-NEXT: movabsq $-281479271743490, %r9 # imm = 0xFFFEFFFEFFFEFFFE
-; WIN64-NEXT: imulq %rcx, %r9
-; WIN64-NEXT: movabsq $-281479271743489, %r10 # imm = 0xFFFEFFFEFFFEFFFF
+; WIN64-NEXT: movabsq $-281479271743490, %r10 # imm = 0xFFFEFFFEFFFEFFFE
+; WIN64-NEXT: imulq %r9, %r10
+; WIN64-NEXT: movabsq $-281479271743489, %r11 # imm = 0xFFFEFFFEFFFEFFFF
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: addq %r10, %rdx
+; WIN64-NEXT: imulq %r11, %r8
+; WIN64-NEXT: addq %rdx, %r8
+; WIN64-NEXT: movq %rax, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %r10
-; WIN64-NEXT: addq %r9, %rdx
-; WIN64-NEXT: imulq %r10, %r8
-; WIN64-NEXT: addq %r8, %rdx
; WIN64-NEXT: retq
entry:
%rem = udiv i128 %x, 65535
@@ -886,27 +926,30 @@ define i128 @udiv_i128_65537(i128 %x) nounwind {
;
; WIN64-LABEL: udiv_i128_65537:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rcx, %r9
-; WIN64-NEXT: addq %rdx, %r9
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: movabsq $-281470681808895, %r10 # imm = 0xFFFF0000FFFF0001
-; WIN64-NEXT: movq %r9, %rax
-; WIN64-NEXT: mulq %r10
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: addq %r8, %r10
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: movabsq $-281470681808895, %r11 # imm = 0xFFFF0000FFFF0001
+; WIN64-NEXT: movq %r10, %rax
+; WIN64-NEXT: mulq %r11
; WIN64-NEXT: movq %rdx, %rax
; WIN64-NEXT: andq $-65536, %rax # imm = 0xFFFF0000
; WIN64-NEXT: shrq $16, %rdx
; WIN64-NEXT: addq %rax, %rdx
-; WIN64-NEXT: subq %rdx, %r9
-; WIN64-NEXT: subq %r9, %rcx
+; WIN64-NEXT: subq %rdx, %r10
+; WIN64-NEXT: subq %r10, %r9
; WIN64-NEXT: sbbq $0, %r8
-; WIN64-NEXT: movabsq $-281470681808896, %r9 # imm = 0xFFFF0000FFFF0000
-; WIN64-NEXT: imulq %rcx, %r9
+; WIN64-NEXT: movabsq $-281470681808896, %r10 # imm = 0xFFFF0000FFFF0000
+; WIN64-NEXT: imulq %r9, %r10
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: addq %r10, %rdx
+; WIN64-NEXT: imulq %r11, %r8
+; WIN64-NEXT: addq %rdx, %r8
+; WIN64-NEXT: movq %rax, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %r10
-; WIN64-NEXT: addq %r9, %rdx
-; WIN64-NEXT: imulq %r10, %r8
-; WIN64-NEXT: addq %r8, %rdx
; WIN64-NEXT: retq
entry:
%rem = udiv i128 %x, 65537
@@ -940,27 +983,30 @@ define i128 @udiv_i128_12(i128 %x) nounwind {
;
; WIN64-LABEL: udiv_i128_12:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: shrdq $2, %rdx, %rcx
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: shrdq $2, %r8, %r9
; WIN64-NEXT: shrq $2, %r8
-; WIN64-NEXT: movq %rcx, %r9
-; WIN64-NEXT: addq %r8, %r9
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: movabsq $-6148914691236517205, %r10 # imm = 0xAAAAAAAAAAAAAAAB
-; WIN64-NEXT: movq %r9, %rax
-; WIN64-NEXT: mulq %r10
+; WIN64-NEXT: movq %r9, %r10
+; WIN64-NEXT: addq %r8, %r10
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: movabsq $-6148914691236517205, %r11 # imm = 0xAAAAAAAAAAAAAAAB
+; WIN64-NEXT: movq %r10, %rax
+; WIN64-NEXT: mulq %r11
; WIN64-NEXT: shrq %rdx
; WIN64-NEXT: leaq (%rdx,%rdx,2), %rax
-; WIN64-NEXT: subq %rax, %r9
-; WIN64-NEXT: subq %r9, %rcx
+; WIN64-NEXT: subq %rax, %r10
+; WIN64-NEXT: subq %r10, %r9
; WIN64-NEXT: sbbq $0, %r8
-; WIN64-NEXT: movabsq $-6148914691236517206, %r9 # imm = 0xAAAAAAAAAAAAAAAA
-; WIN64-NEXT: imulq %rcx, %r9
+; WIN64-NEXT: movabsq $-6148914691236517206, %r10 # imm = 0xAAAAAAAAAAAAAAAA
+; WIN64-NEXT: imulq %r9, %r10
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: addq %r10, %rdx
+; WIN64-NEXT: imulq %r11, %r8
+; WIN64-NEXT: addq %rdx, %r8
+; WIN64-NEXT: movq %rax, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %r10
-; WIN64-NEXT: addq %r9, %rdx
-; WIN64-NEXT: imulq %r10, %r8
-; WIN64-NEXT: addq %r8, %rdx
; WIN64-NEXT: retq
entry:
%rem = udiv i128 %x, 12
@@ -981,19 +1027,26 @@ define i128 @urem_i128_3_minsize(i128 %x) nounwind minsize {
;
; WIN64-LABEL: urem_i128_3_minsize:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: subq $72, %rsp
+; WIN64-NEXT: pushq %rsi
+; WIN64-NEXT: pushq %rdi
+; WIN64-NEXT: subq $88, %rsp
+; WIN64-NEXT: movq %rcx, %rsi
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rax
-; WIN64-NEXT: movq %rdx, 8(%rax)
-; WIN64-NEXT: movq %rcx, (%rax)
-; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
-; WIN64-NEXT: movq $3, (%rdx)
-; WIN64-NEXT: andq $0, 8(%rdx)
-; WIN64-NEXT: movq %rax, %rcx
+; WIN64-NEXT: movq %r8, 8(%rax)
+; WIN64-NEXT: movq %rdx, (%rax)
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %r8
+; WIN64-NEXT: movq $3, (%r8)
+; WIN64-NEXT: andq $0, 8(%r8)
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rdi
+; WIN64-NEXT: movq %rdi, %rcx
+; WIN64-NEXT: movq %rax, %rdx
; WIN64-NEXT: callq __umodti3
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; WIN64-NEXT: movq %xmm0, %rdx
-; WIN64-NEXT: addq $72, %rsp
+; WIN64-NEXT: movaps (%rdi), %xmm0
+; WIN64-NEXT: movaps %xmm0, (%rsi)
+; WIN64-NEXT: movq %rsi, %rax
+; WIN64-NEXT: addq $88, %rsp
+; WIN64-NEXT: popq %rdi
+; WIN64-NEXT: popq %rsi
; WIN64-NEXT: retq
entry:
%rem = urem i128 %x, 3
@@ -1013,19 +1066,26 @@ define i128 @urem_i128_3_optsize(i128 %x) nounwind optsize {
;
; WIN64-LABEL: urem_i128_3_optsize:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: subq $72, %rsp
+; WIN64-NEXT: pushq %rsi
+; WIN64-NEXT: pushq %rdi
+; WIN64-NEXT: subq $88, %rsp
+; WIN64-NEXT: movq %rcx, %rsi
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rax
-; WIN64-NEXT: movq %rdx, 8(%rax)
-; WIN64-NEXT: movq %rcx, (%rax)
-; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
-; WIN64-NEXT: movq $3, (%rdx)
-; WIN64-NEXT: movq $0, 8(%rdx)
-; WIN64-NEXT: movq %rax, %rcx
+; WIN64-NEXT: movq %r8, 8(%rax)
+; WIN64-NEXT: movq %rdx, (%rax)
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %r8
+; WIN64-NEXT: movq $3, (%r8)
+; WIN64-NEXT: movq $0, 8(%r8)
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rdi
+; WIN64-NEXT: movq %rdi, %rcx
+; WIN64-NEXT: movq %rax, %rdx
; WIN64-NEXT: callq __umodti3
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; WIN64-NEXT: movq %xmm0, %rdx
-; WIN64-NEXT: addq $72, %rsp
+; WIN64-NEXT: movaps (%rdi), %xmm0
+; WIN64-NEXT: movaps %xmm0, (%rsi)
+; WIN64-NEXT: movq %rsi, %rax
+; WIN64-NEXT: addq $88, %rsp
+; WIN64-NEXT: popq %rdi
+; WIN64-NEXT: popq %rsi
; WIN64-NEXT: retq
entry:
%rem = urem i128 %x, 3
@@ -1067,35 +1127,38 @@ define i128 @udiv_i128_100(i128 %x) nounwind {
;
; WIN64-LABEL: udiv_i128_100:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: shrdq $62, %rdx, %rax
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq %rdx, %rax
+; WIN64-NEXT: shrdq $62, %r8, %rax
; WIN64-NEXT: movabsq $1152921504606846975, %rdx # imm = 0xFFFFFFFFFFFFFFF
; WIN64-NEXT: andq %rdx, %rax
-; WIN64-NEXT: shrdq $2, %r8, %rcx
-; WIN64-NEXT: andq %rcx, %rdx
+; WIN64-NEXT: shrdq $2, %r8, %r9
+; WIN64-NEXT: andq %r9, %rdx
; WIN64-NEXT: addq %rax, %rdx
-; WIN64-NEXT: movq %r8, %r9
-; WIN64-NEXT: shrq $58, %r9
-; WIN64-NEXT: addq %rdx, %r9
+; WIN64-NEXT: movq %r8, %r10
+; WIN64-NEXT: shrq $58, %r10
+; WIN64-NEXT: addq %rdx, %r10
; WIN64-NEXT: movabsq $2951479051793528259, %rdx # imm = 0x28F5C28F5C28F5C3
-; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: movq %r10, %rax
; WIN64-NEXT: mulq %rdx
; WIN64-NEXT: shrq $2, %rdx
; WIN64-NEXT: leaq (%rdx,%rdx,4), %rax
; WIN64-NEXT: leaq (%rax,%rax,4), %rax
-; WIN64-NEXT: subq %rax, %r9
+; WIN64-NEXT: subq %rax, %r10
; WIN64-NEXT: shrq $2, %r8
-; WIN64-NEXT: subq %r9, %rcx
+; WIN64-NEXT: subq %r10, %r9
; WIN64-NEXT: sbbq $0, %r8
-; WIN64-NEXT: movabsq $2951479051793528258, %r9 # imm = 0x28F5C28F5C28F5C2
-; WIN64-NEXT: imulq %rcx, %r9
-; WIN64-NEXT: movabsq $-8116567392432202711, %r10 # imm = 0x8F5C28F5C28F5C29
+; WIN64-NEXT: movabsq $2951479051793528258, %r10 # imm = 0x28F5C28F5C28F5C2
+; WIN64-NEXT: imulq %r9, %r10
+; WIN64-NEXT: movabsq $-8116567392432202711, %r11 # imm = 0x8F5C28F5C28F5C29
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: addq %r10, %rdx
+; WIN64-NEXT: imulq %r11, %r8
+; WIN64-NEXT: addq %rdx, %r8
+; WIN64-NEXT: movq %rax, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %r10
-; WIN64-NEXT: addq %r9, %rdx
-; WIN64-NEXT: imulq %r10, %r8
-; WIN64-NEXT: addq %r8, %rdx
; WIN64-NEXT: retq
entry:
%rem = udiv i128 %x, 100
@@ -1136,34 +1199,36 @@ define i128 @udiv_i128_13(i128 %x) nounwind {
;
; WIN64-LABEL: udiv_i128_13:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
+; WIN64-NEXT: movq %rdx, %r9
; WIN64-NEXT: movabsq $1152921504606846975, %rax # imm = 0xFFFFFFFFFFFFFFF
-; WIN64-NEXT: movq %rcx, %rdx
; WIN64-NEXT: andq %rax, %rdx
-; WIN64-NEXT: movq %rcx, %r10
-; WIN64-NEXT: shrdq $60, %r8, %r10
-; WIN64-NEXT: andq %rax, %r10
-; WIN64-NEXT: addq %rdx, %r10
-; WIN64-NEXT: movq %r8, %r9
-; WIN64-NEXT: shrq $56, %r9
-; WIN64-NEXT: addq %r10, %r9
+; WIN64-NEXT: movq %r9, %r11
+; WIN64-NEXT: shrdq $60, %r8, %r11
+; WIN64-NEXT: andq %rax, %r11
+; WIN64-NEXT: addq %rdx, %r11
+; WIN64-NEXT: movq %r8, %r10
+; WIN64-NEXT: shrq $56, %r10
+; WIN64-NEXT: addq %r11, %r10
; WIN64-NEXT: movabsq $2837960626724546403, %rdx # imm = 0x2762762762762763
-; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: movq %r10, %rax
; WIN64-NEXT: mulq %rdx
; WIN64-NEXT: shrq %rdx
; WIN64-NEXT: leaq (%rdx,%rdx,2), %rax
; WIN64-NEXT: leaq (%rdx,%rax,4), %rax
-; WIN64-NEXT: subq %rax, %r9
-; WIN64-NEXT: subq %r9, %rcx
+; WIN64-NEXT: subq %rax, %r10
+; WIN64-NEXT: subq %r10, %r9
; WIN64-NEXT: sbbq $0, %r8
-; WIN64-NEXT: movabsq $-4256940940086819604, %r9 # imm = 0xC4EC4EC4EC4EC4EC
-; WIN64-NEXT: imulq %rcx, %r9
-; WIN64-NEXT: movabsq $5675921253449092805, %r10 # imm = 0x4EC4EC4EC4EC4EC5
+; WIN64-NEXT: movabsq $-4256940940086819604, %r10 # imm = 0xC4EC4EC4EC4EC4EC
+; WIN64-NEXT: imulq %r9, %r10
+; WIN64-NEXT: movabsq $5675921253449092805, %r11 # imm = 0x4EC4EC4EC4EC4EC5
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: addq %r10, %rdx
+; WIN64-NEXT: imulq %r11, %r8
+; WIN64-NEXT: addq %rdx, %r8
+; WIN64-NEXT: movq %rax, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %r10
-; WIN64-NEXT: addq %r9, %rdx
-; WIN64-NEXT: imulq %r10, %r8
-; WIN64-NEXT: addq %r8, %rdx
; WIN64-NEXT: retq
entry:
%rem = udiv i128 %x, 13
@@ -1207,36 +1272,39 @@ define i128 @udiv_magic_preshift_and_postshift(i128 %x) nounwind {
; WIN64-LABEL: udiv_magic_preshift_and_postshift:
; WIN64: # %bb.0:
; WIN64-NEXT: pushq %rsi
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rcx, %r9
-; WIN64-NEXT: shrdq $1, %rdx, %r9
-; WIN64-NEXT: movabsq $3297741957311204758, %r11 # imm = 0x2DC3EED6866F8D96
-; WIN64-NEXT: movq %r9, %rax
-; WIN64-NEXT: mulq %r11
-; WIN64-NEXT: movq %rax, %r10
-; WIN64-NEXT: movq %rdx, %rcx
-; WIN64-NEXT: movabsq $3091633084979254461, %rsi # imm = 0x2AE7AFE91E0894BD
-; WIN64-NEXT: movq %r9, %rax
-; WIN64-NEXT: mulq %rsi
-; WIN64-NEXT: movq %rdx, %r9
-; WIN64-NEXT: addq %r10, %r9
-; WIN64-NEXT: adcq $0, %rcx
-; WIN64-NEXT: shrq %r8
-; WIN64-NEXT: movq %r8, %rax
-; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: pushq %rdi
+; WIN64-NEXT: movq %r8, %r9
; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: shrdq $1, %r8, %r10
+; WIN64-NEXT: movabsq $3297741957311204758, %rsi # imm = 0x2DC3EED6866F8D96
+; WIN64-NEXT: movq %r10, %rax
+; WIN64-NEXT: mulq %rsi
; WIN64-NEXT: movq %rax, %r11
-; WIN64-NEXT: movq %r8, %rax
+; WIN64-NEXT: movq %rdx, %r8
+; WIN64-NEXT: movabsq $3091633084979254461, %rdi # imm = 0x2AE7AFE91E0894BD
+; WIN64-NEXT: movq %r10, %rax
+; WIN64-NEXT: mulq %rdi
+; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: addq %r11, %r10
+; WIN64-NEXT: adcq $0, %r8
+; WIN64-NEXT: shrq %r9
+; WIN64-NEXT: movq %r9, %rax
; WIN64-NEXT: mulq %rsi
-; WIN64-NEXT: addq %r9, %rax
-; WIN64-NEXT: adcq %rdx, %rcx
-; WIN64-NEXT: adcq $0, %r10
-; WIN64-NEXT: addq %r11, %rcx
-; WIN64-NEXT: adcq $0, %r10
-; WIN64-NEXT: shrdq $5, %r10, %rcx
-; WIN64-NEXT: shrq $5, %r10
+; WIN64-NEXT: movq %rdx, %r11
+; WIN64-NEXT: movq %rax, %rsi
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %rdi
+; WIN64-NEXT: addq %r10, %rax
+; WIN64-NEXT: adcq %r8, %rdx
+; WIN64-NEXT: adcq $0, %r11
+; WIN64-NEXT: addq %rsi, %rdx
+; WIN64-NEXT: adcq $0, %r11
+; WIN64-NEXT: shrdq $5, %r11, %rdx
+; WIN64-NEXT: shrq $5, %r11
+; WIN64-NEXT: movq %r11, 8(%rcx)
+; WIN64-NEXT: movq %rdx, (%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: movq %r10, %rdx
+; WIN64-NEXT: popq %rdi
; WIN64-NEXT: popq %rsi
; WIN64-NEXT: retq
%ret = udiv i128 %x, 358
@@ -1290,50 +1358,53 @@ define i128 @urem_magic_preshift_and_postshift(i128 %x) nounwind {
;
; WIN64-LABEL: urem_magic_preshift_and_postshift:
; WIN64: # %bb.0:
+; WIN64-NEXT: pushq %r14
; WIN64-NEXT: pushq %rsi
; WIN64-NEXT: pushq %rdi
; WIN64-NEXT: pushq %rbx
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rcx, %r10
-; WIN64-NEXT: shrdq $1, %rdx, %r10
-; WIN64-NEXT: movabsq $3297741957311204758, %rdi # imm = 0x2DC3EED6866F8D96
-; WIN64-NEXT: movq %r10, %rax
-; WIN64-NEXT: mulq %rdi
-; WIN64-NEXT: movq %rax, %r11
; WIN64-NEXT: movq %rdx, %r9
-; WIN64-NEXT: movabsq $3091633084979254461, %rbx # imm = 0x2AE7AFE91E0894BD
-; WIN64-NEXT: movq %r10, %rax
-; WIN64-NEXT: mulq %rbx
-; WIN64-NEXT: movq %rdx, %rsi
-; WIN64-NEXT: addq %r11, %rsi
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: movq %r8, %r11
-; WIN64-NEXT: shrq %r11
+; WIN64-NEXT: movq %rdx, %r11
+; WIN64-NEXT: shrdq $1, %r8, %r11
+; WIN64-NEXT: movabsq $3297741957311204758, %rbx # imm = 0x2DC3EED6866F8D96
; WIN64-NEXT: movq %r11, %rax
-; WIN64-NEXT: mulq %rdi
+; WIN64-NEXT: mulq %rbx
+; WIN64-NEXT: movq %rax, %rsi
; WIN64-NEXT: movq %rdx, %r10
-; WIN64-NEXT: movq %rax, %rdi
+; WIN64-NEXT: movabsq $3091633084979254461, %r14 # imm = 0x2AE7AFE91E0894BD
; WIN64-NEXT: movq %r11, %rax
-; WIN64-NEXT: mulq %rbx
-; WIN64-NEXT: addq %rsi, %rax
-; WIN64-NEXT: adcq %rdx, %r9
+; WIN64-NEXT: mulq %r14
+; WIN64-NEXT: movq %rdx, %rdi
+; WIN64-NEXT: addq %rsi, %rdi
; WIN64-NEXT: adcq $0, %r10
-; WIN64-NEXT: addq %rdi, %r9
-; WIN64-NEXT: adcq $0, %r10
-; WIN64-NEXT: shrdq $5, %r10, %r9
+; WIN64-NEXT: movq %r8, %rsi
+; WIN64-NEXT: shrq %rsi
+; WIN64-NEXT: movq %rsi, %rax
+; WIN64-NEXT: mulq %rbx
+; WIN64-NEXT: movq %rdx, %r11
+; WIN64-NEXT: movq %rax, %rbx
+; WIN64-NEXT: movq %rsi, %rax
+; WIN64-NEXT: mulq %r14
+; WIN64-NEXT: addq %rdi, %rax
+; WIN64-NEXT: adcq %rdx, %r10
+; WIN64-NEXT: adcq $0, %r11
+; WIN64-NEXT: addq %rbx, %r10
+; WIN64-NEXT: adcq $0, %r11
+; WIN64-NEXT: shrdq $5, %r11, %r10
; WIN64-NEXT: movl $358, %edx # imm = 0x166
-; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: movq %r10, %rax
; WIN64-NEXT: mulq %rdx
-; WIN64-NEXT: shrq $5, %r10
-; WIN64-NEXT: imulq $358, %r10, %r9 # imm = 0x166
-; WIN64-NEXT: addq %rdx, %r9
-; WIN64-NEXT: subq %rax, %rcx
-; WIN64-NEXT: sbbq %r9, %r8
+; WIN64-NEXT: shrq $5, %r11
+; WIN64-NEXT: imulq $358, %r11, %r10 # imm = 0x166
+; WIN64-NEXT: addq %rdx, %r10
+; WIN64-NEXT: subq %rax, %r9
+; WIN64-NEXT: sbbq %r10, %r8
+; WIN64-NEXT: movq %r9, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: movq %r8, %rdx
; WIN64-NEXT: popq %rbx
; WIN64-NEXT: popq %rdi
; WIN64-NEXT: popq %rsi
+; WIN64-NEXT: popq %r14
; WIN64-NEXT: retq
%ret = urem i128 %x, 358
ret i128 %ret
@@ -1360,20 +1431,20 @@ define i128 @udiv_magic_large_preshift(i128 %x) nounwind {
;
; WIN64-LABEL: udiv_magic_large_preshift:
; WIN64: # %bb.0:
-; WIN64-NEXT: movq %rdx, %rcx
-; WIN64-NEXT: shrq $36, %rcx
+; WIN64-NEXT: shrq $36, %r8
; WIN64-NEXT: movabsq $103054436165975148, %rdx # imm = 0x16E1F76B4337C6C
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: movq %r8, %rax
; WIN64-NEXT: mulq %rdx
-; WIN64-NEXT: movq %rax, %r8
-; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq %rax, %r9
+; WIN64-NEXT: movq %rdx, %r10
; WIN64-NEXT: movabsq $-5667993989128633178, %rdx # imm = 0xB1573D7F48F044A6
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: movq %r8, %rax
; WIN64-NEXT: mulq %rdx
-; WIN64-NEXT: addq %r8, %rdx
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: movq %r9, %rax
-; WIN64-NEXT: xorl %edx, %edx
+; WIN64-NEXT: addq %r9, %rdx
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: movq %r10, (%rcx)
+; WIN64-NEXT: movq $0, 8(%rcx)
+; WIN64-NEXT: movq %rcx, %rax
; WIN64-NEXT: retq
%ret = udiv i128 %x, 226909457440853062867909873762304 ; = 179 * 2^100
ret i128 %ret
@@ -1404,24 +1475,27 @@ define i128 @urem_magic_large_preshift(i128 %x) nounwind {
;
; WIN64-LABEL: urem_magic_large_preshift:
; WIN64: # %bb.0:
-; WIN64-NEXT: movq %rdx, %r8
+; WIN64-NEXT: pushq %rsi
; WIN64-NEXT: movq %rdx, %r9
-; WIN64-NEXT: shrq $36, %r9
+; WIN64-NEXT: movq %r8, %r10
+; WIN64-NEXT: shrq $36, %r10
; WIN64-NEXT: movabsq $103054436165975148, %rdx # imm = 0x16E1F76B4337C6C
-; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: movq %r10, %rax
; WIN64-NEXT: mulq %rdx
-; WIN64-NEXT: movq %rax, %r10
-; WIN64-NEXT: movq %rdx, %r11
+; WIN64-NEXT: movq %rax, %r11
+; WIN64-NEXT: movq %rdx, %rsi
; WIN64-NEXT: movabsq $-5667993989128633178, %rdx # imm = 0xB1573D7F48F044A6
-; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: movq %r10, %rax
; WIN64-NEXT: mulq %rdx
-; WIN64-NEXT: addq %r10, %rdx
-; WIN64-NEXT: adcq $0, %r11
+; WIN64-NEXT: addq %r11, %rdx
+; WIN64-NEXT: adcq $0, %rsi
; WIN64-NEXT: movabsq $12300786335744, %rax # imm = 0xB3000000000
-; WIN64-NEXT: imulq %r11, %rax
+; WIN64-NEXT: imulq %rsi, %rax
; WIN64-NEXT: subq %rax, %r8
+; WIN64-NEXT: movq %r9, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: movq %r8, %rdx
+; WIN64-NEXT: popq %rsi
; WIN64-NEXT: retq
%ret = urem i128 %x, 226909457440853062867909873762304 ; = 179 * 2^100
ret i128 %ret
@@ -1449,21 +1523,24 @@ define i128 @udiv_magic_large_postshift(i128 %x) nounwind {
;
; WIN64-LABEL: udiv_magic_large_postshift:
; WIN64: # %bb.0:
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq $-1, %r11
-; WIN64-NEXT: movq %rdx, %rax
-; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: pushq %rsi
; WIN64-NEXT: movq %rdx, %r9
-; WIN64-NEXT: movq %rax, %r10
-; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: movq $-1, %rsi
+; WIN64-NEXT: movq %r8, %rax
+; WIN64-NEXT: mulq %rsi
+; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: movq %rax, %r11
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %rsi
; WIN64-NEXT: addq %r8, %rax
; WIN64-NEXT: adcq $0, %rdx
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: addq %r10, %rdx
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: movq %r9, %rax
-; WIN64-NEXT: xorl %edx, %edx
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: addq %r11, %rdx
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: movq %r10, (%rcx)
+; WIN64-NEXT: movq $0, 8(%rcx)
+; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: popq %rsi
; WIN64-NEXT: retq
%ret = udiv i128 %x, 18446744073709551617 ; = 2^64 + 1
ret i128 %ret
@@ -1493,23 +1570,26 @@ define i128 @urem_magic_large_postshift(i128 %x) nounwind {
;
; WIN64-LABEL: urem_magic_large_postshift:
; WIN64: # %bb.0:
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq $-1, %r11
-; WIN64-NEXT: movq %rdx, %rax
-; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: pushq %rsi
; WIN64-NEXT: movq %rdx, %r9
-; WIN64-NEXT: movq %rax, %r10
-; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %r11
+; WIN64-NEXT: movq $-1, %rsi
+; WIN64-NEXT: movq %r8, %rax
+; WIN64-NEXT: mulq %rsi
+; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: movq %rax, %r11
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %rsi
; WIN64-NEXT: addq %r8, %rax
; WIN64-NEXT: adcq $0, %rdx
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: addq %r10, %rdx
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: subq %r9, %rcx
-; WIN64-NEXT: sbbq %r9, %r8
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: addq %r11, %rdx
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: subq %r10, %r9
+; WIN64-NEXT: sbbq %r10, %r8
+; WIN64-NEXT: movq %r9, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: movq %r8, %rdx
+; WIN64-NEXT: popq %rsi
; WIN64-NEXT: retq
%ret = urem i128 %x, 18446744073709551617 ; = 2^64 + 1
ret i128 %ret
@@ -1556,38 +1636,41 @@ define i128 @udiv_magic_is_add(i128 %x) nounwind {
; WIN64: # %bb.0:
; WIN64-NEXT: pushq %rsi
; WIN64-NEXT: pushq %rdi
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq $-1, %rsi
-; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %rsi
-; WIN64-NEXT: movq %rax, %r9
-; WIN64-NEXT: movq %rdx, %r10
-; WIN64-NEXT: movq $-3, %rdi
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: pushq %rbx
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq $-1, %rdi
+; WIN64-NEXT: movq %rdx, %rax
; WIN64-NEXT: mulq %rdi
+; WIN64-NEXT: movq %rax, %r10
; WIN64-NEXT: movq %rdx, %r11
-; WIN64-NEXT: addq %r9, %r11
-; WIN64-NEXT: adcq $0, %r10
-; WIN64-NEXT: movq %r8, %rax
-; WIN64-NEXT: mulq %rsi
-; WIN64-NEXT: movq %rdx, %r9
-; WIN64-NEXT: movq %rax, %rsi
+; WIN64-NEXT: movq $-3, %rbx
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %rbx
+; WIN64-NEXT: movq %rdx, %rsi
+; WIN64-NEXT: addq %r10, %rsi
+; WIN64-NEXT: adcq $0, %r11
; WIN64-NEXT: movq %r8, %rax
; WIN64-NEXT: mulq %rdi
-; WIN64-NEXT: addq %r11, %rax
-; WIN64-NEXT: adcq %r10, %rdx
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: addq %rsi, %rdx
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: subq %rdx, %rcx
-; WIN64-NEXT: sbbq %r9, %r8
-; WIN64-NEXT: shrdq $1, %r8, %rcx
+; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: movq %rax, %rdi
+; WIN64-NEXT: movq %r8, %rax
+; WIN64-NEXT: mulq %rbx
+; WIN64-NEXT: addq %rsi, %rax
+; WIN64-NEXT: adcq %r11, %rdx
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: addq %rdi, %rdx
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: subq %rdx, %r9
+; WIN64-NEXT: sbbq %r10, %r8
+; WIN64-NEXT: shrdq $1, %r8, %r9
; WIN64-NEXT: shrq %r8
-; WIN64-NEXT: addq %rdx, %rcx
-; WIN64-NEXT: adcq %r9, %r8
+; WIN64-NEXT: addq %rdx, %r9
+; WIN64-NEXT: adcq %r10, %r8
; WIN64-NEXT: shrq $63, %r8
-; WIN64-NEXT: movq %r8, %rax
-; WIN64-NEXT: xorl %edx, %edx
+; WIN64-NEXT: movq %r8, (%rcx)
+; WIN64-NEXT: movq $0, 8(%rcx)
+; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: popq %rbx
; WIN64-NEXT: popq %rdi
; WIN64-NEXT: popq %rsi
; WIN64-NEXT: retq
@@ -1642,44 +1725,47 @@ define i128 @urem_magic_is_add(i128 %x) nounwind {
; WIN64: # %bb.0:
; WIN64-NEXT: pushq %rsi
; WIN64-NEXT: pushq %rdi
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq $-1, %rsi
-; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: mulq %rsi
-; WIN64-NEXT: movq %rax, %r9
-; WIN64-NEXT: movq %rdx, %r10
-; WIN64-NEXT: movq $-3, %rdi
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: pushq %rbx
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: movq $-1, %rdi
+; WIN64-NEXT: movq %rdx, %rax
; WIN64-NEXT: mulq %rdi
+; WIN64-NEXT: movq %rax, %r10
; WIN64-NEXT: movq %rdx, %r11
-; WIN64-NEXT: addq %r9, %r11
-; WIN64-NEXT: adcq $0, %r10
-; WIN64-NEXT: movq %r8, %rax
-; WIN64-NEXT: mulq %rsi
-; WIN64-NEXT: movq %rdx, %r9
-; WIN64-NEXT: movq %rax, %rsi
+; WIN64-NEXT: movq $-3, %rbx
+; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: mulq %rbx
+; WIN64-NEXT: movq %rdx, %rsi
+; WIN64-NEXT: addq %r10, %rsi
+; WIN64-NEXT: adcq $0, %r11
; WIN64-NEXT: movq %r8, %rax
; WIN64-NEXT: mulq %rdi
-; WIN64-NEXT: addq %r11, %rax
-; WIN64-NEXT: adcq %r10, %rdx
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: addq %rsi, %rdx
-; WIN64-NEXT: adcq $0, %r9
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: movq %rax, %rdi
+; WIN64-NEXT: movq %r8, %rax
+; WIN64-NEXT: mulq %rbx
+; WIN64-NEXT: addq %rsi, %rax
+; WIN64-NEXT: adcq %r11, %rdx
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: addq %rdi, %rdx
+; WIN64-NEXT: adcq $0, %r10
+; WIN64-NEXT: movq %r9, %rax
; WIN64-NEXT: subq %rdx, %rax
-; WIN64-NEXT: movq %r8, %r10
-; WIN64-NEXT: sbbq %r9, %r10
-; WIN64-NEXT: shrdq $1, %r10, %rax
-; WIN64-NEXT: shrq %r10
+; WIN64-NEXT: movq %r8, %r11
+; WIN64-NEXT: sbbq %r10, %r11
+; WIN64-NEXT: shrdq $1, %r11, %rax
+; WIN64-NEXT: shrq %r11
; WIN64-NEXT: addq %rdx, %rax
-; WIN64-NEXT: adcq %r9, %r10
+; WIN64-NEXT: adcq %r10, %r11
; WIN64-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
-; WIN64-NEXT: andq %r10, %rax
-; WIN64-NEXT: shrq $63, %r10
-; WIN64-NEXT: subq %r10, %rcx
+; WIN64-NEXT: andq %r11, %rax
+; WIN64-NEXT: shrq $63, %r11
+; WIN64-NEXT: subq %r11, %r9
; WIN64-NEXT: sbbq %rax, %r8
+; WIN64-NEXT: movq %r9, (%rcx)
+; WIN64-NEXT: movq %r8, 8(%rcx)
; WIN64-NEXT: movq %rcx, %rax
-; WIN64-NEXT: movq %r8, %rdx
+; WIN64-NEXT: popq %rbx
; WIN64-NEXT: popq %rdi
; WIN64-NEXT: popq %rsi
; WIN64-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/fp128-cast.ll b/llvm/test/CodeGen/X86/fp128-cast.ll
index 36ec50efa70b38..bbbd72a0ed476c 100644
--- a/llvm/test/CodeGen/X86/fp128-cast.ll
+++ b/llvm/test/CodeGen/X86/fp128-cast.ll
@@ -518,13 +518,15 @@ define dso_local void @TestFPToSIF128_I128() nounwind {
;
; WIN64-SSE-LABEL: TestFPToSIF128_I128:
; WIN64-SSE: # %bb.0: # %entry
-; WIN64-SSE-NEXT: subq $56, %rsp
+; WIN64-SSE-NEXT: subq $72, %rsp
; WIN64-SSE-NEXT: movaps vf128(%rip), %xmm0
; WIN64-SSE-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
; WIN64-SSE-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-SSE-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; WIN64-SSE-NEXT: callq __fixtfti
+; WIN64-SSE-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
; WIN64-SSE-NEXT: movaps %xmm0, vi128(%rip)
-; WIN64-SSE-NEXT: addq $56, %rsp
+; WIN64-SSE-NEXT: addq $72, %rsp
; WIN64-SSE-NEXT: retq
;
; X86-LABEL: TestFPToSIF128_I128:
@@ -584,13 +586,15 @@ define dso_local void @TestFPToUIF128_U128() nounwind {
;
; WIN64-SSE-LABEL: TestFPToUIF128_U128:
; WIN64-SSE: # %bb.0: # %entry
-; WIN64-SSE-NEXT: subq $56, %rsp
+; WIN64-SSE-NEXT: subq $72, %rsp
; WIN64-SSE-NEXT: movaps vf128(%rip), %xmm0
; WIN64-SSE-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
; WIN64-SSE-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-SSE-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; WIN64-SSE-NEXT: callq __fixunstfti
+; WIN64-SSE-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
; WIN64-SSE-NEXT: movaps %xmm0, vu128(%rip)
-; WIN64-SSE-NEXT: addq $56, %rsp
+; WIN64-SSE-NEXT: addq $72, %rsp
; WIN64-SSE-NEXT: retq
;
; X86-LABEL: TestFPToUIF128_U128:
diff --git a/llvm/test/CodeGen/X86/fp128-return-calling-conv.ll b/llvm/test/CodeGen/X86/fp128-return-calling-conv.ll
index a7949479506628..a4930602bfe14d 100644
--- a/llvm/test/CodeGen/X86/fp128-return-calling-conv.ll
+++ b/llvm/test/CodeGen/X86/fp128-return-calling-conv.ll
@@ -1,15 +1,52 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-pc-windows-gnu | FileCheck %s --check-prefixes=WIN
; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc | FileCheck %s --check-prefixes=WIN
+; RUN: llc < %s -mtriple=x86_64-unknown-uefi | FileCheck %s --check-prefixes=WIN
; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefix=LINUX
; On Windows, for calling conventions that have GCC compatibility requirements,
-; follow the GCC convention of returning f128 via the stack (an sret).
-; Elsewhere, and for other calling conventions, return it in XMM0.
+; follow the GCC convention of returning f128 and i128 via the stack (an sret).
+; Elsewhere, and for other calling conventions, f128 is returned in XMM0.
;
; NOTE: f128 arguments are passed indirectly on Windows, but in XMM registers
; elsewhere.
+define i128 @ret_i128_ccc() {
+; WIN-LABEL: ret_i128_ccc:
+; WIN: # %bb.0:
+; WIN-NEXT: movq %rcx, %rax
+; WIN-NEXT: movq $0, 8(%rcx)
+; WIN-NEXT: movq $42, (%rcx)
+; WIN-NEXT: retq
+;
+; LINUX-LABEL: ret_i128_ccc:
+; LINUX: # %bb.0:
+; LINUX-NEXT: movl $42, %eax
+; LINUX-NEXT: xorl %edx, %edx
+; LINUX-NEXT: retq
+ ret i128 42
+}
+
+define { i128, i1 } @ret_i128_aggregate(ptr %p) {
+; WIN-LABEL: ret_i128_aggregate:
+; WIN: # %bb.0:
+; WIN-NEXT: movq %rcx, %rax
+; WIN-NEXT: movaps (%rdx), %xmm0
+; WIN-NEXT: movzbl 16(%rdx), %ecx
+; WIN-NEXT: movb %cl, 16(%rax)
+; WIN-NEXT: movaps %xmm0, (%rax)
+; WIN-NEXT: retq
+;
+; LINUX-LABEL: ret_i128_aggregate:
+; LINUX: # %bb.0:
+; LINUX-NEXT: movq (%rdi), %rax
+; LINUX-NEXT: movq 8(%rdi), %rdx
+; LINUX-NEXT: movzbl 16(%rdi), %ecx
+; LINUX-NEXT: retq
+ %v = load { i128, i1 }, ptr %p
+ ret { i128, i1 } %v
+}
+
define fp128 @ret_ccc(ptr %p) {
; WIN-LABEL: ret_ccc:
; WIN: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/fp80-conv-libcalls.ll b/llvm/test/CodeGen/X86/fp80-conv-libcalls.ll
index 9a41d25feb0307..6d7defa167d37c 100644
--- a/llvm/test/CodeGen/X86/fp80-conv-libcalls.ll
+++ b/llvm/test/CodeGen/X86/fp80-conv-libcalls.ll
@@ -92,15 +92,19 @@ define i128 @test_fptosi(x86_fp80 %x) nounwind {
;
; MSVC-LABEL: test_fptosi:
; MSVC: # %bb.0:
-; MSVC-NEXT: subq $56, %rsp
-; MSVC-NEXT: fldt (%rcx)
+; MSVC-NEXT: pushq %rsi
+; MSVC-NEXT: subq $64, %rsp
+; MSVC-NEXT: movq %rcx, %rsi
+; MSVC-NEXT: fldt (%rdx)
; MSVC-NEXT: fstpt {{[0-9]+}}(%rsp)
; MSVC-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; MSVC-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; MSVC-NEXT: callq __fixxfti
-; MSVC-NEXT: movq %xmm0, %rax
-; MSVC-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; MSVC-NEXT: movq %xmm0, %rdx
-; MSVC-NEXT: addq $56, %rsp
+; MSVC-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
+; MSVC-NEXT: movaps %xmm0, (%rsi)
+; MSVC-NEXT: movq %rsi, %rax
+; MSVC-NEXT: addq $64, %rsp
+; MSVC-NEXT: popq %rsi
; MSVC-NEXT: retq
%r = fptosi x86_fp80 %x to i128
ret i128 %r
diff --git a/llvm/test/CodeGen/X86/fptosi-f16-wide.ll b/llvm/test/CodeGen/X86/fptosi-f16-wide.ll
index 49131fb1ada9bd..d41c8c2c88afb9 100644
--- a/llvm/test/CodeGen/X86/fptosi-f16-wide.ll
+++ b/llvm/test/CodeGen/X86/fptosi-f16-wide.ll
@@ -34,19 +34,29 @@ define i128 @fptosi_f16_to_i128(half %x) nounwind {
;
; WIN-SSE2-LABEL: fptosi_f16_to_i128:
; WIN-SSE2: # %bb.0:
-; WIN-SSE2-NEXT: subq $40, %rsp
+; WIN-SSE2-NEXT: pushq %rsi
+; WIN-SSE2-NEXT: subq $32, %rsp
+; WIN-SSE2-NEXT: movaps %xmm1, %xmm0
+; WIN-SSE2-NEXT: movq %rcx, %rsi
; WIN-SSE2-NEXT: callq __extendhfsf2
; WIN-SSE2-NEXT: cvttss2si %xmm0, %rax
-; WIN-SSE2-NEXT: movq %rax, %rdx
-; WIN-SSE2-NEXT: sarq $63, %rdx
-; WIN-SSE2-NEXT: addq $40, %rsp
+; WIN-SSE2-NEXT: movq %rax, %rcx
+; WIN-SSE2-NEXT: sarq $63, %rcx
+; WIN-SSE2-NEXT: movq %rcx, 8(%rsi)
+; WIN-SSE2-NEXT: movq %rax, (%rsi)
+; WIN-SSE2-NEXT: movq %rsi, %rax
+; WIN-SSE2-NEXT: addq $32, %rsp
+; WIN-SSE2-NEXT: popq %rsi
; WIN-SSE2-NEXT: retq
;
; WIN-FP16-LABEL: fptosi_f16_to_i128:
; WIN-FP16: # %bb.0:
-; WIN-FP16-NEXT: vcvttsh2si %xmm0, %rax
-; WIN-FP16-NEXT: movq %rax, %rdx
+; WIN-FP16-NEXT: movq %rcx, %rax
+; WIN-FP16-NEXT: vcvttsh2si %xmm1, %rcx
+; WIN-FP16-NEXT: movq %rcx, %rdx
; WIN-FP16-NEXT: sarq $63, %rdx
+; WIN-FP16-NEXT: movq %rdx, 8(%rax)
+; WIN-FP16-NEXT: movq %rcx, (%rax)
; WIN-FP16-NEXT: retq
%r = fptosi half %x to i128
ret i128 %r
@@ -83,19 +93,29 @@ define i128 @fptosi_f16_to_i128_strict(half %x) nounwind strictfp {
;
; WIN-SSE2-LABEL: fptosi_f16_to_i128_strict:
; WIN-SSE2: # %bb.0:
-; WIN-SSE2-NEXT: subq $40, %rsp
+; WIN-SSE2-NEXT: pushq %rsi
+; WIN-SSE2-NEXT: subq $32, %rsp
+; WIN-SSE2-NEXT: movaps %xmm1, %xmm0
+; WIN-SSE2-NEXT: movq %rcx, %rsi
; WIN-SSE2-NEXT: callq __extendhfsf2
; WIN-SSE2-NEXT: cvttss2si %xmm0, %rax
-; WIN-SSE2-NEXT: movq %rax, %rdx
-; WIN-SSE2-NEXT: sarq $63, %rdx
-; WIN-SSE2-NEXT: addq $40, %rsp
+; WIN-SSE2-NEXT: movq %rax, %rcx
+; WIN-SSE2-NEXT: sarq $63, %rcx
+; WIN-SSE2-NEXT: movq %rcx, 8(%rsi)
+; WIN-SSE2-NEXT: movq %rax, (%rsi)
+; WIN-SSE2-NEXT: movq %rsi, %rax
+; WIN-SSE2-NEXT: addq $32, %rsp
+; WIN-SSE2-NEXT: popq %rsi
; WIN-SSE2-NEXT: retq
;
; WIN-FP16-LABEL: fptosi_f16_to_i128_strict:
; WIN-FP16: # %bb.0:
-; WIN-FP16-NEXT: vcvttsh2si %xmm0, %rax
-; WIN-FP16-NEXT: movq %rax, %rdx
+; WIN-FP16-NEXT: movq %rcx, %rax
+; WIN-FP16-NEXT: vcvttsh2si %xmm1, %rcx
+; WIN-FP16-NEXT: movq %rcx, %rdx
; WIN-FP16-NEXT: sarq $63, %rdx
+; WIN-FP16-NEXT: movq %rdx, 8(%rax)
+; WIN-FP16-NEXT: movq %rcx, (%rax)
; WIN-FP16-NEXT: retq
%r = call i128 @llvm.experimental.constrained.fptosi.i128.f16(half %x, metadata !"fpexcept.strict") strictfp
ret i128 %r
diff --git a/llvm/test/CodeGen/X86/fptoui-f16-wide.ll b/llvm/test/CodeGen/X86/fptoui-f16-wide.ll
index e2edd6232d9dec..0d7f84af5cce70 100644
--- a/llvm/test/CodeGen/X86/fptoui-f16-wide.ll
+++ b/llvm/test/CodeGen/X86/fptoui-f16-wide.ll
@@ -31,17 +31,25 @@ define i128 @fptoui_f16_to_i128(half %x) nounwind {
;
; WIN-SSE2-LABEL: fptoui_f16_to_i128:
; WIN-SSE2: # %bb.0:
-; WIN-SSE2-NEXT: subq $40, %rsp
+; WIN-SSE2-NEXT: pushq %rsi
+; WIN-SSE2-NEXT: subq $32, %rsp
+; WIN-SSE2-NEXT: movaps %xmm1, %xmm0
+; WIN-SSE2-NEXT: movq %rcx, %rsi
; WIN-SSE2-NEXT: callq __extendhfsf2
; WIN-SSE2-NEXT: cvttss2si %xmm0, %rax
-; WIN-SSE2-NEXT: xorl %edx, %edx
-; WIN-SSE2-NEXT: addq $40, %rsp
+; WIN-SSE2-NEXT: movq %rax, (%rsi)
+; WIN-SSE2-NEXT: movq $0, 8(%rsi)
+; WIN-SSE2-NEXT: movq %rsi, %rax
+; WIN-SSE2-NEXT: addq $32, %rsp
+; WIN-SSE2-NEXT: popq %rsi
; WIN-SSE2-NEXT: retq
;
; WIN-FP16-LABEL: fptoui_f16_to_i128:
; WIN-FP16: # %bb.0:
-; WIN-FP16-NEXT: vcvttsh2si %xmm0, %rax
-; WIN-FP16-NEXT: xorl %edx, %edx
+; WIN-FP16-NEXT: movq %rcx, %rax
+; WIN-FP16-NEXT: vcvttsh2si %xmm1, %rcx
+; WIN-FP16-NEXT: movq %rcx, (%rax)
+; WIN-FP16-NEXT: movq $0, 8(%rax)
; WIN-FP16-NEXT: retq
%r = fptoui half %x to i128
ret i128 %r
@@ -75,17 +83,25 @@ define i128 @fptoui_f16_to_i128_strict(half %x) nounwind strictfp {
;
; WIN-SSE2-LABEL: fptoui_f16_to_i128_strict:
; WIN-SSE2: # %bb.0:
-; WIN-SSE2-NEXT: subq $40, %rsp
+; WIN-SSE2-NEXT: pushq %rsi
+; WIN-SSE2-NEXT: subq $32, %rsp
+; WIN-SSE2-NEXT: movaps %xmm1, %xmm0
+; WIN-SSE2-NEXT: movq %rcx, %rsi
; WIN-SSE2-NEXT: callq __extendhfsf2
; WIN-SSE2-NEXT: cvttss2si %xmm0, %rax
-; WIN-SSE2-NEXT: xorl %edx, %edx
-; WIN-SSE2-NEXT: addq $40, %rsp
+; WIN-SSE2-NEXT: movq %rax, (%rsi)
+; WIN-SSE2-NEXT: movq $0, 8(%rsi)
+; WIN-SSE2-NEXT: movq %rsi, %rax
+; WIN-SSE2-NEXT: addq $32, %rsp
+; WIN-SSE2-NEXT: popq %rsi
; WIN-SSE2-NEXT: retq
;
; WIN-FP16-LABEL: fptoui_f16_to_i128_strict:
; WIN-FP16: # %bb.0:
-; WIN-FP16-NEXT: vcvttsh2si %xmm0, %rax
-; WIN-FP16-NEXT: xorl %edx, %edx
+; WIN-FP16-NEXT: movq %rcx, %rax
+; WIN-FP16-NEXT: vcvttsh2si %xmm1, %rcx
+; WIN-FP16-NEXT: movq %rcx, (%rax)
+; WIN-FP16-NEXT: movq $0, 8(%rax)
; WIN-FP16-NEXT: retq
%r = call i128 @llvm.experimental.constrained.fptoui.i128.f16(half %x, metadata !"fpexcept.strict") strictfp
ret i128 %r
diff --git a/llvm/test/CodeGen/X86/i128-fp128-abi.ll b/llvm/test/CodeGen/X86/i128-fp128-abi.ll
index 77881225408dd1..77bdf3bc7ae4f4 100644
--- a/llvm/test/CodeGen/X86/i128-fp128-abi.ll
+++ b/llvm/test/CodeGen/X86/i128-fp128-abi.ll
@@ -197,8 +197,9 @@ define PrimTy @return(ptr %p) nounwind {
;
; CHECK-MSVC64-I128-LABEL: return:
; CHECK-MSVC64-I128: # %bb.0:
-; CHECK-MSVC64-I128-NEXT: movq (%rcx), %rax
-; CHECK-MSVC64-I128-NEXT: movq 8(%rcx), %rdx
+; CHECK-MSVC64-I128-NEXT: movq %rcx, %rax
+; CHECK-MSVC64-I128-NEXT: movaps (%rdx), %xmm0
+; CHECK-MSVC64-I128-NEXT: movaps %xmm0, (%rcx)
; CHECK-MSVC64-I128-NEXT: retq
;
; CHECK-MINGW-F128-LABEL: return:
@@ -210,8 +211,9 @@ define PrimTy @return(ptr %p) nounwind {
;
; CHECK-MINGW-I128-LABEL: return:
; CHECK-MINGW-I128: # %bb.0:
-; CHECK-MINGW-I128-NEXT: movq (%rcx), %rax
-; CHECK-MINGW-I128-NEXT: movq 8(%rcx), %rdx
+; CHECK-MINGW-I128-NEXT: movq %rcx, %rax
+; CHECK-MINGW-I128-NEXT: movaps (%rdx), %xmm0
+; CHECK-MINGW-I128-NEXT: movaps %xmm0, (%rcx)
; CHECK-MINGW-I128-NEXT: retq
;
; CHECK-X86-LABEL: return:
@@ -274,6 +276,8 @@ define PrimTy @first_arg(PrimTy %x) nounwind {
; CHECK-MSVC64-I128-LABEL: first_arg:
; CHECK-MSVC64-I128: # %bb.0:
; CHECK-MSVC64-I128-NEXT: movq %rcx, %rax
+; CHECK-MSVC64-I128-NEXT: movq %r8, 8(%rcx)
+; CHECK-MSVC64-I128-NEXT: movq %rdx, (%rcx)
; CHECK-MSVC64-I128-NEXT: retq
;
; CHECK-MINGW-F128-LABEL: first_arg:
@@ -286,6 +290,8 @@ define PrimTy @first_arg(PrimTy %x) nounwind {
; CHECK-MINGW-I128-LABEL: first_arg:
; CHECK-MINGW-I128: # %bb.0:
; CHECK-MINGW-I128-NEXT: movq %rcx, %rax
+; CHECK-MINGW-I128-NEXT: movq %r8, 8(%rcx)
+; CHECK-MINGW-I128-NEXT: movq %rdx, (%rcx)
; CHECK-MINGW-I128-NEXT: retq
;
; CHECK-X86-LABEL: first_arg:
@@ -354,8 +360,11 @@ define PrimTy @leading_args(i64 %_0, i64 %_1, i64 %_2, i64 %_3, PrimTy %x) nounw
;
; CHECK-MSVC64-I128-LABEL: leading_args:
; CHECK-MSVC64-I128: # %bb.0:
-; CHECK-MSVC64-I128-NEXT: movq 40(%rsp), %rax
-; CHECK-MSVC64-I128-NEXT: movq 48(%rsp), %rdx
+; CHECK-MSVC64-I128-NEXT: movq %rcx, %rax
+; CHECK-MSVC64-I128-NEXT: movq 48(%rsp), %rcx
+; CHECK-MSVC64-I128-NEXT: movq 56(%rsp), %rdx
+; CHECK-MSVC64-I128-NEXT: movq %rdx, 8(%rax)
+; CHECK-MSVC64-I128-NEXT: movq %rcx, (%rax)
; CHECK-MSVC64-I128-NEXT: retq
;
; CHECK-MINGW-F128-LABEL: leading_args:
@@ -368,8 +377,11 @@ define PrimTy @leading_args(i64 %_0, i64 %_1, i64 %_2, i64 %_3, PrimTy %x) nounw
;
; CHECK-MINGW-I128-LABEL: leading_args:
; CHECK-MINGW-I128: # %bb.0:
-; CHECK-MINGW-I128-NEXT: movq 40(%rsp), %rax
-; CHECK-MINGW-I128-NEXT: movq 48(%rsp), %rdx
+; CHECK-MINGW-I128-NEXT: movq %rcx, %rax
+; CHECK-MINGW-I128-NEXT: movq 48(%rsp), %rcx
+; CHECK-MINGW-I128-NEXT: movq 56(%rsp), %rdx
+; CHECK-MINGW-I128-NEXT: movq %rdx, 8(%rax)
+; CHECK-MINGW-I128-NEXT: movq %rcx, (%rax)
; CHECK-MINGW-I128-NEXT: retq
;
; CHECK-X86-LABEL: leading_args:
@@ -438,8 +450,9 @@ define PrimTy @many_leading_args(i64 %_0, i64 %_1, i64 %_2, i64 %_3, i64 %_4, Pr
;
; CHECK-MSVC64-I128-LABEL: many_leading_args:
; CHECK-MSVC64-I128: # %bb.0:
-; CHECK-MSVC64-I128-NEXT: movq 64(%rsp), %rax
-; CHECK-MSVC64-I128-NEXT: movq 72(%rsp), %rdx
+; CHECK-MSVC64-I128-NEXT: movq %rcx, %rax
+; CHECK-MSVC64-I128-NEXT: movaps 72(%rsp), %xmm0
+; CHECK-MSVC64-I128-NEXT: movaps %xmm0, (%rcx)
; CHECK-MSVC64-I128-NEXT: retq
;
; CHECK-MINGW-F128-LABEL: many_leading_args:
@@ -452,8 +465,9 @@ define PrimTy @many_leading_args(i64 %_0, i64 %_1, i64 %_2, i64 %_3, i64 %_4, Pr
;
; CHECK-MINGW-I128-LABEL: many_leading_args:
; CHECK-MINGW-I128: # %bb.0:
-; CHECK-MINGW-I128-NEXT: movq 64(%rsp), %rax
-; CHECK-MINGW-I128-NEXT: movq 72(%rsp), %rdx
+; CHECK-MINGW-I128-NEXT: movq %rcx, %rax
+; CHECK-MINGW-I128-NEXT: movaps 72(%rsp), %xmm0
+; CHECK-MINGW-I128-NEXT: movaps %xmm0, (%rcx)
; CHECK-MINGW-I128-NEXT: retq
;
; CHECK-X86-LABEL: many_leading_args:
@@ -520,8 +534,9 @@ define PrimTy @trailing_arg(i64 %_0, i64 %_1, i64 %_2, i64 %_3, i64 %_4, PrimTy
;
; CHECK-MSVC64-I128-LABEL: trailing_arg:
; CHECK-MSVC64-I128: # %bb.0:
-; CHECK-MSVC64-I128-NEXT: movq 48(%rsp), %rax
-; CHECK-MSVC64-I128-NEXT: movq 56(%rsp), %rdx
+; CHECK-MSVC64-I128-NEXT: movq %rcx, %rax
+; CHECK-MSVC64-I128-NEXT: movaps 56(%rsp), %xmm0
+; CHECK-MSVC64-I128-NEXT: movaps %xmm0, (%rcx)
; CHECK-MSVC64-I128-NEXT: retq
;
; CHECK-MINGW-F128-LABEL: trailing_arg:
@@ -534,8 +549,9 @@ define PrimTy @trailing_arg(i64 %_0, i64 %_1, i64 %_2, i64 %_3, i64 %_4, PrimTy
;
; CHECK-MINGW-I128-LABEL: trailing_arg:
; CHECK-MINGW-I128: # %bb.0:
-; CHECK-MINGW-I128-NEXT: movq 48(%rsp), %rax
-; CHECK-MINGW-I128-NEXT: movq 56(%rsp), %rdx
+; CHECK-MINGW-I128-NEXT: movq %rcx, %rax
+; CHECK-MINGW-I128-NEXT: movaps 56(%rsp), %xmm0
+; CHECK-MINGW-I128-NEXT: movaps %xmm0, (%rcx)
; CHECK-MINGW-I128-NEXT: retq
;
; CHECK-X86-LABEL: trailing_arg:
@@ -609,9 +625,12 @@ define void @call_first_arg(PrimTy %x) nounwind {
;
; CHECK-MSVC64-I128-LABEL: call_first_arg:
; CHECK-MSVC64-I128: # %bb.0:
-; CHECK-MSVC64-I128-NEXT: subq $40, %rsp
+; CHECK-MSVC64-I128-NEXT: subq $56, %rsp
+; CHECK-MSVC64-I128-NEXT: movq %rdx, %r8
+; CHECK-MSVC64-I128-NEXT: movq %rcx, %rdx
+; CHECK-MSVC64-I128-NEXT: leaq 32(%rsp), %rcx
; CHECK-MSVC64-I128-NEXT: callq first_arg
-; CHECK-MSVC64-I128-NEXT: addq $40, %rsp
+; CHECK-MSVC64-I128-NEXT: addq $56, %rsp
; CHECK-MSVC64-I128-NEXT: retq
;
; CHECK-MINGW-F128-LABEL: call_first_arg:
@@ -627,9 +646,12 @@ define void @call_first_arg(PrimTy %x) nounwind {
;
; CHECK-MINGW-I128-LABEL: call_first_arg:
; CHECK-MINGW-I128: # %bb.0:
-; CHECK-MINGW-I128-NEXT: subq $40, %rsp
+; CHECK-MINGW-I128-NEXT: subq $56, %rsp
+; CHECK-MINGW-I128-NEXT: movq %rdx, %r8
+; CHECK-MINGW-I128-NEXT: movq %rcx, %rdx
+; CHECK-MINGW-I128-NEXT: leaq 32(%rsp), %rcx
; CHECK-MINGW-I128-NEXT: callq first_arg
-; CHECK-MINGW-I128-NEXT: addq $40, %rsp
+; CHECK-MINGW-I128-NEXT: addq $56, %rsp
; CHECK-MINGW-I128-NEXT: retq
;
; CHECK-X86-LABEL: call_first_arg:
@@ -720,15 +742,16 @@ define void @call_leading_args(PrimTy %x) nounwind {
;
; CHECK-MSVC64-I128-LABEL: call_leading_args:
; CHECK-MSVC64-I128: # %bb.0:
-; CHECK-MSVC64-I128-NEXT: subq $56, %rsp
-; CHECK-MSVC64-I128-NEXT: movq %rdx, 40(%rsp)
-; CHECK-MSVC64-I128-NEXT: movq %rcx, 32(%rsp)
-; CHECK-MSVC64-I128-NEXT: xorl %ecx, %ecx
+; CHECK-MSVC64-I128-NEXT: subq $88, %rsp
+; CHECK-MSVC64-I128-NEXT: movq %rdx, 48(%rsp)
+; CHECK-MSVC64-I128-NEXT: movq %rcx, 40(%rsp)
+; CHECK-MSVC64-I128-NEXT: movq $0, 32(%rsp)
+; CHECK-MSVC64-I128-NEXT: leaq 64(%rsp), %rcx
; CHECK-MSVC64-I128-NEXT: xorl %edx, %edx
; CHECK-MSVC64-I128-NEXT: xorl %r8d, %r8d
; CHECK-MSVC64-I128-NEXT: xorl %r9d, %r9d
; CHECK-MSVC64-I128-NEXT: callq leading_args
-; CHECK-MSVC64-I128-NEXT: addq $56, %rsp
+; CHECK-MSVC64-I128-NEXT: addq $88, %rsp
; CHECK-MSVC64-I128-NEXT: retq
;
; CHECK-MINGW-F128-LABEL: call_leading_args:
@@ -749,15 +772,16 @@ define void @call_leading_args(PrimTy %x) nounwind {
;
; CHECK-MINGW-I128-LABEL: call_leading_args:
; CHECK-MINGW-I128: # %bb.0:
-; CHECK-MINGW-I128-NEXT: subq $56, %rsp
-; CHECK-MINGW-I128-NEXT: movq %rdx, 40(%rsp)
-; CHECK-MINGW-I128-NEXT: movq %rcx, 32(%rsp)
-; CHECK-MINGW-I128-NEXT: xorl %ecx, %ecx
+; CHECK-MINGW-I128-NEXT: subq $88, %rsp
+; CHECK-MINGW-I128-NEXT: movq %rdx, 48(%rsp)
+; CHECK-MINGW-I128-NEXT: movq %rcx, 40(%rsp)
+; CHECK-MINGW-I128-NEXT: movq $0, 32(%rsp)
+; CHECK-MINGW-I128-NEXT: leaq 64(%rsp), %rcx
; CHECK-MINGW-I128-NEXT: xorl %edx, %edx
; CHECK-MINGW-I128-NEXT: xorl %r8d, %r8d
; CHECK-MINGW-I128-NEXT: xorl %r9d, %r9d
; CHECK-MINGW-I128-NEXT: callq leading_args
-; CHECK-MINGW-I128-NEXT: addq $56, %rsp
+; CHECK-MINGW-I128-NEXT: addq $88, %rsp
; CHECK-MINGW-I128-NEXT: retq
;
; CHECK-X86-LABEL: call_leading_args:
@@ -875,17 +899,18 @@ define void @call_many_leading_args(PrimTy %x) nounwind {
;
; CHECK-MSVC64-I128-LABEL: call_many_leading_args:
; CHECK-MSVC64-I128: # %bb.0:
-; CHECK-MSVC64-I128-NEXT: subq $72, %rsp
-; CHECK-MSVC64-I128-NEXT: movq %rdx, 56(%rsp)
-; CHECK-MSVC64-I128-NEXT: movq %rcx, 48(%rsp)
+; CHECK-MSVC64-I128-NEXT: subq $104, %rsp
+; CHECK-MSVC64-I128-NEXT: movq %rdx, 64(%rsp)
+; CHECK-MSVC64-I128-NEXT: movq %rcx, 56(%rsp)
+; CHECK-MSVC64-I128-NEXT: movq $0, 48(%rsp)
; CHECK-MSVC64-I128-NEXT: movq $0, 40(%rsp)
; CHECK-MSVC64-I128-NEXT: movq $0, 32(%rsp)
-; CHECK-MSVC64-I128-NEXT: xorl %ecx, %ecx
+; CHECK-MSVC64-I128-NEXT: leaq 80(%rsp), %rcx
; CHECK-MSVC64-I128-NEXT: xorl %edx, %edx
; CHECK-MSVC64-I128-NEXT: xorl %r8d, %r8d
; CHECK-MSVC64-I128-NEXT: xorl %r9d, %r9d
; CHECK-MSVC64-I128-NEXT: callq many_leading_args
-; CHECK-MSVC64-I128-NEXT: addq $72, %rsp
+; CHECK-MSVC64-I128-NEXT: addq $104, %rsp
; CHECK-MSVC64-I128-NEXT: retq
;
; CHECK-MINGW-F128-LABEL: call_many_leading_args:
@@ -910,17 +935,18 @@ define void @call_many_leading_args(PrimTy %x) nounwind {
;
; CHECK-MINGW-I128-LABEL: call_many_leading_args:
; CHECK-MINGW-I128: # %bb.0:
-; CHECK-MINGW-I128-NEXT: subq $72, %rsp
-; CHECK-MINGW-I128-NEXT: movq %rdx, 56(%rsp)
-; CHECK-MINGW-I128-NEXT: movq %rcx, 48(%rsp)
+; CHECK-MINGW-I128-NEXT: subq $104, %rsp
+; CHECK-MINGW-I128-NEXT: movq %rdx, 64(%rsp)
+; CHECK-MINGW-I128-NEXT: movq %rcx, 56(%rsp)
+; CHECK-MINGW-I128-NEXT: movq $0, 48(%rsp)
; CHECK-MINGW-I128-NEXT: movq $0, 40(%rsp)
; CHECK-MINGW-I128-NEXT: movq $0, 32(%rsp)
-; CHECK-MINGW-I128-NEXT: xorl %ecx, %ecx
+; CHECK-MINGW-I128-NEXT: leaq 80(%rsp), %rcx
; CHECK-MINGW-I128-NEXT: xorl %edx, %edx
; CHECK-MINGW-I128-NEXT: xorl %r8d, %r8d
; CHECK-MINGW-I128-NEXT: xorl %r9d, %r9d
; CHECK-MINGW-I128-NEXT: callq many_leading_args
-; CHECK-MINGW-I128-NEXT: addq $72, %rsp
+; CHECK-MINGW-I128-NEXT: addq $104, %rsp
; CHECK-MINGW-I128-NEXT: retq
;
; CHECK-X86-LABEL: call_many_leading_args:
@@ -1035,15 +1061,16 @@ define void @call_trailing_arg(PrimTy %x) nounwind {
;
; CHECK-MSVC64-I128-LABEL: call_trailing_arg:
; CHECK-MSVC64-I128: # %bb.0:
-; CHECK-MSVC64-I128-NEXT: subq $56, %rsp
-; CHECK-MSVC64-I128-NEXT: movq %rdx, 40(%rsp)
-; CHECK-MSVC64-I128-NEXT: movq %rcx, 32(%rsp)
-; CHECK-MSVC64-I128-NEXT: xorl %ecx, %ecx
+; CHECK-MSVC64-I128-NEXT: subq $88, %rsp
+; CHECK-MSVC64-I128-NEXT: movq %rdx, 48(%rsp)
+; CHECK-MSVC64-I128-NEXT: movq %rcx, 40(%rsp)
+; CHECK-MSVC64-I128-NEXT: movq $0, 32(%rsp)
+; CHECK-MSVC64-I128-NEXT: leaq 64(%rsp), %rcx
; CHECK-MSVC64-I128-NEXT: xorl %edx, %edx
; CHECK-MSVC64-I128-NEXT: xorl %r8d, %r8d
; CHECK-MSVC64-I128-NEXT: xorl %r9d, %r9d
; CHECK-MSVC64-I128-NEXT: callq trailing_arg
-; CHECK-MSVC64-I128-NEXT: addq $56, %rsp
+; CHECK-MSVC64-I128-NEXT: addq $88, %rsp
; CHECK-MSVC64-I128-NEXT: retq
;
; CHECK-MINGW-F128-LABEL: call_trailing_arg:
@@ -1064,15 +1091,16 @@ define void @call_trailing_arg(PrimTy %x) nounwind {
;
; CHECK-MINGW-I128-LABEL: call_trailing_arg:
; CHECK-MINGW-I128: # %bb.0:
-; CHECK-MINGW-I128-NEXT: subq $56, %rsp
-; CHECK-MINGW-I128-NEXT: movq %rdx, 40(%rsp)
-; CHECK-MINGW-I128-NEXT: movq %rcx, 32(%rsp)
-; CHECK-MINGW-I128-NEXT: xorl %ecx, %ecx
+; CHECK-MINGW-I128-NEXT: subq $88, %rsp
+; CHECK-MINGW-I128-NEXT: movq %rdx, 48(%rsp)
+; CHECK-MINGW-I128-NEXT: movq %rcx, 40(%rsp)
+; CHECK-MINGW-I128-NEXT: movq $0, 32(%rsp)
+; CHECK-MINGW-I128-NEXT: leaq 64(%rsp), %rcx
; CHECK-MINGW-I128-NEXT: xorl %edx, %edx
; CHECK-MINGW-I128-NEXT: xorl %r8d, %r8d
; CHECK-MINGW-I128-NEXT: xorl %r9d, %r9d
; CHECK-MINGW-I128-NEXT: callq trailing_arg
-; CHECK-MINGW-I128-NEXT: addq $56, %rsp
+; CHECK-MINGW-I128-NEXT: addq $88, %rsp
; CHECK-MINGW-I128-NEXT: retq
;
; CHECK-X86-LABEL: call_trailing_arg:
diff --git a/llvm/test/CodeGen/X86/i128-fpconv-win64-strict.ll b/llvm/test/CodeGen/X86/i128-fpconv-win64-strict.ll
index fe506cc75cff69..9f8a7757b0c73c 100644
--- a/llvm/test/CodeGen/X86/i128-fpconv-win64-strict.ll
+++ b/llvm/test/CodeGen/X86/i128-fpconv-win64-strict.ll
@@ -6,10 +6,12 @@
define i64 @double_to_i128(double %d) nounwind strictfp {
; WIN64-LABEL: double_to_i128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $40, %rsp
+; WIN64-NEXT: subq $56, %rsp
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT: movaps %xmm0, %xmm1
; WIN64-NEXT: callq __fixdfti
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $40, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $56, %rsp
; WIN64-NEXT: retq
%1 = tail call i128 @llvm.experimental.constrained.fptosi.i128.f64(double %d, metadata !"fpexcept.strict")
%2 = trunc i128 %1 to i64
@@ -19,10 +21,12 @@ define i64 @double_to_i128(double %d) nounwind strictfp {
define i64 @double_to_ui128(double %d) nounwind strictfp {
; WIN64-LABEL: double_to_ui128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $40, %rsp
+; WIN64-NEXT: subq $56, %rsp
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT: movaps %xmm0, %xmm1
; WIN64-NEXT: callq __fixunsdfti
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $40, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $56, %rsp
; WIN64-NEXT: retq
%1 = tail call i128 @llvm.experimental.constrained.fptoui.i128.f64(double %d, metadata !"fpexcept.strict")
%2 = trunc i128 %1 to i64
@@ -32,10 +36,12 @@ define i64 @double_to_ui128(double %d) nounwind strictfp {
define i64 @float_to_i128(float %d) nounwind strictfp {
; WIN64-LABEL: float_to_i128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $40, %rsp
+; WIN64-NEXT: subq $56, %rsp
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT: movaps %xmm0, %xmm1
; WIN64-NEXT: callq __fixsfti
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $40, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $56, %rsp
; WIN64-NEXT: retq
%1 = tail call i128 @llvm.experimental.constrained.fptosi.i128.f32(float %d, metadata !"fpexcept.strict")
%2 = trunc i128 %1 to i64
@@ -45,10 +51,12 @@ define i64 @float_to_i128(float %d) nounwind strictfp {
define i64 @float_to_ui128(float %d) nounwind strictfp {
; WIN64-LABEL: float_to_ui128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $40, %rsp
+; WIN64-NEXT: subq $56, %rsp
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT: movaps %xmm0, %xmm1
; WIN64-NEXT: callq __fixunssfti
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $40, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $56, %rsp
; WIN64-NEXT: retq
%1 = tail call i128 @llvm.experimental.constrained.fptoui.i128.f32(float %d, metadata !"fpexcept.strict")
%2 = trunc i128 %1 to i64
@@ -58,14 +66,15 @@ define i64 @float_to_ui128(float %d) nounwind strictfp {
define i64 @longdouble_to_i128(ptr nocapture readonly %0) nounwind strictfp {
; WIN64-LABEL: longdouble_to_i128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $56, %rsp
+; WIN64-NEXT: subq $72, %rsp
; WIN64-NEXT: fldt (%rcx)
; WIN64-NEXT: fstpt {{[0-9]+}}(%rsp)
; WIN64-NEXT: wait
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; WIN64-NEXT: callq __fixxfti
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $56, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $72, %rsp
; WIN64-NEXT: retq
%2 = load x86_fp80, ptr %0, align 16
%3 = tail call i128 @llvm.experimental.constrained.fptosi.i128.f80(x86_fp80 %2, metadata !"fpexcept.strict")
@@ -76,14 +85,15 @@ define i64 @longdouble_to_i128(ptr nocapture readonly %0) nounwind strictfp {
define i64 @longdouble_to_ui128(ptr nocapture readonly %0) nounwind strictfp {
; WIN64-LABEL: longdouble_to_ui128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $56, %rsp
+; WIN64-NEXT: subq $72, %rsp
; WIN64-NEXT: fldt (%rcx)
; WIN64-NEXT: fstpt {{[0-9]+}}(%rsp)
; WIN64-NEXT: wait
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; WIN64-NEXT: callq __fixunsxfti
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $56, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $72, %rsp
; WIN64-NEXT: retq
%2 = load x86_fp80, ptr %0, align 16
%3 = tail call i128 @llvm.experimental.constrained.fptoui.i128.f80(x86_fp80 %2, metadata !"fpexcept.strict")
diff --git a/llvm/test/CodeGen/X86/i128-fpconv-win64.ll b/llvm/test/CodeGen/X86/i128-fpconv-win64.ll
index b036a3b0004bb7..4c7ab056a90b4e 100644
--- a/llvm/test/CodeGen/X86/i128-fpconv-win64.ll
+++ b/llvm/test/CodeGen/X86/i128-fpconv-win64.ll
@@ -6,10 +6,12 @@
define i64 @double_to_i128(double %d) nounwind {
; WIN64-LABEL: double_to_i128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $40, %rsp
+; WIN64-NEXT: subq $56, %rsp
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT: movaps %xmm0, %xmm1
; WIN64-NEXT: callq __fixdfti
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $40, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $56, %rsp
; WIN64-NEXT: retq
%1 = fptosi double %d to i128
%2 = trunc i128 %1 to i64
@@ -19,10 +21,12 @@ define i64 @double_to_i128(double %d) nounwind {
define i64 @double_to_ui128(double %d) nounwind {
; WIN64-LABEL: double_to_ui128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $40, %rsp
+; WIN64-NEXT: subq $56, %rsp
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT: movaps %xmm0, %xmm1
; WIN64-NEXT: callq __fixunsdfti
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $40, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $56, %rsp
; WIN64-NEXT: retq
%1 = fptoui double %d to i128
%2 = trunc i128 %1 to i64
@@ -32,10 +36,12 @@ define i64 @double_to_ui128(double %d) nounwind {
define i64 @float_to_i128(float %d) nounwind {
; WIN64-LABEL: float_to_i128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $40, %rsp
+; WIN64-NEXT: subq $56, %rsp
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT: movaps %xmm0, %xmm1
; WIN64-NEXT: callq __fixsfti
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $40, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $56, %rsp
; WIN64-NEXT: retq
%1 = fptosi float %d to i128
%2 = trunc i128 %1 to i64
@@ -45,10 +51,12 @@ define i64 @float_to_i128(float %d) nounwind {
define i64 @float_to_ui128(float %d) nounwind {
; WIN64-LABEL: float_to_ui128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $40, %rsp
+; WIN64-NEXT: subq $56, %rsp
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT: movaps %xmm0, %xmm1
; WIN64-NEXT: callq __fixunssfti
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $40, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $56, %rsp
; WIN64-NEXT: retq
%1 = fptoui float %d to i128
%2 = trunc i128 %1 to i64
@@ -58,13 +66,14 @@ define i64 @float_to_ui128(float %d) nounwind {
define i64 @longdouble_to_i128(ptr nocapture readonly %0) nounwind {
; WIN64-LABEL: longdouble_to_i128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $56, %rsp
+; WIN64-NEXT: subq $72, %rsp
; WIN64-NEXT: fldt (%rcx)
; WIN64-NEXT: fstpt {{[0-9]+}}(%rsp)
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; WIN64-NEXT: callq __fixxfti
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $56, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $72, %rsp
; WIN64-NEXT: retq
%2 = load x86_fp80, ptr %0, align 16
%3 = fptosi x86_fp80 %2 to i128
@@ -75,13 +84,14 @@ define i64 @longdouble_to_i128(ptr nocapture readonly %0) nounwind {
define i64 @longdouble_to_ui128(ptr nocapture readonly %0) nounwind {
; WIN64-LABEL: longdouble_to_ui128:
; WIN64: # %bb.0:
-; WIN64-NEXT: subq $56, %rsp
+; WIN64-NEXT: subq $72, %rsp
; WIN64-NEXT: fldt (%rcx)
; WIN64-NEXT: fstpt {{[0-9]+}}(%rsp)
; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT: leaq {{[0-9]+}}(%rsp), %rdx
; WIN64-NEXT: callq __fixunsxfti
-; WIN64-NEXT: movq %xmm0, %rax
-; WIN64-NEXT: addq $56, %rsp
+; WIN64-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT: addq $72, %rsp
; WIN64-NEXT: retq
%2 = load x86_fp80, ptr %0, align 16
%3 = fptoui x86_fp80 %2 to i128
diff --git a/llvm/test/CodeGen/X86/i128-ret.ll b/llvm/test/CodeGen/X86/i128-ret.ll
index 6d208bdef4f343..773bc207e15975 100644
--- a/llvm/test/CodeGen/X86/i128-ret.ll
+++ b/llvm/test/CodeGen/X86/i128-ret.ll
@@ -1,7 +1,10 @@
-; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-win32 | FileCheck %s
-; CHECK: movq ([[A0:%rdi|%rcx]]), %rax
-; CHECK: movq 8([[A0]]), %rdx
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefix=SYSV
+; RUN: llc < %s -mtriple=x86_64-win32 | FileCheck %s --check-prefix=WIN64
+; SYSV: movq (%rdi), %rax
+; SYSV: movq 8(%rdi), %rdx
+; WIN64: movq %rcx, %rax
+; WIN64: movaps (%rdx), %xmm0
+; WIN64: movaps %xmm0, (%rcx)
define i128 @test(ptr%P) {
%A = load i128, ptr %P
diff --git a/llvm/test/CodeGen/X86/mul128.ll b/llvm/test/CodeGen/X86/mul128.ll
index 90d5fc0cd37707..87fefa6cd8d121 100644
--- a/llvm/test/CodeGen/X86/mul128.ll
+++ b/llvm/test/CodeGen/X86/mul128.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-windows-gnu | FileCheck %s --check-prefix=WIN64
; RUN: llc < %s -mtriple=i386-unknown | FileCheck %s --check-prefix=X86
define i128 @foo(i128 %t, i128 %u) {
@@ -14,6 +15,20 @@ define i128 @foo(i128 %t, i128 %u) {
; X64-NEXT: addq %r8, %rdx
; X64-NEXT: retq
;
+; WIN64-LABEL: foo:
+; WIN64: # %bb.0:
+; WIN64-NEXT: movq %rdx, %r10
+; WIN64-NEXT: imulq %r9, %r8
+; WIN64-NEXT: movq %rdx, %rax
+; WIN64-NEXT: mulq %r9
+; WIN64-NEXT: imulq {{[0-9]+}}(%rsp), %r10
+; WIN64-NEXT: addq %r8, %rdx
+; WIN64-NEXT: addq %r10, %rdx
+; WIN64-NEXT: movq %rdx, 8(%rcx)
+; WIN64-NEXT: movq %rax, (%rcx)
+; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: retq
+;
; X86-LABEL: foo:
; X86: # %bb.0:
; X86-NEXT: pushl %ebp
@@ -111,6 +126,16 @@ define void @PR13897() nounwind {
; X64-NEXT: movq %rax, aaa(%rip)
; X64-NEXT: retq
;
+; WIN64-LABEL: PR13897:
+; WIN64: # %bb.0: # %"0x0"
+; WIN64-NEXT: movq bbb(%rip), %rax
+; WIN64-NEXT: movl %eax, %ecx
+; WIN64-NEXT: shlq $32, %rax
+; WIN64-NEXT: orq %rcx, %rax
+; WIN64-NEXT: movq %rax, aaa+8(%rip)
+; WIN64-NEXT: movq %rax, aaa(%rip)
+; WIN64-NEXT: retq
+;
; X86-LABEL: PR13897:
; X86: # %bb.0: # %"0x0"
; X86-NEXT: movl bbb, %eax
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-non-integer.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-non-integer.ll
index 142ebefa5439a4..fc66280e7c5301 100644
--- a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-non-integer.ll
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-non-integer.ll
@@ -1,7 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=CHECK,CHECK64
-; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=CHECK,CHECK32
-; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=x86_64-linux-gnu -mattr=+cx16,-avx | FileCheck %s --check-prefixes=CX16
+; RUN: opt -S %s -passes="require<libcall-lowering-info>,atomic-expand" -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=CHECK,CHECK64
+; RUN: opt -S %s -passes="require<libcall-lowering-info>,atomic-expand" -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=CHECK,CHECK32
+; RUN: opt -S %s -passes="require<libcall-lowering-info>,atomic-expand" -mtriple=x86_64-linux-gnu -mattr=+cx16,-avx | FileCheck %s --check-prefixes=CX16
+; RUN: opt -S %s -passes="require<libcall-lowering-info>,atomic-expand" -mtriple=x86_64-windows-gnu | FileCheck %s --check-prefixes=WIN64
; This file tests AtomicExpand's non-integer atomic type conversions and
; X86-specific expansion choices. If X86 stops using this functionality, please
@@ -19,6 +20,12 @@ define float @float_load_expand(ptr %ptr) {
; CX16-NEXT: [[TMP1:%.*]] = load atomic i32, ptr [[PTR]] unordered, align 4
; CX16-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
; CX16-NEXT: ret float [[TMP2]]
+;
+; WIN64-LABEL: define float @float_load_expand(
+; WIN64-SAME: ptr [[PTR:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = load atomic i32, ptr [[PTR]] unordered, align 4
+; WIN64-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
+; WIN64-NEXT: ret float [[TMP2]]
;
%res = load atomic float, ptr %ptr unordered, align 4
ret float %res
@@ -36,6 +43,12 @@ define float @float_load_expand_seq_cst(ptr %ptr) {
; CX16-NEXT: [[TMP1:%.*]] = load atomic i32, ptr [[PTR]] seq_cst, align 4
; CX16-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
; CX16-NEXT: ret float [[TMP2]]
+;
+; WIN64-LABEL: define float @float_load_expand_seq_cst(
+; WIN64-SAME: ptr [[PTR:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = load atomic i32, ptr [[PTR]] seq_cst, align 4
+; WIN64-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
+; WIN64-NEXT: ret float [[TMP2]]
;
%res = load atomic float, ptr %ptr seq_cst, align 4
ret float %res
@@ -53,6 +66,12 @@ define float @float_load_expand_vol(ptr %ptr) {
; CX16-NEXT: [[TMP1:%.*]] = load atomic volatile i32, ptr [[PTR]] unordered, align 4
; CX16-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
; CX16-NEXT: ret float [[TMP2]]
+;
+; WIN64-LABEL: define float @float_load_expand_vol(
+; WIN64-SAME: ptr [[PTR:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = load atomic volatile i32, ptr [[PTR]] unordered, align 4
+; WIN64-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
+; WIN64-NEXT: ret float [[TMP2]]
;
%res = load atomic volatile float, ptr %ptr unordered, align 4
ret float %res
@@ -70,6 +89,12 @@ define float @float_load_expand_addr1(ptr addrspace(1) %ptr) {
; CX16-NEXT: [[TMP1:%.*]] = load atomic i32, ptr addrspace(1) [[PTR]] unordered, align 4
; CX16-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
; CX16-NEXT: ret float [[TMP2]]
+;
+; WIN64-LABEL: define float @float_load_expand_addr1(
+; WIN64-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = load atomic i32, ptr addrspace(1) [[PTR]] unordered, align 4
+; WIN64-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
+; WIN64-NEXT: ret float [[TMP2]]
;
%res = load atomic float, ptr addrspace(1) %ptr unordered, align 4
ret float %res
@@ -87,6 +112,12 @@ define void @float_store_expand(ptr %ptr, float %v) {
; CX16-NEXT: [[TMP1:%.*]] = bitcast float [[V]] to i32
; CX16-NEXT: store atomic i32 [[TMP1]], ptr [[PTR]] unordered, align 4
; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @float_store_expand(
+; WIN64-SAME: ptr [[PTR:%.*]], float [[V:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = bitcast float [[V]] to i32
+; WIN64-NEXT: store atomic i32 [[TMP1]], ptr [[PTR]] unordered, align 4
+; WIN64-NEXT: ret void
;
store atomic float %v, ptr %ptr unordered, align 4
ret void
@@ -104,6 +135,12 @@ define void @float_store_expand_seq_cst(ptr %ptr, float %v) {
; CX16-NEXT: [[TMP1:%.*]] = bitcast float [[V]] to i32
; CX16-NEXT: store atomic i32 [[TMP1]], ptr [[PTR]] seq_cst, align 4
; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @float_store_expand_seq_cst(
+; WIN64-SAME: ptr [[PTR:%.*]], float [[V:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = bitcast float [[V]] to i32
+; WIN64-NEXT: store atomic i32 [[TMP1]], ptr [[PTR]] seq_cst, align 4
+; WIN64-NEXT: ret void
;
store atomic float %v, ptr %ptr seq_cst, align 4
ret void
@@ -121,6 +158,12 @@ define void @float_store_expand_vol(ptr %ptr, float %v) {
; CX16-NEXT: [[TMP1:%.*]] = bitcast float [[V]] to i32
; CX16-NEXT: store atomic volatile i32 [[TMP1]], ptr [[PTR]] unordered, align 4
; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @float_store_expand_vol(
+; WIN64-SAME: ptr [[PTR:%.*]], float [[V:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = bitcast float [[V]] to i32
+; WIN64-NEXT: store atomic volatile i32 [[TMP1]], ptr [[PTR]] unordered, align 4
+; WIN64-NEXT: ret void
;
store atomic volatile float %v, ptr %ptr unordered, align 4
ret void
@@ -138,6 +181,12 @@ define void @float_store_expand_addr1(ptr addrspace(1) %ptr, float %v) {
; CX16-NEXT: [[TMP1:%.*]] = bitcast float [[V]] to i32
; CX16-NEXT: store atomic i32 [[TMP1]], ptr addrspace(1) [[PTR]] unordered, align 4
; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @float_store_expand_addr1(
+; WIN64-SAME: ptr addrspace(1) [[PTR:%.*]], float [[V:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = bitcast float [[V]] to i32
+; WIN64-NEXT: store atomic i32 [[TMP1]], ptr addrspace(1) [[PTR]] unordered, align 4
+; WIN64-NEXT: ret void
;
store atomic float %v, ptr addrspace(1) %ptr unordered, align 4
ret void
@@ -176,6 +225,17 @@ define void @pointer_cmpxchg_expand(ptr %ptr, ptr %v) {
; CX16-NEXT: [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CX16-NEXT: [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @pointer_cmpxchg_expand(
+; WIN64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
+; WIN64-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 0, i64 [[TMP1]] seq_cst monotonic, align 8
+; WIN64-NEXT: [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
+; WIN64-NEXT: [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; WIN64-NEXT: [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; WIN64-NEXT: [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
+; WIN64-NEXT: [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
+; WIN64-NEXT: ret void
;
cmpxchg ptr %ptr, ptr null, ptr %v seq_cst monotonic
ret void
@@ -214,6 +274,17 @@ define void @pointer_cmpxchg_expand2(ptr %ptr, ptr %v) {
; CX16-NEXT: [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CX16-NEXT: [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @pointer_cmpxchg_expand2(
+; WIN64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
+; WIN64-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 0, i64 [[TMP1]] release monotonic, align 8
+; WIN64-NEXT: [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
+; WIN64-NEXT: [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; WIN64-NEXT: [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; WIN64-NEXT: [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
+; WIN64-NEXT: [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
+; WIN64-NEXT: ret void
;
cmpxchg ptr %ptr, ptr null, ptr %v release monotonic
ret void
@@ -252,6 +323,17 @@ define void @pointer_cmpxchg_expand3(ptr %ptr, ptr %v) {
; CX16-NEXT: [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CX16-NEXT: [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @pointer_cmpxchg_expand3(
+; WIN64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
+; WIN64-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 0, i64 [[TMP1]] seq_cst seq_cst, align 8
+; WIN64-NEXT: [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
+; WIN64-NEXT: [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; WIN64-NEXT: [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; WIN64-NEXT: [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
+; WIN64-NEXT: [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
+; WIN64-NEXT: ret void
;
cmpxchg ptr %ptr, ptr null, ptr %v seq_cst seq_cst
ret void
@@ -290,6 +372,17 @@ define void @pointer_cmpxchg_expand4(ptr %ptr, ptr %v) {
; CX16-NEXT: [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CX16-NEXT: [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @pointer_cmpxchg_expand4(
+; WIN64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
+; WIN64-NEXT: [[TMP2:%.*]] = cmpxchg weak ptr [[PTR]], i64 0, i64 [[TMP1]] seq_cst seq_cst, align 8
+; WIN64-NEXT: [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
+; WIN64-NEXT: [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; WIN64-NEXT: [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; WIN64-NEXT: [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
+; WIN64-NEXT: [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
+; WIN64-NEXT: ret void
;
cmpxchg weak ptr %ptr, ptr null, ptr %v seq_cst seq_cst
ret void
@@ -328,6 +421,17 @@ define void @pointer_cmpxchg_expand5(ptr %ptr, ptr %v) {
; CX16-NEXT: [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CX16-NEXT: [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @pointer_cmpxchg_expand5(
+; WIN64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
+; WIN64-NEXT: [[TMP2:%.*]] = cmpxchg volatile ptr [[PTR]], i64 0, i64 [[TMP1]] seq_cst seq_cst, align 8
+; WIN64-NEXT: [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
+; WIN64-NEXT: [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; WIN64-NEXT: [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; WIN64-NEXT: [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
+; WIN64-NEXT: [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
+; WIN64-NEXT: ret void
;
cmpxchg volatile ptr %ptr, ptr null, ptr %v seq_cst seq_cst
ret void
@@ -366,6 +470,17 @@ define void @pointer_cmpxchg_expand6(ptr addrspace(1) %ptr, ptr addrspace(2) %v)
; CX16-NEXT: [[TMP6:%.*]] = insertvalue { ptr addrspace(2), i1 } poison, ptr addrspace(2) [[TMP5]], 0
; CX16-NEXT: [[TMP7:%.*]] = insertvalue { ptr addrspace(2), i1 } [[TMP6]], i1 [[TMP4]], 1
; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @pointer_cmpxchg_expand6(
+; WIN64-SAME: ptr addrspace(1) [[PTR:%.*]], ptr addrspace(2) [[V:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(2) [[V]] to i64
+; WIN64-NEXT: [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 0, i64 [[TMP1]] seq_cst seq_cst, align 8
+; WIN64-NEXT: [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
+; WIN64-NEXT: [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; WIN64-NEXT: [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr addrspace(2)
+; WIN64-NEXT: [[TMP6:%.*]] = insertvalue { ptr addrspace(2), i1 } poison, ptr addrspace(2) [[TMP5]], 0
+; WIN64-NEXT: [[TMP7:%.*]] = insertvalue { ptr addrspace(2), i1 } [[TMP6]], i1 [[TMP4]], 1
+; WIN64-NEXT: ret void
;
cmpxchg ptr addrspace(1) %ptr, ptr addrspace(2) null, ptr addrspace(2) %v seq_cst seq_cst
ret void
@@ -388,6 +503,13 @@ define <2 x ptr> @atomic_vec2_ptr_align(ptr %x) nounwind {
; CX16-SAME: ptr [[X:%.*]]) #[[ATTR1:[0-9]+]] {
; CX16-NEXT: [[RET:%.*]] = load atomic <2 x ptr>, ptr [[X]] acquire, align 16
; CX16-NEXT: ret <2 x ptr> [[RET]]
+;
+; WIN64-LABEL: define <2 x ptr> @atomic_vec2_ptr_align(
+; WIN64-SAME: ptr [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; WIN64-NEXT: [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[X]], i32 2)
+; WIN64-NEXT: [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <2 x i64>
+; WIN64-NEXT: [[TMP3:%.*]] = inttoptr <2 x i64> [[TMP2]] to <2 x ptr>
+; WIN64-NEXT: ret <2 x ptr> [[TMP3]]
;
%ret = load atomic <2 x ptr>, ptr %x acquire, align 16
ret <2 x ptr> %ret
@@ -414,6 +536,13 @@ define <4 x ptr addrspace(270)> @atomic_vec4_ptr_align(ptr %x) nounwind {
; CX16-SAME: ptr [[X:%.*]]) #[[ATTR1]] {
; CX16-NEXT: [[RET:%.*]] = load atomic <4 x ptr addrspace(270)>, ptr [[X]] acquire, align 16
; CX16-NEXT: ret <4 x ptr addrspace(270)> [[RET]]
+;
+; WIN64-LABEL: define <4 x ptr addrspace(270)> @atomic_vec4_ptr_align(
+; WIN64-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT: [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[X]], i32 2)
+; WIN64-NEXT: [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <4 x i32>
+; WIN64-NEXT: [[TMP3:%.*]] = inttoptr <4 x i32> [[TMP2]] to <4 x ptr addrspace(270)>
+; WIN64-NEXT: ret <4 x ptr addrspace(270)> [[TMP3]]
;
%ret = load atomic <4 x ptr addrspace(270)>, ptr %x acquire, align 16
ret <4 x ptr addrspace(270)> %ret
@@ -429,6 +558,11 @@ define <2 x i16> @atomic_vec2_i16(ptr %x) nounwind {
; CX16-SAME: ptr [[X:%.*]]) #[[ATTR1]] {
; CX16-NEXT: [[RET:%.*]] = load atomic <2 x i16>, ptr [[X]] acquire, align 8
; CX16-NEXT: ret <2 x i16> [[RET]]
+;
+; WIN64-LABEL: define <2 x i16> @atomic_vec2_i16(
+; WIN64-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT: [[RET:%.*]] = load atomic <2 x i16>, ptr [[X]] acquire, align 8
+; WIN64-NEXT: ret <2 x i16> [[RET]]
;
%ret = load atomic <2 x i16>, ptr %x acquire, align 8
ret <2 x i16> %ret
@@ -444,6 +578,11 @@ define <2 x half> @atomic_vec2_half(ptr %x) nounwind {
; CX16-SAME: ptr [[X:%.*]]) #[[ATTR1]] {
; CX16-NEXT: [[TMP2:%.*]] = load atomic <2 x half>, ptr [[X]] acquire, align 8
; CX16-NEXT: ret <2 x half> [[TMP2]]
+;
+; WIN64-LABEL: define <2 x half> @atomic_vec2_half(
+; WIN64-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT: [[RET:%.*]] = load atomic <2 x half>, ptr [[X]] acquire, align 8
+; WIN64-NEXT: ret <2 x half> [[RET]]
;
%ret = load atomic <2 x half>, ptr %x acquire, align 8
ret <2 x half> %ret
@@ -471,6 +610,12 @@ define <4 x i32> @atomic_vec4_i32(ptr %x) nounwind {
; CX16-NEXT: [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
; CX16-NEXT: [[TMP2:%.*]] = bitcast i128 [[LOADED]] to <4 x i32>
; CX16-NEXT: ret <4 x i32> [[TMP2]]
+;
+; WIN64-LABEL: define <4 x i32> @atomic_vec4_i32(
+; WIN64-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT: [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[X]], i32 2)
+; WIN64-NEXT: [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <4 x i32>
+; WIN64-NEXT: ret <4 x i32> [[TMP2]]
;
%ret = load atomic <4 x i32>, ptr %x acquire, align 16
ret <4 x i32> %ret
@@ -498,6 +643,12 @@ define <4 x float> @atomic_vec4_float(ptr %x) nounwind {
; CX16-NEXT: [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
; CX16-NEXT: [[TMP2:%.*]] = bitcast i128 [[LOADED]] to <4 x float>
; CX16-NEXT: ret <4 x float> [[TMP2]]
+;
+; WIN64-LABEL: define <4 x float> @atomic_vec4_float(
+; WIN64-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT: [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[X]], i32 2)
+; WIN64-NEXT: [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <4 x float>
+; WIN64-NEXT: ret <4 x float> [[TMP2]]
;
%ret = load atomic <4 x float>, ptr %x acquire, align 16
ret <4 x float> %ret
@@ -528,6 +679,12 @@ define <2 x i64> @load_v2i64_cmpxchg(ptr %p) {
; CX16-NEXT: [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
; CX16-NEXT: [[TMP2:%.*]] = bitcast i128 [[LOADED]] to <2 x i64>
; CX16-NEXT: ret <2 x i64> [[TMP2]]
+;
+; WIN64-LABEL: define <2 x i64> @load_v2i64_cmpxchg(
+; WIN64-SAME: ptr [[P:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[P]], i32 5)
+; WIN64-NEXT: [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <2 x i64>
+; WIN64-NEXT: ret <2 x i64> [[TMP2]]
;
%r = load atomic <2 x i64>, ptr %p seq_cst, align 16
ret <2 x i64> %r
@@ -555,6 +712,12 @@ define <4 x i32> @load_v4i32_cmpxchg(ptr %p) {
; CX16-NEXT: [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
; CX16-NEXT: [[TMP2:%.*]] = bitcast i128 [[LOADED]] to <4 x i32>
; CX16-NEXT: ret <4 x i32> [[TMP2]]
+;
+; WIN64-LABEL: define <4 x i32> @load_v4i32_cmpxchg(
+; WIN64-SAME: ptr [[P:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[P]], i32 5)
+; WIN64-NEXT: [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <4 x i32>
+; WIN64-NEXT: ret <4 x i32> [[TMP2]]
;
%r = load atomic <4 x i32>, ptr %p seq_cst, align 16
ret <4 x i32> %r
@@ -582,6 +745,12 @@ define <16 x i8> @load_v16i8_cmpxchg(ptr %p) {
; CX16-NEXT: [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
; CX16-NEXT: [[TMP2:%.*]] = bitcast i128 [[LOADED]] to <16 x i8>
; CX16-NEXT: ret <16 x i8> [[TMP2]]
+;
+; WIN64-LABEL: define <16 x i8> @load_v16i8_cmpxchg(
+; WIN64-SAME: ptr [[P:%.*]]) {
+; WIN64-NEXT: [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[P]], i32 5)
+; WIN64-NEXT: [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <16 x i8>
+; WIN64-NEXT: ret <16 x i8> [[TMP2]]
;
%r = load atomic <16 x i8>, ptr %p seq_cst, align 16
ret <16 x i8> %r
@@ -610,6 +779,11 @@ define i128 @load_i128_volatile_syncscope(ptr %p) {
; CX16-NEXT: [[TMP1:%.*]] = cmpxchg volatile ptr [[P]], i128 0, i128 0 syncscope("singlethread") seq_cst seq_cst, align 16
; CX16-NEXT: [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
; CX16-NEXT: ret i128 [[LOADED]]
+;
+; WIN64-LABEL: define i128 @load_i128_volatile_syncscope(
+; WIN64-SAME: ptr [[P:%.*]]) {
+; WIN64-NEXT: [[VALUE:%.*]] = call i128 @__atomic_load_16(ptr [[P]], i32 5)
+; WIN64-NEXT: ret i128 [[VALUE]]
;
%v = load atomic volatile i128, ptr %p syncscope("singlethread") seq_cst, align 16
ret i128 %v
@@ -642,11 +816,120 @@ define void @store_i128_volatile_syncscope(ptr %p, i128 %x) {
; CX16-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
; CX16: [[ATOMICRMW_END]]:
; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @store_i128_volatile_syncscope(
+; WIN64-SAME: ptr [[P:%.*]], i128 [[X:%.*]]) {
+; WIN64-NEXT: [[VALUE:%.*]] = alloca i128, align 16
+; WIN64-NEXT: call void @llvm.lifetime.start.p0(ptr [[VALUE]])
+; WIN64-NEXT: store i128 [[X]], ptr [[VALUE]], align 16
+; WIN64-NEXT: call void @__atomic_store_16(ptr [[P]], ptr [[VALUE]], i32 5)
+; WIN64-NEXT: call void @llvm.lifetime.end.p0(ptr [[VALUE]])
+; WIN64-NEXT: ret void
;
store atomic volatile i128 %x, ptr %p syncscope("singlethread") seq_cst, align 16
ret void
}
+define i128 @xchg_i128(ptr %p, i128 %x) {
+; CHECK64-LABEL: define i128 @xchg_i128(
+; CHECK64-SAME: ptr [[P:%.*]], i128 [[X:%.*]]) {
+; CHECK64-NEXT: [[TMP1:%.*]] = call i128 @__atomic_exchange_16(ptr [[P]], i128 [[X]], i32 5)
+; CHECK64-NEXT: ret i128 [[TMP1]]
+;
+; CHECK32-LABEL: define i128 @xchg_i128(
+; CHECK32-SAME: ptr [[P:%.*]], i128 [[X:%.*]]) {
+; CHECK32-NEXT: [[TMP1:%.*]] = alloca i128, align 16
+; CHECK32-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK32-NEXT: store i128 [[X]], ptr [[TMP1]], align 16
+; CHECK32-NEXT: [[TMP2:%.*]] = alloca i128, align 16
+; CHECK32-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP2]])
+; CHECK32-NEXT: call void @__atomic_exchange(i32 16, ptr [[P]], ptr [[TMP1]], ptr [[TMP2]], i32 5)
+; CHECK32-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK32-NEXT: [[TMP3:%.*]] = load i128, ptr [[TMP2]], align 16
+; CHECK32-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP2]])
+; CHECK32-NEXT: ret i128 [[TMP3]]
+;
+; CX16-LABEL: define i128 @xchg_i128(
+; CX16-SAME: ptr [[P:%.*]], i128 [[X:%.*]]) #[[ATTR0]] {
+; CX16-NEXT: [[TMP1:%.*]] = load i128, ptr [[P]], align 16
+; CX16-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CX16: [[ATOMICRMW_START]]:
+; CX16-NEXT: [[LOADED:%.*]] = phi i128 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CX16-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[P]], i128 [[LOADED]], i128 [[X]] seq_cst seq_cst, align 16
+; CX16-NEXT: [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1
+; CX16-NEXT: [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP2]], 0
+; CX16-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CX16: [[ATOMICRMW_END]]:
+; CX16-NEXT: ret i128 [[NEWLOADED]]
+;
+; WIN64-LABEL: define i128 @xchg_i128(
+; WIN64-SAME: ptr [[P:%.*]], i128 [[X:%.*]]) {
+; WIN64-NEXT: [[VALUE:%.*]] = alloca i128, align 16
+; WIN64-NEXT: call void @llvm.lifetime.start.p0(ptr [[VALUE]])
+; WIN64-NEXT: store i128 [[X]], ptr [[VALUE]], align 16
+; WIN64-NEXT: [[RESULT:%.*]] = call i128 @__atomic_exchange_16(ptr [[P]], ptr [[VALUE]], i32 5)
+; WIN64-NEXT: call void @llvm.lifetime.end.p0(ptr [[VALUE]])
+; WIN64-NEXT: ret i128 [[RESULT]]
+;
+ %result = atomicrmw xchg ptr %p, i128 %x seq_cst, align 16
+ ret i128 %result
+}
+
+define { i128, i1 } @cmpxchg_i128(ptr %p, i128 %expected, i128 %desired) {
+; CHECK64-LABEL: define { i128, i1 } @cmpxchg_i128(
+; CHECK64-SAME: ptr [[P:%.*]], i128 [[EXPECTED:%.*]], i128 [[DESIRED:%.*]]) {
+; CHECK64-NEXT: [[TMP1:%.*]] = alloca i128, align 16
+; CHECK64-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK64-NEXT: store i128 [[EXPECTED]], ptr [[TMP1]], align 16
+; CHECK64-NEXT: [[TMP2:%.*]] = call zeroext i1 @__atomic_compare_exchange_16(ptr [[P]], ptr [[TMP1]], i128 [[DESIRED]], i32 5, i32 5)
+; CHECK64-NEXT: [[TMP3:%.*]] = load i128, ptr [[TMP1]], align 16
+; CHECK64-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK64-NEXT: [[TMP4:%.*]] = insertvalue { i128, i1 } poison, i128 [[TMP3]], 0
+; CHECK64-NEXT: [[TMP5:%.*]] = insertvalue { i128, i1 } [[TMP4]], i1 [[TMP2]], 1
+; CHECK64-NEXT: ret { i128, i1 } [[TMP5]]
+;
+; CHECK32-LABEL: define { i128, i1 } @cmpxchg_i128(
+; CHECK32-SAME: ptr [[P:%.*]], i128 [[EXPECTED:%.*]], i128 [[DESIRED:%.*]]) {
+; CHECK32-NEXT: [[TMP1:%.*]] = alloca i128, align 16
+; CHECK32-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK32-NEXT: store i128 [[EXPECTED]], ptr [[TMP1]], align 16
+; CHECK32-NEXT: [[TMP2:%.*]] = alloca i128, align 16
+; CHECK32-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP2]])
+; CHECK32-NEXT: store i128 [[DESIRED]], ptr [[TMP2]], align 16
+; CHECK32-NEXT: [[TMP3:%.*]] = call zeroext i1 @__atomic_compare_exchange(i32 16, ptr [[P]], ptr [[TMP1]], ptr [[TMP2]], i32 5, i32 5)
+; CHECK32-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP2]])
+; CHECK32-NEXT: [[TMP4:%.*]] = load i128, ptr [[TMP1]], align 16
+; CHECK32-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK32-NEXT: [[TMP5:%.*]] = insertvalue { i128, i1 } poison, i128 [[TMP4]], 0
+; CHECK32-NEXT: [[TMP6:%.*]] = insertvalue { i128, i1 } [[TMP5]], i1 [[TMP3]], 1
+; CHECK32-NEXT: ret { i128, i1 } [[TMP6]]
+;
+; CX16-LABEL: define { i128, i1 } @cmpxchg_i128(
+; CX16-SAME: ptr [[P:%.*]], i128 [[EXPECTED:%.*]], i128 [[DESIRED:%.*]]) #[[ATTR0]] {
+; CX16-NEXT: [[RESULT:%.*]] = cmpxchg ptr [[P]], i128 [[EXPECTED]], i128 [[DESIRED]] seq_cst seq_cst, align 16
+; CX16-NEXT: ret { i128, i1 } [[RESULT]]
+;
+; WIN64-LABEL: define { i128, i1 } @cmpxchg_i128(
+; WIN64-SAME: ptr [[P:%.*]], i128 [[EXPECTED:%.*]], i128 [[DESIRED:%.*]]) {
+; WIN64-NEXT: [[EXPECTED_ADDR:%.*]] = alloca i128, align 16
+; WIN64-NEXT: call void @llvm.lifetime.start.p0(ptr [[EXPECTED_ADDR]])
+; WIN64-NEXT: store i128 [[EXPECTED]], ptr [[EXPECTED_ADDR]], align 16
+; WIN64-NEXT: [[DESIRED_ADDR:%.*]] = alloca i128, align 16
+; WIN64-NEXT: call void @llvm.lifetime.start.p0(ptr [[DESIRED_ADDR]])
+; WIN64-NEXT: store i128 [[DESIRED]], ptr [[DESIRED_ADDR]], align 16
+; WIN64-NEXT: [[SUCCESS:%.*]] = call zeroext i1 @__atomic_compare_exchange_16(ptr [[P]], ptr [[EXPECTED_ADDR]], ptr [[DESIRED_ADDR]], i32 5, i32 5)
+; WIN64-NEXT: call void @llvm.lifetime.end.p0(ptr [[DESIRED_ADDR]])
+; WIN64-NEXT: [[OLD:%.*]] = load i128, ptr [[EXPECTED_ADDR]], align 16
+; WIN64-NEXT: call void @llvm.lifetime.end.p0(ptr [[EXPECTED_ADDR]])
+; WIN64-NEXT: [[WITH_OLD:%.*]] = insertvalue { i128, i1 } poison, i128 [[OLD]], 0
+; WIN64-NEXT: [[RESULT:%.*]] = insertvalue { i128, i1 } [[WITH_OLD]], i1 [[SUCCESS]], 1
+; WIN64-NEXT: ret { i128, i1 } [[RESULT]]
+;
+ %result = cmpxchg ptr %p, i128 %expected, i128 %desired seq_cst seq_cst,
+ align 16
+ ret { i128, i1 } %result
+}
+
define void @store_atomic_vec2_ptr_align(ptr %x, <2 x ptr> %v) nounwind {
; CHECK64-LABEL: define void @store_atomic_vec2_ptr_align(
; CHECK64-SAME: ptr [[X:%.*]], <2 x ptr> [[V:%.*]]) #[[ATTR0]] {
@@ -659,6 +942,22 @@ define void @store_atomic_vec2_ptr_align(ptr %x, <2 x ptr> %v) nounwind {
; CHECK32-SAME: ptr [[X:%.*]], <2 x ptr> [[V:%.*]]) #[[ATTR0]] {
; CHECK32-NEXT: store atomic <2 x ptr> [[V]], ptr [[X]] release, align 16
; CHECK32-NEXT: ret void
+;
+; CX16-LABEL: define void @store_atomic_vec2_ptr_align(
+; CX16-SAME: ptr [[X:%.*]], <2 x ptr> [[V:%.*]]) #[[ATTR1]] {
+; CX16-NEXT: store atomic <2 x ptr> [[V]], ptr [[X]] release, align 16
+; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @store_atomic_vec2_ptr_align(
+; WIN64-SAME: ptr [[X:%.*]], <2 x ptr> [[V:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT: [[TMP1:%.*]] = ptrtoint <2 x ptr> [[V]] to <2 x i64>
+; WIN64-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; WIN64-NEXT: [[TMP3:%.*]] = alloca i128, align 16
+; WIN64-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP3]])
+; WIN64-NEXT: store i128 [[TMP2]], ptr [[TMP3]], align 16
+; WIN64-NEXT: call void @__atomic_store_16(ptr [[X]], ptr [[TMP3]], i32 3)
+; WIN64-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP3]])
+; WIN64-NEXT: ret void
;
store atomic <2 x ptr> %v, ptr %x release, align 16
ret void
@@ -680,6 +979,22 @@ define void @store_atomic_vec4_ptr270_align(ptr %x, <4 x ptr addrspace(270)> %v)
; CHECK32-NEXT: call void @__atomic_store(i32 16, ptr [[X]], ptr [[TMP1]], i32 3)
; CHECK32-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP1]])
; CHECK32-NEXT: ret void
+;
+; CX16-LABEL: define void @store_atomic_vec4_ptr270_align(
+; CX16-SAME: ptr [[X:%.*]], <4 x ptr addrspace(270)> [[V:%.*]]) #[[ATTR1]] {
+; CX16-NEXT: store atomic <4 x ptr addrspace(270)> [[V]], ptr [[X]] release, align 16
+; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @store_atomic_vec4_ptr270_align(
+; WIN64-SAME: ptr [[X:%.*]], <4 x ptr addrspace(270)> [[V:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT: [[TMP1:%.*]] = ptrtoint <4 x ptr addrspace(270)> [[V]] to <4 x i32>
+; WIN64-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; WIN64-NEXT: [[TMP3:%.*]] = alloca i128, align 16
+; WIN64-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP3]])
+; WIN64-NEXT: store i128 [[TMP2]], ptr [[TMP3]], align 16
+; WIN64-NEXT: call void @__atomic_store_16(ptr [[X]], ptr [[TMP3]], i32 3)
+; WIN64-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP3]])
+; WIN64-NEXT: ret void
;
store atomic <4 x ptr addrspace(270)> %v, ptr %x release, align 16
ret void
@@ -690,6 +1005,16 @@ define void @store_atomic_vec2_i16(ptr %x, <2 x i16> %v) nounwind {
; CHECK-SAME: ptr [[X:%.*]], <2 x i16> [[V:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: store atomic <2 x i16> [[V]], ptr [[X]] release, align 8
; CHECK-NEXT: ret void
+;
+; CX16-LABEL: define void @store_atomic_vec2_i16(
+; CX16-SAME: ptr [[X:%.*]], <2 x i16> [[V:%.*]]) #[[ATTR1]] {
+; CX16-NEXT: store atomic <2 x i16> [[V]], ptr [[X]] release, align 8
+; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @store_atomic_vec2_i16(
+; WIN64-SAME: ptr [[X:%.*]], <2 x i16> [[V:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT: store atomic <2 x i16> [[V]], ptr [[X]] release, align 8
+; WIN64-NEXT: ret void
;
store atomic <2 x i16> %v, ptr %x release, align 8
ret void
@@ -700,6 +1025,16 @@ define void @store_atomic_vec2_half(ptr %x, <2 x half> %v) nounwind {
; CHECK-SAME: ptr [[X:%.*]], <2 x half> [[V:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: store atomic <2 x half> [[V]], ptr [[X]] release, align 8
; CHECK-NEXT: ret void
+;
+; CX16-LABEL: define void @store_atomic_vec2_half(
+; CX16-SAME: ptr [[X:%.*]], <2 x half> [[V:%.*]]) #[[ATTR1]] {
+; CX16-NEXT: store atomic <2 x half> [[V]], ptr [[X]] release, align 8
+; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @store_atomic_vec2_half(
+; WIN64-SAME: ptr [[X:%.*]], <2 x half> [[V:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT: store atomic <2 x half> [[V]], ptr [[X]] release, align 8
+; WIN64-NEXT: ret void
;
store atomic <2 x half> %v, ptr %x release, align 8
ret void
@@ -720,6 +1055,32 @@ define void @store_atomic_vec4_i32(ptr %x, <4 x i32> %v) nounwind {
; CHECK32-NEXT: call void @__atomic_store(i32 16, ptr [[X]], ptr [[TMP1]], i32 3)
; CHECK32-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP1]])
; CHECK32-NEXT: ret void
+;
+; CX16-LABEL: define void @store_atomic_vec4_i32(
+; CX16-SAME: ptr [[X:%.*]], <4 x i32> [[V:%.*]]) #[[ATTR1]] {
+; CX16-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[X]], align 16
+; CX16-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CX16: [[ATOMICRMW_START]]:
+; CX16-NEXT: [[LOADED:%.*]] = phi <4 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CX16-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[V]] to i128
+; CX16-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[LOADED]] to i128
+; CX16-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[X]], i128 [[TMP3]], i128 [[TMP2]] release monotonic, align 16
+; CX16-NEXT: [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP4]], 1
+; CX16-NEXT: [[NEWLOADED:%.*]] = extractvalue { i128, i1 } [[TMP4]], 0
+; CX16-NEXT: [[TMP5]] = bitcast i128 [[NEWLOADED]] to <4 x i32>
+; CX16-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CX16: [[ATOMICRMW_END]]:
+; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @store_atomic_vec4_i32(
+; WIN64-SAME: ptr [[X:%.*]], <4 x i32> [[V:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[V]] to i128
+; WIN64-NEXT: [[TMP2:%.*]] = alloca i128, align 16
+; WIN64-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP2]])
+; WIN64-NEXT: store i128 [[TMP1]], ptr [[TMP2]], align 16
+; WIN64-NEXT: call void @__atomic_store_16(ptr [[X]], ptr [[TMP2]], i32 3)
+; WIN64-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP2]])
+; WIN64-NEXT: ret void
;
store atomic <4 x i32> %v, ptr %x release, align 16
ret void
@@ -740,6 +1101,32 @@ define void @store_atomic_vec4_float(ptr %x, <4 x float> %v) nounwind {
; CHECK32-NEXT: call void @__atomic_store(i32 16, ptr [[X]], ptr [[TMP1]], i32 3)
; CHECK32-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP1]])
; CHECK32-NEXT: ret void
+;
+; CX16-LABEL: define void @store_atomic_vec4_float(
+; CX16-SAME: ptr [[X:%.*]], <4 x float> [[V:%.*]]) #[[ATTR1]] {
+; CX16-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[X]], align 16
+; CX16-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CX16: [[ATOMICRMW_START]]:
+; CX16-NEXT: [[LOADED:%.*]] = phi <4 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CX16-NEXT: [[TMP2:%.*]] = bitcast <4 x float> [[V]] to i128
+; CX16-NEXT: [[TMP3:%.*]] = bitcast <4 x float> [[LOADED]] to i128
+; CX16-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[X]], i128 [[TMP3]], i128 [[TMP2]] release monotonic, align 16
+; CX16-NEXT: [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP4]], 1
+; CX16-NEXT: [[NEWLOADED:%.*]] = extractvalue { i128, i1 } [[TMP4]], 0
+; CX16-NEXT: [[TMP5]] = bitcast i128 [[NEWLOADED]] to <4 x float>
+; CX16-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CX16: [[ATOMICRMW_END]]:
+; CX16-NEXT: ret void
+;
+; WIN64-LABEL: define void @store_atomic_vec4_float(
+; WIN64-SAME: ptr [[X:%.*]], <4 x float> [[V:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT: [[TMP1:%.*]] = bitcast <4 x float> [[V]] to i128
+; WIN64-NEXT: [[TMP2:%.*]] = alloca i128, align 16
+; WIN64-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP2]])
+; WIN64-NEXT: store i128 [[TMP1]], ptr [[TMP2]], align 16
+; WIN64-NEXT: call void @__atomic_store_16(ptr [[X]], ptr [[TMP2]], i32 3)
+; WIN64-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP2]])
+; WIN64-NEXT: ret void
;
store atomic <4 x float> %v, ptr %x release, align 16
ret void
More information about the cfe-commits
mailing list