[clang] [llvm] [clang][X86] Return Win64 __int128 indirectly (PR #210358)

via cfe-commits cfe-commits at lists.llvm.org
Wed Nov 4 09:54:07 PST 2026


https://github.com/oltolm updated https://github.com/llvm/llvm-project/pull/210358

>From ab7ae6021b8a084a2e812994406f738912c91191 Mon Sep 17 00:00:00 2001
From: Oleg Tolmatcev <oleg.tolmatcev at gmail.com>
Date: Wed, 4 Nov 2026 18:46:09 +0100
Subject: [PATCH] [clang][X86] Return Win64 __int128 indirectly

Return Win64 __int128 and unsigned __int128 values through a hidden sret pointer instead of treating them as v2i64 values returned in XMM0. For instance methods, pass the sret pointer after this to match Microsoft ABI argument ordering. -fclang-abi-compat=23 restores Clang's previous source-level convention.

Make the X86 backend apply the same return convention to i128 libcalls and pass 16-byte atomic libcall operands indirectly on Windows. This matches GCC's ABI while preserving inline i128 multiplication lowering.

Assisted-by: OpenAI Codex
---
 clang/docs/ReleaseNotes.md                    |   7 +
 clang/lib/CodeGen/Targets/X86.cpp             |  56 +-
 clang/test/CodeGen/win64-i128.c               |  45 +-
 clang/test/CodeGenCXX/win64-i128.cpp          |  37 +
 llvm/lib/CodeGen/AtomicExpandPass.cpp         |  28 +-
 llvm/lib/Target/X86/X86ISelLowering.cpp       |  37 +-
 llvm/lib/Target/X86/X86ISelLoweringCall.cpp   |  16 +-
 llvm/test/CodeGen/X86/atomic-nocx16-win.ll    |   9 +-
 llvm/test/CodeGen/X86/divmod128.ll            | 894 ++++++++++--------
 llvm/test/CodeGen/X86/fp128-cast.ll           |  12 +-
 .../CodeGen/X86/fp128-return-calling-conv.ll  |  41 +-
 llvm/test/CodeGen/X86/fp80-conv-libcalls.ll   |  16 +-
 llvm/test/CodeGen/X86/fptosi-f16-wide.ll      |  44 +-
 llvm/test/CodeGen/X86/fptoui-f16-wide.ll      |  36 +-
 llvm/test/CodeGen/X86/i128-fp128-abi.ll       | 128 ++-
 .../CodeGen/X86/i128-fpconv-win64-strict.ll   |  46 +-
 llvm/test/CodeGen/X86/i128-fpconv-win64.ll    |  46 +-
 llvm/test/CodeGen/X86/i128-ret.ll             |  11 +-
 llvm/test/CodeGen/X86/mul128.ll               |  25 +
 .../X86/expand-atomic-non-integer.ll          | 393 +++++++-
 20 files changed, 1330 insertions(+), 597 deletions(-)
 create mode 100644 clang/test/CodeGenCXX/win64-i128.cpp

diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md
index 409d2459b0580b..f1c663c4349226 100644
--- a/clang/docs/ReleaseNotes.md
+++ b/clang/docs/ReleaseNotes.md
@@ -125,6 +125,13 @@ features cannot lower the translation-unit ABI level;
   always passed the parts separately. `-fclang-abi-compat=23` restores the previous
   behavior. (#GH212109)
 
+- On x86_64 Windows targets, Clang now returns `__int128` and `unsigned
+  __int128` indirectly through a hidden return pointer. This matches how Clang
+  already returns 16-byte structs on the same targets. This is ABI-incompatible
+  with objects and runtime libraries using Clang's previous direct XMM return
+  convention. `-fclang-abi-compat=23` restores the previous convention for
+  source-level function calls.
+
 - On MIPS N32/N64, an `__int128` now correctly start in an even-numbered register
   or 16-byte aligned stack slot, matching GCC.
 
diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 5324c8d366b829..e6dcbf1a9a6fe2 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -3673,7 +3673,7 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
 
     case BuiltinType::Int128:
     case BuiltinType::UInt128:
-    case BuiltinType::Float128:
+    case BuiltinType::Float128: {
       // If it's a parameter type, the normal ABI rule is that arguments larger
       // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
       // even though it isn't particularly efficient.
@@ -3682,32 +3682,32 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
             Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
             /*ByVal=*/false);
 
-      // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
-      // Clang matches them for compatibility.
+      // Win64 C returns i128 and f128 via sret. Other calling conventions,
+      // including vectorcall and regcall, return them in XMM registers.
+      auto IsWin64StackCC = [this](unsigned CC) -> bool {
+        switch (CC) {
+        case llvm::CallingConv::Win64:
+          return true;
+        case llvm::CallingConv::C:
+          return getTarget().getTriple().isOSWindowsOrUEFI();
+        default:
+          return false;
+        }
+      };
+
+      if (IsWin64StackCC(CC) &&
+          (BT->getKind() == BuiltinType::Float128 ||
+           getContext().getLangOpts().getClangABICompat() >
+               LangOptions::ClangABI::Ver23))
+        return getNaturalAlignIndirect(
+            Ty, getDataLayout().getAllocaAddrSpace(), /*ByVal=*/false);
+
       if (BT->getKind() == BuiltinType::Int128 ||
           BT->getKind() == BuiltinType::UInt128)
         return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
             llvm::Type::getInt64Ty(getVMContext()), 2));
-
-      // Mingw64 GCC returns f128 via sret, and Clang matches that for
-      // compatibility. This mirrors the X86 backend's CanLowerReturn logic.
-      if (BT->getKind() == BuiltinType::Float128) {
-        auto IsWin64F128StackCC = [this](unsigned CC) -> bool {
-          switch (CC) {
-          case llvm::CallingConv::Win64:
-            return true;
-          case llvm::CallingConv::C:
-            return getTarget().getTriple().isOSWindowsOrUEFI();
-          default:
-            return false;
-          }
-        };
-
-        if (IsWin64F128StackCC(CC))
-          return getNaturalAlignIndirect(
-              Ty, getDataLayout().getAllocaAddrSpace(), /*ByVal=*/false);
-      }
       break;
+    }
 
     default:
       break;
@@ -3766,6 +3766,18 @@ void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
     FI.getReturnInfo() =
         classify(FI.getReturnType(), FreeSSERegs, ClassifyKind::Return, CC);
 
+  const BuiltinType *BT = FI.getReturnType()->getAs<BuiltinType>();
+  bool IsI128OrF128 =
+      BT && (BT->getKind() == BuiltinType::Int128 ||
+             BT->getKind() == BuiltinType::UInt128 ||
+             BT->getKind() == BuiltinType::Float128);
+  if (IsI128OrF128 &&
+      getContext().getLangOpts().getClangABICompat() >
+          LangOptions::ClangABI::Ver23 &&
+      FI.getReturnInfo().isIndirect() && FI.isInstanceMethod() &&
+      getCXXABI().isSRetParameterAfterThis())
+    FI.getReturnInfo().setSRetAfterThis(true);
+
   if (IsVectorCall) {
     // We can use up to 6 SSE register parameters with vectorcall.
     FreeSSERegs = 6;
diff --git a/clang/test/CodeGen/win64-i128.c b/clang/test/CodeGen/win64-i128.c
index ce338d451115c3..8c3e61c74040b8 100644
--- a/clang/test/CodeGen/win64-i128.c
+++ b/clang/test/CodeGen/win64-i128.c
@@ -2,6 +2,11 @@
 // RUN:    | FileCheck %s --check-prefixes=CHECK,X64
 // RUN: %clang_cc1 -triple x86_64-windows-msvc -emit-llvm -o - %s \
 // RUN:    | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -target-feature -sse \
+// RUN:    -emit-llvm -o - %s \
+// RUN:    | FileCheck %s --check-prefixes=CHECK,NOSSE
+// RUN: %clang_cc1 -triple x86_64-windows-gnu -fclang-abi-compat=23 \
+// RUN:    -emit-llvm -o - %s | FileCheck %s --check-prefix=COMPAT23
 // RUN: %clang_cc1 -triple aarch64-windows-msvc -emit-llvm -o - %s \
 // RUN:    | FileCheck %s --check-prefixes=CHECK,ARM,ARM64
 // RUN: %clang_cc1 -triple arm64ec-windows-msvc -emit-llvm -o - %s \
@@ -11,14 +16,50 @@ typedef int int128_t __attribute__((mode(TI)));
 
 int128_t foo(void) { return 0; }
 
-// X64: define dso_local <2 x i64> @foo()
+// X64-LABEL: define dso_local void @foo(
+// X64-SAME: sret(i128) align 16
+// COMPAT23: define dso_local <2 x i64> @foo()
 // ARM: define dso_local i128 @foo()
 
 int128_t bar(int128_t a, int128_t b) { return a * b; }
 
-// X64: define dso_local <2 x i64> @bar(ptr nofreeobj noundef align 16 dead_on_return dereferenceable(16) %0, ptr nofreeobj noundef align 16 dead_on_return dereferenceable(16) %1)
+// X64-LABEL: define dso_local void @bar(
+// X64-SAME: sret(i128) align 16
+// X64-SAME: ptr nofreeobj noundef align 16 dead_on_return
+// X64-SAME: ptr nofreeobj noundef align 16 dead_on_return
+// NOSSE-LABEL: define dso_local void @bar(
+// NOSSE-SAME: sret(i128) align 16
+// NOSSE-SAME: ptr nofreeobj noundef align 16 dead_on_return
+// NOSSE-SAME: ptr nofreeobj noundef align 16 dead_on_return
+// COMPAT23: define dso_local <2 x i64> @bar(ptr nofreeobj noundef align 16 dead_on_return dereferenceable(16) %0, ptr nofreeobj noundef align 16 dead_on_return dereferenceable(16) %1)
 // ARM: define dso_local i128 @bar(i128 noundef %a, i128 noundef %b)
 
+#if defined(__x86_64__) && !defined(__arm64ec__)
+int128_t __attribute__((vectorcall)) vectorcall_foo(void) { return 0; }
+
+// X64-LABEL: define dso_local x86_vectorcallcc <2 x i64> @"\01vectorcall_foo@@0"()
+
+int128_t __attribute__((vectorcall)) vectorcall_bar(int128_t a, int128_t b) {
+  return a * b;
+}
+
+// X64-LABEL: define dso_local x86_vectorcallcc <2 x i64> @"\01vectorcall_bar@@32"(
+// X64-SAME: ptr nofreeobj noundef align 16 dead_on_return
+// X64-SAME: ptr nofreeobj noundef align 16 dead_on_return
+
+int128_t __attribute__((regcall)) regcall_foo(void) { return 0; }
+
+// X64-LABEL: define dso_local x86_regcallcc <2 x i64> @__regcall3__regcall_foo()
+
+int128_t __attribute__((regcall)) regcall_bar(int128_t a, int128_t b) {
+  return a * b;
+}
+
+// X64-LABEL: define dso_local x86_regcallcc <2 x i64> @__regcall3__regcall_bar(
+// X64-SAME: ptr nofreeobj noundef align 16 dead_on_return
+// X64-SAME: ptr nofreeobj noundef align 16 dead_on_return
+#endif
+
 void vararg(int a, ...) {
   // CHECK: define{{.*}} void @vararg
   __builtin_va_list ap;
diff --git a/clang/test/CodeGenCXX/win64-i128.cpp b/clang/test/CodeGenCXX/win64-i128.cpp
new file mode 100644
index 00000000000000..08ffd21bccefcd
--- /dev/null
+++ b/clang/test/CodeGenCXX/win64-i128.cpp
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -triple x86_64-windows-gnu -emit-llvm -o - %s \
+// RUN:   | FileCheck %s --check-prefix=GNU
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -emit-llvm -o - %s \
+// RUN:   | FileCheck %s --check-prefixes=MSVC,BITINT
+// RUN: %clang_cc1 -triple x86_64-windows-gnu -fclang-abi-compat=23 \
+// RUN:   -emit-llvm -o - %s | FileCheck %s --check-prefix=COMPAT23-GNU
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -fclang-abi-compat=23 \
+// RUN:   -emit-llvm -o - %s | FileCheck %s --check-prefixes=COMPAT23-MSVC,BITINT
+
+typedef int int128_t __attribute__((mode(TI)));
+
+struct S {
+  int128_t f();
+};
+
+int128_t S::f() { return 0; }
+
+// GNU-LABEL: define dso_local void @_ZN1S1fEv(
+// GNU-SAME: ptr dead_on_unwind noalias writable sret(i128) align 16 %agg.result,
+// GNU-SAME: ptr noundef nonnull align 1 dereferenceable(1) %this)
+// MSVC-LABEL: define dso_local void @"?f at S@@QEAA_LXZ"(
+// MSVC-SAME: ptr noundef nonnull align 1 dereferenceable(1) %this,
+// MSVC-SAME: ptr dead_on_unwind noalias writable sret(i128) align 16 %agg.result)
+// COMPAT23-GNU-LABEL: define dso_local{{.*}} <2 x i64> @_ZN1S1fEv(
+// COMPAT23-GNU-SAME: ptr noundef nonnull align 1 dereferenceable(1) %this)
+// COMPAT23-MSVC-LABEL: define dso_local{{.*}} <2 x i64> @"?f at S@@QEAA_LXZ"(
+// COMPAT23-MSVC-SAME: ptr noundef nonnull align 1 dereferenceable(1) %this)
+
+struct BitInt {
+  _BitInt(128) f();
+};
+
+_BitInt(128) BitInt::f() { return 0; }
+
+// BITINT-LABEL: define dso_local void @"?f at BitInt
+// BITINT-SAME: ptr dead_on_unwind noalias writable sret(i128) align 8 %agg.result,
+// BITINT-SAME: ptr noundef nonnull align 1 dereferenceable(1) %this)
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 0134c3f672d499..8dc0e6910f3b62 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -2255,16 +2255,9 @@ bool AtomicExpandImpl::expandAtomicOpToLibcall(
 
   bool UseSizedLibcall = canUseSizedAtomicCall(Size, Alignment, DL);
   Type *SizedIntTy = Type::getIntNTy(Ctx, Size * 8);
-
-  if (M->getTargetTriple().isOSWindows() && M->getTargetTriple().isX86_64() &&
-      Size == 16) {
-    // x86_64 Windows passes i128 as an XMM vector; on return, it is in
-    // XMM0, and as a parameter, it is passed indirectly. The generic lowering
-    // rules handles this correctly if we pass it as a v2i64 rather than
-    // i128. This is what Clang does in the frontend for such types as well
-    // (see WinX86_64ABIInfo::classify in Clang).
-    SizedIntTy = FixedVectorType::get(Type::getInt64Ty(Ctx), 2);
-  }
+  bool PassI128Indirectly =
+      UseSizedLibcall && Size == 16 && M->getTargetTriple().isX86_64() &&
+      M->getTargetTriple().isOSWindowsOrUEFI();
 
   const Align AllocaAlignment = DL.getPrefTypeAlign(SizedIntTy);
 
@@ -2377,7 +2370,18 @@ bool AtomicExpandImpl::expandAtomicOpToLibcall(
     if (UseSizedLibcall) {
       Value *IntValue =
           Builder.CreateBitPreservingCastChain(DL, ValueOperand, SizedIntTy);
-      Args.push_back(IntValue);
+      if (PassI128Indirectly) {
+        // Win64 passes i128 arguments indirectly. Materialize that ABI here
+        // because SelectionDAG cannot recover the original i128 after it has
+        // been split for calling convention analysis.
+        AllocaValue = AllocaBuilder.CreateAlloca(SizedIntTy);
+        AllocaValue->setAlignment(AllocaAlignment);
+        Builder.CreateLifetimeStart(AllocaValue);
+        Builder.CreateAlignedStore(IntValue, AllocaValue, AllocaAlignment);
+        Args.push_back(AllocaValue);
+      } else {
+        Args.push_back(IntValue);
+      }
     } else {
       AllocaValue = AllocaBuilder.CreateAlloca(ValueOperand->getType());
       AllocaValue->setAlignment(AllocaAlignment);
@@ -2433,7 +2437,7 @@ bool AtomicExpandImpl::expandAtomicOpToLibcall(
   Value *Result = Call;
 
   // And then, extract the results...
-  if (ValueOperand && !UseSizedLibcall)
+  if (AllocaValue)
     Builder.CreateLifetimeEnd(AllocaValue);
 
   if (CASExpected) {
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 637e96308270d0..0ac423c98ddef4 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2741,7 +2741,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
   setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
 
-  // The Win64 C ABI passes i128 by reference and returns it in XMM0. The
+  // The Win64 C ABI passes i128 by reference and returns it via sret. The
   // libcalls for i128 division, remainder and floating point conversions are
   // compiled with that ABI wherever it is the default C convention (Windows and
   // UEFI), so lower them accordingly there.
@@ -22478,12 +22478,14 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
 
   // fp128 needs to use a libcall.
   if (SrcVT == MVT::f128) {
+    if (Subtarget.isCallingConvWin64(CallingConv::C) && VT == MVT::i128)
+      return LowerWin64_FP_TO_INT128(Op, DAG, Chain);
+
     RTLIB::Libcall LC;
     if (IsSigned)
       LC = RTLIB::getFPTOSINT(SrcVT, VT);
     else
       LC = RTLIB::getFPTOUINT(SrcVT, VT);
-
     MakeLibCallOptions CallOptions;
     std::pair<SDValue, SDValue> Tmp =
         makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
@@ -30977,7 +30979,8 @@ static SDValue LowerMULO(SDValue Op, const X86Subtarget &Subtarget,
   return DAG.getMergeValues({Low, Ovf}, dl);
 }
 
-SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op,
+                                             SelectionDAG &DAG) const {
   assert(Subtarget.isCallingConvWin64(CallingConv::C) && "Unexpected target");
   EVT VT = Op.getValueType();
   assert(VT.isInteger() && VT.getSizeInBits() == 128 &&
@@ -30990,14 +30993,13 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) cons
   }
 
   RTLIB::Libcall LC;
-  bool isSigned;
   switch (Op->getOpcode()) {
   // clang-format off
   default: llvm_unreachable("Unexpected request for libcall!");
-  case ISD::SDIV:      isSigned = true;  LC = RTLIB::SDIV_I128;    break;
-  case ISD::UDIV:      isSigned = false; LC = RTLIB::UDIV_I128;    break;
-  case ISD::SREM:      isSigned = true;  LC = RTLIB::SREM_I128;    break;
-  case ISD::UREM:      isSigned = false; LC = RTLIB::UREM_I128;    break;
+  case ISD::SDIV:      LC = RTLIB::SDIV_I128; break;
+  case ISD::UDIV:      LC = RTLIB::UDIV_I128; break;
+  case ISD::SREM:      LC = RTLIB::SREM_I128; break;
+  case ISD::UREM:      LC = RTLIB::UREM_I128; break;
   // clang-format on
   }
 
@@ -31015,7 +31017,10 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) cons
         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
     InChain =
         DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, MPI, Align(16));
-    Args.emplace_back(StackPtr, PointerType::get(*DAG.getContext(), 0));
+    Args.emplace_back(
+        StackPtr,
+        PointerType::get(*DAG.getContext(),
+                         DAG.getDataLayout().getAllocaAddrSpace()));
   }
 
   RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
@@ -31029,14 +31034,11 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) cons
   CLI.setDebugLoc(dl)
       .setChain(InChain)
       .setLibCallee(DAG.getLibcalls().getLibcallImplCallingConv(LCImpl),
-                    EVT(MVT::v2i64).getTypeForEVT(*DAG.getContext()), Callee,
-                    std::move(Args))
-      .setInRegister()
-      .setSExtResult(isSigned)
-      .setZExtResult(!isSigned);
+                    VT.getTypeForEVT(*DAG.getContext()), Callee,
+                    std::move(Args));
 
   std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
-  return DAG.getBitcast(VT, CallInfo.first);
+  return CallInfo.first;
 }
 
 SDValue X86TargetLowering::LowerWin64_FP_TO_INT128(SDValue Op,
@@ -31065,11 +31067,8 @@ SDValue X86TargetLowering::LowerWin64_FP_TO_INT128(SDValue Op,
   Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
 
   SDValue Result;
-  // Expect the i128 argument returned as a v2i64 in xmm0, cast back to the
-  // expected VT (i128).
   std::tie(Result, Chain) =
-      makeLibCall(DAG, LC, MVT::v2i64, Arg, CallOptions, dl, Chain);
-  Result = DAG.getBitcast(VT, Result);
+      makeLibCall(DAG, LC, VT, Arg, CallOptions, dl, Chain);
   return Result;
 }
 
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index a78621fd897b35..df60644caec790 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -671,9 +671,9 @@ bool X86TargetLowering::CanLowerReturn(
     CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
     const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
     const Type *RetTy) const {
-  // Mingw64 GCC returns f128 via sret, and LLVM matches it for compatibility.
-  // This logic exists for libcalls, a frontend should explicitly use sret
-  // rather than rely on the sret demotion here.
+  // Mingw64 GCC returns f128 and i128 via sret, and LLVM matches it for
+  // compatibility. This logic exists for libcalls, a frontend should
+  // explicitly use sret rather than rely on the sret demotion here.
   //
   // Using sret is a reasonable implementation of the Windows x64 calling
   // convention:
@@ -692,7 +692,7 @@ bool X86TargetLowering::CanLowerReturn(
   // published, this can be revisited.
   //
   // Return false, which will perform sret demotion.
-  auto IsWin64F128StackCC = [this](CallingConv::ID CC) -> bool {
+  auto IsWin64StackCC = [this](CallingConv::ID CC) -> bool {
     switch (CC) {
     case CallingConv::Win64:
       return true;
@@ -703,9 +703,11 @@ bool X86TargetLowering::CanLowerReturn(
     }
   };
 
-  if (IsWin64F128StackCC(CallConv) &&
-      llvm::any_of(
-          Outs, [](const ISD::OutputArg &Out) { return Out.VT == MVT::f128; }))
+  // i128 is legalized into i64 parts, so inspect ArgVT rather than VT.
+  if (IsWin64StackCC(CallConv) &&
+      llvm::any_of(Outs, [](const ISD::OutputArg &Out) {
+        return Out.VT == MVT::f128 || Out.ArgVT == MVT::i128;
+      }))
     return false;
 
   SmallVector<CCValAssign, 16> RVLocs;
diff --git a/llvm/test/CodeGen/X86/atomic-nocx16-win.ll b/llvm/test/CodeGen/X86/atomic-nocx16-win.ll
index 3f0649cd9ae07e..c2c8cec224966f 100644
--- a/llvm/test/CodeGen/X86/atomic-nocx16-win.ll
+++ b/llvm/test/CodeGen/X86/atomic-nocx16-win.ll
@@ -5,13 +5,14 @@ define void @call_load(ptr %dst, ptr %src) nounwind {
 ; CHECK-LABEL: call_load:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rsi
-; CHECK-NEXT:    subq $32, %rsp
+; CHECK-NEXT:    subq $48, %rsp
 ; CHECK-NEXT:    movq %rcx, %rsi
-; CHECK-NEXT:    movq %rdx, %rcx
-; CHECK-NEXT:    movl $5, %edx
+; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT:    movl $5, %r8d
 ; CHECK-NEXT:    callq __atomic_load_16
+; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm0
 ; CHECK-NEXT:    movaps %xmm0, (%rsi)
-; CHECK-NEXT:    addq $32, %rsp
+; CHECK-NEXT:    addq $48, %rsp
 ; CHECK-NEXT:    popq %rsi
 ; CHECK-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/divmod128.ll b/llvm/test/CodeGen/X86/divmod128.ll
index e298e221af2312..c8c2229a37a314 100644
--- a/llvm/test/CodeGen/X86/divmod128.ll
+++ b/llvm/test/CodeGen/X86/divmod128.ll
@@ -17,16 +17,17 @@ define i64 @mod128(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: mod128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $72, %rsp
+; WIN64-NEXT:    subq $88, %rsp
 ; WIN64-NEXT:    movq %rdx, {{[0-9]+}}(%rsp)
 ; WIN64-NEXT:    movq %rcx, {{[0-9]+}}(%rsp)
 ; WIN64-NEXT:    movq $3, {{[0-9]+}}(%rsp)
 ; WIN64-NEXT:    movq $0, {{[0-9]+}}(%rsp)
 ; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
 ; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
 ; WIN64-NEXT:    callq __modti3
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $72, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $88, %rsp
 ; WIN64-NEXT:    retq
 
 
@@ -47,16 +48,17 @@ define i64 @div128(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: div128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $72, %rsp
+; WIN64-NEXT:    subq $88, %rsp
 ; WIN64-NEXT:    movq %rdx, {{[0-9]+}}(%rsp)
 ; WIN64-NEXT:    movq %rcx, {{[0-9]+}}(%rsp)
 ; WIN64-NEXT:    movq $3, {{[0-9]+}}(%rsp)
 ; WIN64-NEXT:    movq $0, {{[0-9]+}}(%rsp)
 ; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
 ; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
 ; WIN64-NEXT:    callq __divti3
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $72, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $88, %rsp
 ; WIN64-NEXT:    retq
 
 
@@ -165,16 +167,18 @@ define i128 @urem_i128_3(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: urem_i128_3:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    addq %rdx, %rcx
-; WIN64-NEXT:    adcq $0, %rcx
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    addq %r8, %r9
+; WIN64-NEXT:    adcq $0, %r9
 ; WIN64-NEXT:    movabsq $-6148914691236517205, %rdx # imm = 0xAAAAAAAAAAAAAAAB
-; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    movq %r9, %rax
 ; WIN64-NEXT:    mulq %rdx
 ; WIN64-NEXT:    shrq %rdx
 ; WIN64-NEXT:    leaq (%rdx,%rdx,2), %rax
-; WIN64-NEXT:    subq %rax, %rcx
+; WIN64-NEXT:    subq %rax, %r9
+; WIN64-NEXT:    movq %r9, (%rcx)
+; WIN64-NEXT:    movq $0, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    xorl %edx, %edx
 ; WIN64-NEXT:    retq
 entry:
   %rem = urem i128 %x, 3
@@ -198,16 +202,18 @@ define i128 @urem_i128_5(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: urem_i128_5:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    addq %rdx, %rcx
-; WIN64-NEXT:    adcq $0, %rcx
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    addq %r8, %r9
+; WIN64-NEXT:    adcq $0, %r9
 ; WIN64-NEXT:    movabsq $-3689348814741910323, %rdx # imm = 0xCCCCCCCCCCCCCCCD
-; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    movq %r9, %rax
 ; WIN64-NEXT:    mulq %rdx
 ; WIN64-NEXT:    shrq $2, %rdx
 ; WIN64-NEXT:    leaq (%rdx,%rdx,4), %rax
-; WIN64-NEXT:    subq %rax, %rcx
+; WIN64-NEXT:    subq %rax, %r9
+; WIN64-NEXT:    movq %r9, (%rcx)
+; WIN64-NEXT:    movq $0, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    xorl %edx, %edx
 ; WIN64-NEXT:    retq
 entry:
   %rem = urem i128 %x, 5
@@ -232,17 +238,19 @@ define i128 @urem_i128_15(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: urem_i128_15:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    addq %rdx, %rcx
-; WIN64-NEXT:    adcq $0, %rcx
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    addq %r8, %r9
+; WIN64-NEXT:    adcq $0, %r9
 ; WIN64-NEXT:    movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889
-; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    movq %r9, %rax
 ; WIN64-NEXT:    mulq %rdx
 ; WIN64-NEXT:    shrq $3, %rdx
 ; WIN64-NEXT:    leaq (%rdx,%rdx,4), %rax
 ; WIN64-NEXT:    leaq (%rax,%rax,2), %rax
-; WIN64-NEXT:    subq %rax, %rcx
+; WIN64-NEXT:    subq %rax, %r9
+; WIN64-NEXT:    movq %r9, (%rcx)
+; WIN64-NEXT:    movq $0, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    xorl %edx, %edx
 ; WIN64-NEXT:    retq
 entry:
   %rem = urem i128 %x, 15
@@ -268,18 +276,20 @@ define i128 @urem_i128_17(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: urem_i128_17:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    addq %rdx, %rcx
-; WIN64-NEXT:    adcq $0, %rcx
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    addq %r8, %r9
+; WIN64-NEXT:    adcq $0, %r9
 ; WIN64-NEXT:    movabsq $-1085102592571150095, %rdx # imm = 0xF0F0F0F0F0F0F0F1
-; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    movq %r9, %rax
 ; WIN64-NEXT:    mulq %rdx
 ; WIN64-NEXT:    movq %rdx, %rax
 ; WIN64-NEXT:    andq $-16, %rax
 ; WIN64-NEXT:    shrq $4, %rdx
 ; WIN64-NEXT:    addq %rax, %rdx
-; WIN64-NEXT:    subq %rdx, %rcx
+; WIN64-NEXT:    subq %rdx, %r9
+; WIN64-NEXT:    movq %r9, (%rcx)
+; WIN64-NEXT:    movq $0, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    xorl %edx, %edx
 ; WIN64-NEXT:    retq
 entry:
   %rem = urem i128 %x, 17
@@ -306,9 +316,9 @@ define i128 @urem_i128_255(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: urem_i128_255:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    addq %rdx, %rax
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq %rdx, %rax
+; WIN64-NEXT:    addq %r8, %rax
 ; WIN64-NEXT:    adcq $0, %rax
 ; WIN64-NEXT:    movabsq $-9187201950435737471, %rdx # imm = 0x8080808080808081
 ; WIN64-NEXT:    mulq %rdx
@@ -316,10 +326,11 @@ define i128 @urem_i128_255(i128 %x) nounwind {
 ; WIN64-NEXT:    movq %rdx, %rax
 ; WIN64-NEXT:    shlq $8, %rax
 ; WIN64-NEXT:    subq %rax, %rdx
-; WIN64-NEXT:    addq %rcx, %r8
-; WIN64-NEXT:    adcq %rdx, %r8
-; WIN64-NEXT:    movq %r8, %rax
-; WIN64-NEXT:    xorl %edx, %edx
+; WIN64-NEXT:    addq %r8, %r9
+; WIN64-NEXT:    adcq %rdx, %r9
+; WIN64-NEXT:    movq %r9, (%rcx)
+; WIN64-NEXT:    movq $0, 8(%rcx)
+; WIN64-NEXT:    movq %rcx, %rax
 ; WIN64-NEXT:    retq
 entry:
   %rem = urem i128 %x, 255
@@ -345,18 +356,20 @@ define i128 @urem_i128_257(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: urem_i128_257:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    addq %rdx, %rcx
-; WIN64-NEXT:    adcq $0, %rcx
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    addq %r8, %r9
+; WIN64-NEXT:    adcq $0, %r9
 ; WIN64-NEXT:    movabsq $-71777214294589695, %rdx # imm = 0xFF00FF00FF00FF01
-; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    movq %r9, %rax
 ; WIN64-NEXT:    mulq %rdx
 ; WIN64-NEXT:    movq %rdx, %rax
 ; WIN64-NEXT:    andq $-256, %rax
 ; WIN64-NEXT:    shrq $8, %rdx
 ; WIN64-NEXT:    addq %rax, %rdx
-; WIN64-NEXT:    subq %rdx, %rcx
+; WIN64-NEXT:    subq %rdx, %r9
+; WIN64-NEXT:    movq %r9, (%rcx)
+; WIN64-NEXT:    movq $0, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    xorl %edx, %edx
 ; WIN64-NEXT:    retq
 entry:
   %rem = urem i128 %x, 257
@@ -383,9 +396,9 @@ define i128 @urem_i128_65535(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: urem_i128_65535:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    addq %rdx, %rax
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq %rdx, %rax
+; WIN64-NEXT:    addq %r8, %rax
 ; WIN64-NEXT:    adcq $0, %rax
 ; WIN64-NEXT:    movabsq $-9223231297218904063, %rdx # imm = 0x8000800080008001
 ; WIN64-NEXT:    mulq %rdx
@@ -393,10 +406,11 @@ define i128 @urem_i128_65535(i128 %x) nounwind {
 ; WIN64-NEXT:    movq %rdx, %rax
 ; WIN64-NEXT:    shlq $16, %rax
 ; WIN64-NEXT:    subq %rax, %rdx
-; WIN64-NEXT:    addq %rcx, %r8
-; WIN64-NEXT:    adcq %rdx, %r8
-; WIN64-NEXT:    movq %r8, %rax
-; WIN64-NEXT:    xorl %edx, %edx
+; WIN64-NEXT:    addq %r8, %r9
+; WIN64-NEXT:    adcq %rdx, %r9
+; WIN64-NEXT:    movq %r9, (%rcx)
+; WIN64-NEXT:    movq $0, 8(%rcx)
+; WIN64-NEXT:    movq %rcx, %rax
 ; WIN64-NEXT:    retq
 entry:
   %rem = urem i128 %x, 65535
@@ -422,18 +436,20 @@ define i128 @urem_i128_65537(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: urem_i128_65537:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    addq %rdx, %rcx
-; WIN64-NEXT:    adcq $0, %rcx
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    addq %r8, %r9
+; WIN64-NEXT:    adcq $0, %r9
 ; WIN64-NEXT:    movabsq $-281470681808895, %rdx # imm = 0xFFFF0000FFFF0001
-; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    movq %r9, %rax
 ; WIN64-NEXT:    mulq %rdx
 ; WIN64-NEXT:    movq %rdx, %rax
 ; WIN64-NEXT:    andq $-65536, %rax # imm = 0xFFFF0000
 ; WIN64-NEXT:    shrq $16, %rdx
 ; WIN64-NEXT:    addq %rax, %rdx
-; WIN64-NEXT:    subq %rdx, %rcx
+; WIN64-NEXT:    subq %rdx, %r9
+; WIN64-NEXT:    movq %r9, (%rcx)
+; WIN64-NEXT:    movq $0, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    xorl %edx, %edx
 ; WIN64-NEXT:    retq
 entry:
   %rem = urem i128 %x, 65537
@@ -461,20 +477,23 @@ define i128 @urem_i128_12(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: urem_i128_12:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rcx, %r8
-; WIN64-NEXT:    shrdq $2, %rdx, %r8
-; WIN64-NEXT:    shrq $2, %rdx
-; WIN64-NEXT:    addq %rdx, %r8
-; WIN64-NEXT:    adcq $0, %r8
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    shrdq $2, %r8, %r10
+; WIN64-NEXT:    shrq $2, %r8
+; WIN64-NEXT:    addq %r8, %r10
+; WIN64-NEXT:    adcq $0, %r10
 ; WIN64-NEXT:    movabsq $-6148914691236517205, %rdx # imm = 0xAAAAAAAAAAAAAAAB
-; WIN64-NEXT:    movq %r8, %rax
+; WIN64-NEXT:    movq %r10, %rax
 ; WIN64-NEXT:    mulq %rdx
 ; WIN64-NEXT:    shrq %rdx
 ; WIN64-NEXT:    leal (%rdx,%rdx,2), %eax
-; WIN64-NEXT:    subl %eax, %r8d
-; WIN64-NEXT:    andl $3, %ecx
-; WIN64-NEXT:    leaq (%rcx,%r8,4), %rax
-; WIN64-NEXT:    xorl %edx, %edx
+; WIN64-NEXT:    subl %eax, %r10d
+; WIN64-NEXT:    andl $3, %r9d
+; WIN64-NEXT:    leaq (%r9,%r10,4), %rax
+; WIN64-NEXT:    movq %rax, (%rcx)
+; WIN64-NEXT:    movq $0, 8(%rcx)
+; WIN64-NEXT:    movq %rcx, %rax
 ; WIN64-NEXT:    retq
 entry:
   %rem = urem i128 %x, 12
@@ -506,25 +525,28 @@ define i128 @udiv_i128_3(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: udiv_i128_3:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq %rcx, %r9
-; WIN64-NEXT:    addq %rdx, %r9
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    movabsq $-6148914691236517205, %r10 # imm = 0xAAAAAAAAAAAAAAAB
-; WIN64-NEXT:    movq %r9, %rax
-; WIN64-NEXT:    mulq %r10
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    addq %r8, %r10
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    movabsq $-6148914691236517205, %r11 # imm = 0xAAAAAAAAAAAAAAAB
+; WIN64-NEXT:    movq %r10, %rax
+; WIN64-NEXT:    mulq %r11
 ; WIN64-NEXT:    shrq %rdx
 ; WIN64-NEXT:    leaq (%rdx,%rdx,2), %rax
-; WIN64-NEXT:    subq %rax, %r9
-; WIN64-NEXT:    subq %r9, %rcx
+; WIN64-NEXT:    subq %rax, %r10
+; WIN64-NEXT:    subq %r10, %r9
 ; WIN64-NEXT:    sbbq $0, %r8
-; WIN64-NEXT:    movabsq $-6148914691236517206, %r9 # imm = 0xAAAAAAAAAAAAAAAA
-; WIN64-NEXT:    imulq %rcx, %r9
+; WIN64-NEXT:    movabsq $-6148914691236517206, %r10 # imm = 0xAAAAAAAAAAAAAAAA
+; WIN64-NEXT:    imulq %r9, %r10
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    addq %r10, %rdx
+; WIN64-NEXT:    imulq %r11, %r8
+; WIN64-NEXT:    addq %rdx, %r8
+; WIN64-NEXT:    movq %rax, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %r10
-; WIN64-NEXT:    addq %r9, %rdx
-; WIN64-NEXT:    imulq %r10, %r8
-; WIN64-NEXT:    addq %r8, %rdx
 ; WIN64-NEXT:    retq
 entry:
   %rem = udiv i128 %x, 3
@@ -556,25 +578,28 @@ define i128 @udiv_i128_5(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: udiv_i128_5:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq %rcx, %r9
-; WIN64-NEXT:    addq %rdx, %r9
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    movabsq $-3689348814741910323, %r10 # imm = 0xCCCCCCCCCCCCCCCD
-; WIN64-NEXT:    movq %r9, %rax
-; WIN64-NEXT:    mulq %r10
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    addq %r8, %r10
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    movabsq $-3689348814741910323, %r11 # imm = 0xCCCCCCCCCCCCCCCD
+; WIN64-NEXT:    movq %r10, %rax
+; WIN64-NEXT:    mulq %r11
 ; WIN64-NEXT:    shrq $2, %rdx
 ; WIN64-NEXT:    leaq (%rdx,%rdx,4), %rax
-; WIN64-NEXT:    subq %rax, %r9
-; WIN64-NEXT:    subq %r9, %rcx
+; WIN64-NEXT:    subq %rax, %r10
+; WIN64-NEXT:    subq %r10, %r9
 ; WIN64-NEXT:    sbbq $0, %r8
-; WIN64-NEXT:    movabsq $-3689348814741910324, %r9 # imm = 0xCCCCCCCCCCCCCCCC
-; WIN64-NEXT:    imulq %rcx, %r9
+; WIN64-NEXT:    movabsq $-3689348814741910324, %r10 # imm = 0xCCCCCCCCCCCCCCCC
+; WIN64-NEXT:    imulq %r9, %r10
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    addq %r10, %rdx
+; WIN64-NEXT:    imulq %r11, %r8
+; WIN64-NEXT:    addq %rdx, %r8
+; WIN64-NEXT:    movq %rax, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %r10
-; WIN64-NEXT:    addq %r9, %rdx
-; WIN64-NEXT:    imulq %r10, %r8
-; WIN64-NEXT:    addq %r8, %rdx
 ; WIN64-NEXT:    retq
 entry:
   %rem = udiv i128 %x, 5
@@ -608,27 +633,30 @@ define i128 @udiv_i128_15(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: udiv_i128_15:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq %rcx, %r9
-; WIN64-NEXT:    addq %rdx, %r9
-; WIN64-NEXT:    adcq $0, %r9
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    addq %r8, %r10
+; WIN64-NEXT:    adcq $0, %r10
 ; WIN64-NEXT:    movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889
-; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    movq %r10, %rax
 ; WIN64-NEXT:    mulq %rdx
 ; WIN64-NEXT:    shrq $3, %rdx
 ; WIN64-NEXT:    leaq (%rdx,%rdx,4), %rax
 ; WIN64-NEXT:    leaq (%rax,%rax,2), %rax
-; WIN64-NEXT:    subq %rax, %r9
-; WIN64-NEXT:    subq %r9, %rcx
+; WIN64-NEXT:    subq %rax, %r10
+; WIN64-NEXT:    subq %r10, %r9
 ; WIN64-NEXT:    sbbq $0, %r8
-; WIN64-NEXT:    movabsq $-1229782938247303442, %r9 # imm = 0xEEEEEEEEEEEEEEEE
-; WIN64-NEXT:    imulq %rcx, %r9
-; WIN64-NEXT:    movabsq $-1229782938247303441, %r10 # imm = 0xEEEEEEEEEEEEEEEF
+; WIN64-NEXT:    movabsq $-1229782938247303442, %r10 # imm = 0xEEEEEEEEEEEEEEEE
+; WIN64-NEXT:    imulq %r9, %r10
+; WIN64-NEXT:    movabsq $-1229782938247303441, %r11 # imm = 0xEEEEEEEEEEEEEEEF
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    addq %r10, %rdx
+; WIN64-NEXT:    imulq %r11, %r8
+; WIN64-NEXT:    addq %rdx, %r8
+; WIN64-NEXT:    movq %rax, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %r10
-; WIN64-NEXT:    addq %r9, %rdx
-; WIN64-NEXT:    imulq %r10, %r8
-; WIN64-NEXT:    addq %r8, %rdx
 ; WIN64-NEXT:    retq
 entry:
   %rem = udiv i128 %x, 15
@@ -662,27 +690,30 @@ define i128 @udiv_i128_17(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: udiv_i128_17:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq %rcx, %r9
-; WIN64-NEXT:    addq %rdx, %r9
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    movabsq $-1085102592571150095, %r10 # imm = 0xF0F0F0F0F0F0F0F1
-; WIN64-NEXT:    movq %r9, %rax
-; WIN64-NEXT:    mulq %r10
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    addq %r8, %r10
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    movabsq $-1085102592571150095, %r11 # imm = 0xF0F0F0F0F0F0F0F1
+; WIN64-NEXT:    movq %r10, %rax
+; WIN64-NEXT:    mulq %r11
 ; WIN64-NEXT:    movq %rdx, %rax
 ; WIN64-NEXT:    andq $-16, %rax
 ; WIN64-NEXT:    shrq $4, %rdx
 ; WIN64-NEXT:    addq %rax, %rdx
-; WIN64-NEXT:    subq %rdx, %r9
-; WIN64-NEXT:    subq %r9, %rcx
+; WIN64-NEXT:    subq %rdx, %r10
+; WIN64-NEXT:    subq %r10, %r9
 ; WIN64-NEXT:    sbbq $0, %r8
-; WIN64-NEXT:    movabsq $-1085102592571150096, %r9 # imm = 0xF0F0F0F0F0F0F0F0
-; WIN64-NEXT:    imulq %rcx, %r9
+; WIN64-NEXT:    movabsq $-1085102592571150096, %r10 # imm = 0xF0F0F0F0F0F0F0F0
+; WIN64-NEXT:    imulq %r9, %r10
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    addq %r10, %rdx
+; WIN64-NEXT:    imulq %r11, %r8
+; WIN64-NEXT:    addq %rdx, %r8
+; WIN64-NEXT:    movq %rax, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %r10
-; WIN64-NEXT:    addq %r9, %rdx
-; WIN64-NEXT:    imulq %r10, %r8
-; WIN64-NEXT:    addq %r8, %rdx
 ; WIN64-NEXT:    retq
 entry:
   %rem = udiv i128 %x, 17
@@ -718,9 +749,9 @@ define i128 @udiv_i128_255(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: udiv_i128_255:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    addq %rdx, %rax
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq %rdx, %rax
+; WIN64-NEXT:    addq %r8, %rax
 ; WIN64-NEXT:    adcq $0, %rax
 ; WIN64-NEXT:    movabsq $-9187201950435737471, %rdx # imm = 0x8080808080808081
 ; WIN64-NEXT:    mulq %rdx
@@ -728,19 +759,22 @@ define i128 @udiv_i128_255(i128 %x) nounwind {
 ; WIN64-NEXT:    movq %rdx, %rax
 ; WIN64-NEXT:    shlq $8, %rax
 ; WIN64-NEXT:    subq %rax, %rdx
-; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    movq %r9, %rax
 ; WIN64-NEXT:    addq %r8, %rax
 ; WIN64-NEXT:    adcq %rdx, %rax
-; WIN64-NEXT:    subq %rax, %rcx
+; WIN64-NEXT:    subq %rax, %r9
 ; WIN64-NEXT:    sbbq $0, %r8
-; WIN64-NEXT:    movabsq $-72340172838076674, %r9 # imm = 0xFEFEFEFEFEFEFEFE
-; WIN64-NEXT:    imulq %rcx, %r9
-; WIN64-NEXT:    movabsq $-72340172838076673, %r10 # imm = 0xFEFEFEFEFEFEFEFF
+; WIN64-NEXT:    movabsq $-72340172838076674, %r10 # imm = 0xFEFEFEFEFEFEFEFE
+; WIN64-NEXT:    imulq %r9, %r10
+; WIN64-NEXT:    movabsq $-72340172838076673, %r11 # imm = 0xFEFEFEFEFEFEFEFF
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    addq %r10, %rdx
+; WIN64-NEXT:    imulq %r11, %r8
+; WIN64-NEXT:    addq %rdx, %r8
+; WIN64-NEXT:    movq %rax, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %r10
-; WIN64-NEXT:    addq %r9, %rdx
-; WIN64-NEXT:    imulq %r10, %r8
-; WIN64-NEXT:    addq %r8, %rdx
 ; WIN64-NEXT:    retq
 entry:
   %rem = udiv i128 %x, 255
@@ -774,27 +808,30 @@ define i128 @udiv_i128_257(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: udiv_i128_257:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq %rcx, %r9
-; WIN64-NEXT:    addq %rdx, %r9
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    movabsq $-71777214294589695, %r10 # imm = 0xFF00FF00FF00FF01
-; WIN64-NEXT:    movq %r9, %rax
-; WIN64-NEXT:    mulq %r10
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    addq %r8, %r10
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    movabsq $-71777214294589695, %r11 # imm = 0xFF00FF00FF00FF01
+; WIN64-NEXT:    movq %r10, %rax
+; WIN64-NEXT:    mulq %r11
 ; WIN64-NEXT:    movq %rdx, %rax
 ; WIN64-NEXT:    andq $-256, %rax
 ; WIN64-NEXT:    shrq $8, %rdx
 ; WIN64-NEXT:    addq %rax, %rdx
-; WIN64-NEXT:    subq %rdx, %r9
-; WIN64-NEXT:    subq %r9, %rcx
+; WIN64-NEXT:    subq %rdx, %r10
+; WIN64-NEXT:    subq %r10, %r9
 ; WIN64-NEXT:    sbbq $0, %r8
-; WIN64-NEXT:    movabsq $-71777214294589696, %r9 # imm = 0xFF00FF00FF00FF00
-; WIN64-NEXT:    imulq %rcx, %r9
+; WIN64-NEXT:    movabsq $-71777214294589696, %r10 # imm = 0xFF00FF00FF00FF00
+; WIN64-NEXT:    imulq %r9, %r10
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    addq %r10, %rdx
+; WIN64-NEXT:    imulq %r11, %r8
+; WIN64-NEXT:    addq %rdx, %r8
+; WIN64-NEXT:    movq %rax, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %r10
-; WIN64-NEXT:    addq %r9, %rdx
-; WIN64-NEXT:    imulq %r10, %r8
-; WIN64-NEXT:    addq %r8, %rdx
 ; WIN64-NEXT:    retq
 entry:
   %rem = udiv i128 %x, 257
@@ -830,9 +867,9 @@ define i128 @udiv_i128_65535(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: udiv_i128_65535:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    addq %rdx, %rax
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq %rdx, %rax
+; WIN64-NEXT:    addq %r8, %rax
 ; WIN64-NEXT:    adcq $0, %rax
 ; WIN64-NEXT:    movabsq $-9223231297218904063, %rdx # imm = 0x8000800080008001
 ; WIN64-NEXT:    mulq %rdx
@@ -840,19 +877,22 @@ define i128 @udiv_i128_65535(i128 %x) nounwind {
 ; WIN64-NEXT:    movq %rdx, %rax
 ; WIN64-NEXT:    shlq $16, %rax
 ; WIN64-NEXT:    subq %rax, %rdx
-; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    movq %r9, %rax
 ; WIN64-NEXT:    addq %r8, %rax
 ; WIN64-NEXT:    adcq %rdx, %rax
-; WIN64-NEXT:    subq %rax, %rcx
+; WIN64-NEXT:    subq %rax, %r9
 ; WIN64-NEXT:    sbbq $0, %r8
-; WIN64-NEXT:    movabsq $-281479271743490, %r9 # imm = 0xFFFEFFFEFFFEFFFE
-; WIN64-NEXT:    imulq %rcx, %r9
-; WIN64-NEXT:    movabsq $-281479271743489, %r10 # imm = 0xFFFEFFFEFFFEFFFF
+; WIN64-NEXT:    movabsq $-281479271743490, %r10 # imm = 0xFFFEFFFEFFFEFFFE
+; WIN64-NEXT:    imulq %r9, %r10
+; WIN64-NEXT:    movabsq $-281479271743489, %r11 # imm = 0xFFFEFFFEFFFEFFFF
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    addq %r10, %rdx
+; WIN64-NEXT:    imulq %r11, %r8
+; WIN64-NEXT:    addq %rdx, %r8
+; WIN64-NEXT:    movq %rax, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %r10
-; WIN64-NEXT:    addq %r9, %rdx
-; WIN64-NEXT:    imulq %r10, %r8
-; WIN64-NEXT:    addq %r8, %rdx
 ; WIN64-NEXT:    retq
 entry:
   %rem = udiv i128 %x, 65535
@@ -886,27 +926,30 @@ define i128 @udiv_i128_65537(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: udiv_i128_65537:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq %rcx, %r9
-; WIN64-NEXT:    addq %rdx, %r9
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    movabsq $-281470681808895, %r10 # imm = 0xFFFF0000FFFF0001
-; WIN64-NEXT:    movq %r9, %rax
-; WIN64-NEXT:    mulq %r10
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    addq %r8, %r10
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    movabsq $-281470681808895, %r11 # imm = 0xFFFF0000FFFF0001
+; WIN64-NEXT:    movq %r10, %rax
+; WIN64-NEXT:    mulq %r11
 ; WIN64-NEXT:    movq %rdx, %rax
 ; WIN64-NEXT:    andq $-65536, %rax # imm = 0xFFFF0000
 ; WIN64-NEXT:    shrq $16, %rdx
 ; WIN64-NEXT:    addq %rax, %rdx
-; WIN64-NEXT:    subq %rdx, %r9
-; WIN64-NEXT:    subq %r9, %rcx
+; WIN64-NEXT:    subq %rdx, %r10
+; WIN64-NEXT:    subq %r10, %r9
 ; WIN64-NEXT:    sbbq $0, %r8
-; WIN64-NEXT:    movabsq $-281470681808896, %r9 # imm = 0xFFFF0000FFFF0000
-; WIN64-NEXT:    imulq %rcx, %r9
+; WIN64-NEXT:    movabsq $-281470681808896, %r10 # imm = 0xFFFF0000FFFF0000
+; WIN64-NEXT:    imulq %r9, %r10
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    addq %r10, %rdx
+; WIN64-NEXT:    imulq %r11, %r8
+; WIN64-NEXT:    addq %rdx, %r8
+; WIN64-NEXT:    movq %rax, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %r10
-; WIN64-NEXT:    addq %r9, %rdx
-; WIN64-NEXT:    imulq %r10, %r8
-; WIN64-NEXT:    addq %r8, %rdx
 ; WIN64-NEXT:    retq
 entry:
   %rem = udiv i128 %x, 65537
@@ -940,27 +983,30 @@ define i128 @udiv_i128_12(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: udiv_i128_12:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    shrdq $2, %rdx, %rcx
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    shrdq $2, %r8, %r9
 ; WIN64-NEXT:    shrq $2, %r8
-; WIN64-NEXT:    movq %rcx, %r9
-; WIN64-NEXT:    addq %r8, %r9
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    movabsq $-6148914691236517205, %r10 # imm = 0xAAAAAAAAAAAAAAAB
-; WIN64-NEXT:    movq %r9, %rax
-; WIN64-NEXT:    mulq %r10
+; WIN64-NEXT:    movq %r9, %r10
+; WIN64-NEXT:    addq %r8, %r10
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    movabsq $-6148914691236517205, %r11 # imm = 0xAAAAAAAAAAAAAAAB
+; WIN64-NEXT:    movq %r10, %rax
+; WIN64-NEXT:    mulq %r11
 ; WIN64-NEXT:    shrq %rdx
 ; WIN64-NEXT:    leaq (%rdx,%rdx,2), %rax
-; WIN64-NEXT:    subq %rax, %r9
-; WIN64-NEXT:    subq %r9, %rcx
+; WIN64-NEXT:    subq %rax, %r10
+; WIN64-NEXT:    subq %r10, %r9
 ; WIN64-NEXT:    sbbq $0, %r8
-; WIN64-NEXT:    movabsq $-6148914691236517206, %r9 # imm = 0xAAAAAAAAAAAAAAAA
-; WIN64-NEXT:    imulq %rcx, %r9
+; WIN64-NEXT:    movabsq $-6148914691236517206, %r10 # imm = 0xAAAAAAAAAAAAAAAA
+; WIN64-NEXT:    imulq %r9, %r10
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    addq %r10, %rdx
+; WIN64-NEXT:    imulq %r11, %r8
+; WIN64-NEXT:    addq %rdx, %r8
+; WIN64-NEXT:    movq %rax, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %r10
-; WIN64-NEXT:    addq %r9, %rdx
-; WIN64-NEXT:    imulq %r10, %r8
-; WIN64-NEXT:    addq %r8, %rdx
 ; WIN64-NEXT:    retq
 entry:
   %rem = udiv i128 %x, 12
@@ -981,19 +1027,26 @@ define i128 @urem_i128_3_minsize(i128 %x) nounwind minsize {
 ;
 ; WIN64-LABEL: urem_i128_3_minsize:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    subq $72, %rsp
+; WIN64-NEXT:    pushq %rsi
+; WIN64-NEXT:    pushq %rdi
+; WIN64-NEXT:    subq $88, %rsp
+; WIN64-NEXT:    movq %rcx, %rsi
 ; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rax
-; WIN64-NEXT:    movq %rdx, 8(%rax)
-; WIN64-NEXT:    movq %rcx, (%rax)
-; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
-; WIN64-NEXT:    movq $3, (%rdx)
-; WIN64-NEXT:    andq $0, 8(%rdx)
-; WIN64-NEXT:    movq %rax, %rcx
+; WIN64-NEXT:    movq %r8, 8(%rax)
+; WIN64-NEXT:    movq %rdx, (%rax)
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
+; WIN64-NEXT:    movq $3, (%r8)
+; WIN64-NEXT:    andq $0, 8(%r8)
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
+; WIN64-NEXT:    movq %rdi, %rcx
+; WIN64-NEXT:    movq %rax, %rdx
 ; WIN64-NEXT:    callq __umodti3
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; WIN64-NEXT:    movq %xmm0, %rdx
-; WIN64-NEXT:    addq $72, %rsp
+; WIN64-NEXT:    movaps (%rdi), %xmm0
+; WIN64-NEXT:    movaps %xmm0, (%rsi)
+; WIN64-NEXT:    movq %rsi, %rax
+; WIN64-NEXT:    addq $88, %rsp
+; WIN64-NEXT:    popq %rdi
+; WIN64-NEXT:    popq %rsi
 ; WIN64-NEXT:    retq
 entry:
   %rem = urem i128 %x, 3
@@ -1013,19 +1066,26 @@ define i128 @urem_i128_3_optsize(i128 %x) nounwind optsize {
 ;
 ; WIN64-LABEL: urem_i128_3_optsize:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    subq $72, %rsp
+; WIN64-NEXT:    pushq %rsi
+; WIN64-NEXT:    pushq %rdi
+; WIN64-NEXT:    subq $88, %rsp
+; WIN64-NEXT:    movq %rcx, %rsi
 ; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rax
-; WIN64-NEXT:    movq %rdx, 8(%rax)
-; WIN64-NEXT:    movq %rcx, (%rax)
-; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
-; WIN64-NEXT:    movq $3, (%rdx)
-; WIN64-NEXT:    movq $0, 8(%rdx)
-; WIN64-NEXT:    movq %rax, %rcx
+; WIN64-NEXT:    movq %r8, 8(%rax)
+; WIN64-NEXT:    movq %rdx, (%rax)
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %r8
+; WIN64-NEXT:    movq $3, (%r8)
+; WIN64-NEXT:    movq $0, 8(%r8)
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
+; WIN64-NEXT:    movq %rdi, %rcx
+; WIN64-NEXT:    movq %rax, %rdx
 ; WIN64-NEXT:    callq __umodti3
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; WIN64-NEXT:    movq %xmm0, %rdx
-; WIN64-NEXT:    addq $72, %rsp
+; WIN64-NEXT:    movaps (%rdi), %xmm0
+; WIN64-NEXT:    movaps %xmm0, (%rsi)
+; WIN64-NEXT:    movq %rsi, %rax
+; WIN64-NEXT:    addq $88, %rsp
+; WIN64-NEXT:    popq %rdi
+; WIN64-NEXT:    popq %rsi
 ; WIN64-NEXT:    retq
 entry:
   %rem = urem i128 %x, 3
@@ -1067,35 +1127,38 @@ define i128 @udiv_i128_100(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: udiv_i128_100:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    shrdq $62, %rdx, %rax
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq %rdx, %rax
+; WIN64-NEXT:    shrdq $62, %r8, %rax
 ; WIN64-NEXT:    movabsq $1152921504606846975, %rdx # imm = 0xFFFFFFFFFFFFFFF
 ; WIN64-NEXT:    andq %rdx, %rax
-; WIN64-NEXT:    shrdq $2, %r8, %rcx
-; WIN64-NEXT:    andq %rcx, %rdx
+; WIN64-NEXT:    shrdq $2, %r8, %r9
+; WIN64-NEXT:    andq %r9, %rdx
 ; WIN64-NEXT:    addq %rax, %rdx
-; WIN64-NEXT:    movq %r8, %r9
-; WIN64-NEXT:    shrq $58, %r9
-; WIN64-NEXT:    addq %rdx, %r9
+; WIN64-NEXT:    movq %r8, %r10
+; WIN64-NEXT:    shrq $58, %r10
+; WIN64-NEXT:    addq %rdx, %r10
 ; WIN64-NEXT:    movabsq $2951479051793528259, %rdx # imm = 0x28F5C28F5C28F5C3
-; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    movq %r10, %rax
 ; WIN64-NEXT:    mulq %rdx
 ; WIN64-NEXT:    shrq $2, %rdx
 ; WIN64-NEXT:    leaq (%rdx,%rdx,4), %rax
 ; WIN64-NEXT:    leaq (%rax,%rax,4), %rax
-; WIN64-NEXT:    subq %rax, %r9
+; WIN64-NEXT:    subq %rax, %r10
 ; WIN64-NEXT:    shrq $2, %r8
-; WIN64-NEXT:    subq %r9, %rcx
+; WIN64-NEXT:    subq %r10, %r9
 ; WIN64-NEXT:    sbbq $0, %r8
-; WIN64-NEXT:    movabsq $2951479051793528258, %r9 # imm = 0x28F5C28F5C28F5C2
-; WIN64-NEXT:    imulq %rcx, %r9
-; WIN64-NEXT:    movabsq $-8116567392432202711, %r10 # imm = 0x8F5C28F5C28F5C29
+; WIN64-NEXT:    movabsq $2951479051793528258, %r10 # imm = 0x28F5C28F5C28F5C2
+; WIN64-NEXT:    imulq %r9, %r10
+; WIN64-NEXT:    movabsq $-8116567392432202711, %r11 # imm = 0x8F5C28F5C28F5C29
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    addq %r10, %rdx
+; WIN64-NEXT:    imulq %r11, %r8
+; WIN64-NEXT:    addq %rdx, %r8
+; WIN64-NEXT:    movq %rax, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %r10
-; WIN64-NEXT:    addq %r9, %rdx
-; WIN64-NEXT:    imulq %r10, %r8
-; WIN64-NEXT:    addq %r8, %rdx
 ; WIN64-NEXT:    retq
 entry:
   %rem = udiv i128 %x, 100
@@ -1136,34 +1199,36 @@ define i128 @udiv_i128_13(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: udiv_i128_13:
 ; WIN64:       # %bb.0: # %entry
-; WIN64-NEXT:    movq %rdx, %r8
+; WIN64-NEXT:    movq %rdx, %r9
 ; WIN64-NEXT:    movabsq $1152921504606846975, %rax # imm = 0xFFFFFFFFFFFFFFF
-; WIN64-NEXT:    movq %rcx, %rdx
 ; WIN64-NEXT:    andq %rax, %rdx
-; WIN64-NEXT:    movq %rcx, %r10
-; WIN64-NEXT:    shrdq $60, %r8, %r10
-; WIN64-NEXT:    andq %rax, %r10
-; WIN64-NEXT:    addq %rdx, %r10
-; WIN64-NEXT:    movq %r8, %r9
-; WIN64-NEXT:    shrq $56, %r9
-; WIN64-NEXT:    addq %r10, %r9
+; WIN64-NEXT:    movq %r9, %r11
+; WIN64-NEXT:    shrdq $60, %r8, %r11
+; WIN64-NEXT:    andq %rax, %r11
+; WIN64-NEXT:    addq %rdx, %r11
+; WIN64-NEXT:    movq %r8, %r10
+; WIN64-NEXT:    shrq $56, %r10
+; WIN64-NEXT:    addq %r11, %r10
 ; WIN64-NEXT:    movabsq $2837960626724546403, %rdx # imm = 0x2762762762762763
-; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    movq %r10, %rax
 ; WIN64-NEXT:    mulq %rdx
 ; WIN64-NEXT:    shrq %rdx
 ; WIN64-NEXT:    leaq (%rdx,%rdx,2), %rax
 ; WIN64-NEXT:    leaq (%rdx,%rax,4), %rax
-; WIN64-NEXT:    subq %rax, %r9
-; WIN64-NEXT:    subq %r9, %rcx
+; WIN64-NEXT:    subq %rax, %r10
+; WIN64-NEXT:    subq %r10, %r9
 ; WIN64-NEXT:    sbbq $0, %r8
-; WIN64-NEXT:    movabsq $-4256940940086819604, %r9 # imm = 0xC4EC4EC4EC4EC4EC
-; WIN64-NEXT:    imulq %rcx, %r9
-; WIN64-NEXT:    movabsq $5675921253449092805, %r10 # imm = 0x4EC4EC4EC4EC4EC5
+; WIN64-NEXT:    movabsq $-4256940940086819604, %r10 # imm = 0xC4EC4EC4EC4EC4EC
+; WIN64-NEXT:    imulq %r9, %r10
+; WIN64-NEXT:    movabsq $5675921253449092805, %r11 # imm = 0x4EC4EC4EC4EC4EC5
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    addq %r10, %rdx
+; WIN64-NEXT:    imulq %r11, %r8
+; WIN64-NEXT:    addq %rdx, %r8
+; WIN64-NEXT:    movq %rax, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %r10
-; WIN64-NEXT:    addq %r9, %rdx
-; WIN64-NEXT:    imulq %r10, %r8
-; WIN64-NEXT:    addq %r8, %rdx
 ; WIN64-NEXT:    retq
 entry:
   %rem = udiv i128 %x, 13
@@ -1207,36 +1272,39 @@ define i128 @udiv_magic_preshift_and_postshift(i128 %x) nounwind {
 ; WIN64-LABEL: udiv_magic_preshift_and_postshift:
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsi
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq %rcx, %r9
-; WIN64-NEXT:    shrdq $1, %rdx, %r9
-; WIN64-NEXT:    movabsq $3297741957311204758, %r11 # imm = 0x2DC3EED6866F8D96
-; WIN64-NEXT:    movq %r9, %rax
-; WIN64-NEXT:    mulq %r11
-; WIN64-NEXT:    movq %rax, %r10
-; WIN64-NEXT:    movq %rdx, %rcx
-; WIN64-NEXT:    movabsq $3091633084979254461, %rsi # imm = 0x2AE7AFE91E0894BD
-; WIN64-NEXT:    movq %r9, %rax
-; WIN64-NEXT:    mulq %rsi
-; WIN64-NEXT:    movq %rdx, %r9
-; WIN64-NEXT:    addq %r10, %r9
-; WIN64-NEXT:    adcq $0, %rcx
-; WIN64-NEXT:    shrq %r8
-; WIN64-NEXT:    movq %r8, %rax
-; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    pushq %rdi
+; WIN64-NEXT:    movq %r8, %r9
 ; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    shrdq $1, %r8, %r10
+; WIN64-NEXT:    movabsq $3297741957311204758, %rsi # imm = 0x2DC3EED6866F8D96
+; WIN64-NEXT:    movq %r10, %rax
+; WIN64-NEXT:    mulq %rsi
 ; WIN64-NEXT:    movq %rax, %r11
-; WIN64-NEXT:    movq %r8, %rax
+; WIN64-NEXT:    movq %rdx, %r8
+; WIN64-NEXT:    movabsq $3091633084979254461, %rdi # imm = 0x2AE7AFE91E0894BD
+; WIN64-NEXT:    movq %r10, %rax
+; WIN64-NEXT:    mulq %rdi
+; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    addq %r11, %r10
+; WIN64-NEXT:    adcq $0, %r8
+; WIN64-NEXT:    shrq %r9
+; WIN64-NEXT:    movq %r9, %rax
 ; WIN64-NEXT:    mulq %rsi
-; WIN64-NEXT:    addq %r9, %rax
-; WIN64-NEXT:    adcq %rdx, %rcx
-; WIN64-NEXT:    adcq $0, %r10
-; WIN64-NEXT:    addq %r11, %rcx
-; WIN64-NEXT:    adcq $0, %r10
-; WIN64-NEXT:    shrdq $5, %r10, %rcx
-; WIN64-NEXT:    shrq $5, %r10
+; WIN64-NEXT:    movq %rdx, %r11
+; WIN64-NEXT:    movq %rax, %rsi
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %rdi
+; WIN64-NEXT:    addq %r10, %rax
+; WIN64-NEXT:    adcq %r8, %rdx
+; WIN64-NEXT:    adcq $0, %r11
+; WIN64-NEXT:    addq %rsi, %rdx
+; WIN64-NEXT:    adcq $0, %r11
+; WIN64-NEXT:    shrdq $5, %r11, %rdx
+; WIN64-NEXT:    shrq $5, %r11
+; WIN64-NEXT:    movq %r11, 8(%rcx)
+; WIN64-NEXT:    movq %rdx, (%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    movq %r10, %rdx
+; WIN64-NEXT:    popq %rdi
 ; WIN64-NEXT:    popq %rsi
 ; WIN64-NEXT:    retq
   %ret = udiv i128 %x, 358
@@ -1290,50 +1358,53 @@ define i128 @urem_magic_preshift_and_postshift(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: urem_magic_preshift_and_postshift:
 ; WIN64:       # %bb.0:
+; WIN64-NEXT:    pushq %r14
 ; WIN64-NEXT:    pushq %rsi
 ; WIN64-NEXT:    pushq %rdi
 ; WIN64-NEXT:    pushq %rbx
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq %rcx, %r10
-; WIN64-NEXT:    shrdq $1, %rdx, %r10
-; WIN64-NEXT:    movabsq $3297741957311204758, %rdi # imm = 0x2DC3EED6866F8D96
-; WIN64-NEXT:    movq %r10, %rax
-; WIN64-NEXT:    mulq %rdi
-; WIN64-NEXT:    movq %rax, %r11
 ; WIN64-NEXT:    movq %rdx, %r9
-; WIN64-NEXT:    movabsq $3091633084979254461, %rbx # imm = 0x2AE7AFE91E0894BD
-; WIN64-NEXT:    movq %r10, %rax
-; WIN64-NEXT:    mulq %rbx
-; WIN64-NEXT:    movq %rdx, %rsi
-; WIN64-NEXT:    addq %r11, %rsi
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    movq %r8, %r11
-; WIN64-NEXT:    shrq %r11
+; WIN64-NEXT:    movq %rdx, %r11
+; WIN64-NEXT:    shrdq $1, %r8, %r11
+; WIN64-NEXT:    movabsq $3297741957311204758, %rbx # imm = 0x2DC3EED6866F8D96
 ; WIN64-NEXT:    movq %r11, %rax
-; WIN64-NEXT:    mulq %rdi
+; WIN64-NEXT:    mulq %rbx
+; WIN64-NEXT:    movq %rax, %rsi
 ; WIN64-NEXT:    movq %rdx, %r10
-; WIN64-NEXT:    movq %rax, %rdi
+; WIN64-NEXT:    movabsq $3091633084979254461, %r14 # imm = 0x2AE7AFE91E0894BD
 ; WIN64-NEXT:    movq %r11, %rax
-; WIN64-NEXT:    mulq %rbx
-; WIN64-NEXT:    addq %rsi, %rax
-; WIN64-NEXT:    adcq %rdx, %r9
+; WIN64-NEXT:    mulq %r14
+; WIN64-NEXT:    movq %rdx, %rdi
+; WIN64-NEXT:    addq %rsi, %rdi
 ; WIN64-NEXT:    adcq $0, %r10
-; WIN64-NEXT:    addq %rdi, %r9
-; WIN64-NEXT:    adcq $0, %r10
-; WIN64-NEXT:    shrdq $5, %r10, %r9
+; WIN64-NEXT:    movq %r8, %rsi
+; WIN64-NEXT:    shrq %rsi
+; WIN64-NEXT:    movq %rsi, %rax
+; WIN64-NEXT:    mulq %rbx
+; WIN64-NEXT:    movq %rdx, %r11
+; WIN64-NEXT:    movq %rax, %rbx
+; WIN64-NEXT:    movq %rsi, %rax
+; WIN64-NEXT:    mulq %r14
+; WIN64-NEXT:    addq %rdi, %rax
+; WIN64-NEXT:    adcq %rdx, %r10
+; WIN64-NEXT:    adcq $0, %r11
+; WIN64-NEXT:    addq %rbx, %r10
+; WIN64-NEXT:    adcq $0, %r11
+; WIN64-NEXT:    shrdq $5, %r11, %r10
 ; WIN64-NEXT:    movl $358, %edx # imm = 0x166
-; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    movq %r10, %rax
 ; WIN64-NEXT:    mulq %rdx
-; WIN64-NEXT:    shrq $5, %r10
-; WIN64-NEXT:    imulq $358, %r10, %r9 # imm = 0x166
-; WIN64-NEXT:    addq %rdx, %r9
-; WIN64-NEXT:    subq %rax, %rcx
-; WIN64-NEXT:    sbbq %r9, %r8
+; WIN64-NEXT:    shrq $5, %r11
+; WIN64-NEXT:    imulq $358, %r11, %r10 # imm = 0x166
+; WIN64-NEXT:    addq %rdx, %r10
+; WIN64-NEXT:    subq %rax, %r9
+; WIN64-NEXT:    sbbq %r10, %r8
+; WIN64-NEXT:    movq %r9, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    movq %r8, %rdx
 ; WIN64-NEXT:    popq %rbx
 ; WIN64-NEXT:    popq %rdi
 ; WIN64-NEXT:    popq %rsi
+; WIN64-NEXT:    popq %r14
 ; WIN64-NEXT:    retq
   %ret = urem i128 %x, 358
   ret i128 %ret
@@ -1360,20 +1431,20 @@ define i128 @udiv_magic_large_preshift(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: udiv_magic_large_preshift:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    movq %rdx, %rcx
-; WIN64-NEXT:    shrq $36, %rcx
+; WIN64-NEXT:    shrq $36, %r8
 ; WIN64-NEXT:    movabsq $103054436165975148, %rdx # imm = 0x16E1F76B4337C6C
-; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    movq %r8, %rax
 ; WIN64-NEXT:    mulq %rdx
-; WIN64-NEXT:    movq %rax, %r8
-; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq %rax, %r9
+; WIN64-NEXT:    movq %rdx, %r10
 ; WIN64-NEXT:    movabsq $-5667993989128633178, %rdx # imm = 0xB1573D7F48F044A6
-; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    movq %r8, %rax
 ; WIN64-NEXT:    mulq %rdx
-; WIN64-NEXT:    addq %r8, %rdx
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    movq %r9, %rax
-; WIN64-NEXT:    xorl %edx, %edx
+; WIN64-NEXT:    addq %r9, %rdx
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    movq %r10, (%rcx)
+; WIN64-NEXT:    movq $0, 8(%rcx)
+; WIN64-NEXT:    movq %rcx, %rax
 ; WIN64-NEXT:    retq
   %ret = udiv i128 %x, 226909457440853062867909873762304 ; = 179 * 2^100
   ret i128 %ret
@@ -1404,24 +1475,27 @@ define i128 @urem_magic_large_preshift(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: urem_magic_large_preshift:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    movq %rdx, %r8
+; WIN64-NEXT:    pushq %rsi
 ; WIN64-NEXT:    movq %rdx, %r9
-; WIN64-NEXT:    shrq $36, %r9
+; WIN64-NEXT:    movq %r8, %r10
+; WIN64-NEXT:    shrq $36, %r10
 ; WIN64-NEXT:    movabsq $103054436165975148, %rdx # imm = 0x16E1F76B4337C6C
-; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    movq %r10, %rax
 ; WIN64-NEXT:    mulq %rdx
-; WIN64-NEXT:    movq %rax, %r10
-; WIN64-NEXT:    movq %rdx, %r11
+; WIN64-NEXT:    movq %rax, %r11
+; WIN64-NEXT:    movq %rdx, %rsi
 ; WIN64-NEXT:    movabsq $-5667993989128633178, %rdx # imm = 0xB1573D7F48F044A6
-; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    movq %r10, %rax
 ; WIN64-NEXT:    mulq %rdx
-; WIN64-NEXT:    addq %r10, %rdx
-; WIN64-NEXT:    adcq $0, %r11
+; WIN64-NEXT:    addq %r11, %rdx
+; WIN64-NEXT:    adcq $0, %rsi
 ; WIN64-NEXT:    movabsq $12300786335744, %rax # imm = 0xB3000000000
-; WIN64-NEXT:    imulq %r11, %rax
+; WIN64-NEXT:    imulq %rsi, %rax
 ; WIN64-NEXT:    subq %rax, %r8
+; WIN64-NEXT:    movq %r9, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    movq %r8, %rdx
+; WIN64-NEXT:    popq %rsi
 ; WIN64-NEXT:    retq
   %ret = urem i128 %x, 226909457440853062867909873762304 ; = 179 * 2^100
   ret i128 %ret
@@ -1449,21 +1523,24 @@ define i128 @udiv_magic_large_postshift(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: udiv_magic_large_postshift:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq $-1, %r11
-; WIN64-NEXT:    movq %rdx, %rax
-; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    pushq %rsi
 ; WIN64-NEXT:    movq %rdx, %r9
-; WIN64-NEXT:    movq %rax, %r10
-; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    movq $-1, %rsi
+; WIN64-NEXT:    movq %r8, %rax
+; WIN64-NEXT:    mulq %rsi
+; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    movq %rax, %r11
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %rsi
 ; WIN64-NEXT:    addq %r8, %rax
 ; WIN64-NEXT:    adcq $0, %rdx
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    addq %r10, %rdx
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    movq %r9, %rax
-; WIN64-NEXT:    xorl %edx, %edx
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    addq %r11, %rdx
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    movq %r10, (%rcx)
+; WIN64-NEXT:    movq $0, 8(%rcx)
+; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    popq %rsi
 ; WIN64-NEXT:    retq
   %ret = udiv i128 %x, 18446744073709551617 ; = 2^64 + 1
   ret i128 %ret
@@ -1493,23 +1570,26 @@ define i128 @urem_magic_large_postshift(i128 %x) nounwind {
 ;
 ; WIN64-LABEL: urem_magic_large_postshift:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq $-1, %r11
-; WIN64-NEXT:    movq %rdx, %rax
-; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    pushq %rsi
 ; WIN64-NEXT:    movq %rdx, %r9
-; WIN64-NEXT:    movq %rax, %r10
-; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %r11
+; WIN64-NEXT:    movq $-1, %rsi
+; WIN64-NEXT:    movq %r8, %rax
+; WIN64-NEXT:    mulq %rsi
+; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    movq %rax, %r11
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %rsi
 ; WIN64-NEXT:    addq %r8, %rax
 ; WIN64-NEXT:    adcq $0, %rdx
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    addq %r10, %rdx
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    subq %r9, %rcx
-; WIN64-NEXT:    sbbq %r9, %r8
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    addq %r11, %rdx
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    subq %r10, %r9
+; WIN64-NEXT:    sbbq %r10, %r8
+; WIN64-NEXT:    movq %r9, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    movq %r8, %rdx
+; WIN64-NEXT:    popq %rsi
 ; WIN64-NEXT:    retq
   %ret = urem i128 %x, 18446744073709551617 ; = 2^64 + 1
   ret i128 %ret
@@ -1556,38 +1636,41 @@ define i128 @udiv_magic_is_add(i128 %x) nounwind {
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsi
 ; WIN64-NEXT:    pushq %rdi
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq $-1, %rsi
-; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %rsi
-; WIN64-NEXT:    movq %rax, %r9
-; WIN64-NEXT:    movq %rdx, %r10
-; WIN64-NEXT:    movq $-3, %rdi
-; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    pushq %rbx
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq $-1, %rdi
+; WIN64-NEXT:    movq %rdx, %rax
 ; WIN64-NEXT:    mulq %rdi
+; WIN64-NEXT:    movq %rax, %r10
 ; WIN64-NEXT:    movq %rdx, %r11
-; WIN64-NEXT:    addq %r9, %r11
-; WIN64-NEXT:    adcq $0, %r10
-; WIN64-NEXT:    movq %r8, %rax
-; WIN64-NEXT:    mulq %rsi
-; WIN64-NEXT:    movq %rdx, %r9
-; WIN64-NEXT:    movq %rax, %rsi
+; WIN64-NEXT:    movq $-3, %rbx
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %rbx
+; WIN64-NEXT:    movq %rdx, %rsi
+; WIN64-NEXT:    addq %r10, %rsi
+; WIN64-NEXT:    adcq $0, %r11
 ; WIN64-NEXT:    movq %r8, %rax
 ; WIN64-NEXT:    mulq %rdi
-; WIN64-NEXT:    addq %r11, %rax
-; WIN64-NEXT:    adcq %r10, %rdx
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    addq %rsi, %rdx
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    subq %rdx, %rcx
-; WIN64-NEXT:    sbbq %r9, %r8
-; WIN64-NEXT:    shrdq $1, %r8, %rcx
+; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    movq %rax, %rdi
+; WIN64-NEXT:    movq %r8, %rax
+; WIN64-NEXT:    mulq %rbx
+; WIN64-NEXT:    addq %rsi, %rax
+; WIN64-NEXT:    adcq %r11, %rdx
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    addq %rdi, %rdx
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    subq %rdx, %r9
+; WIN64-NEXT:    sbbq %r10, %r8
+; WIN64-NEXT:    shrdq $1, %r8, %r9
 ; WIN64-NEXT:    shrq %r8
-; WIN64-NEXT:    addq %rdx, %rcx
-; WIN64-NEXT:    adcq %r9, %r8
+; WIN64-NEXT:    addq %rdx, %r9
+; WIN64-NEXT:    adcq %r10, %r8
 ; WIN64-NEXT:    shrq $63, %r8
-; WIN64-NEXT:    movq %r8, %rax
-; WIN64-NEXT:    xorl %edx, %edx
+; WIN64-NEXT:    movq %r8, (%rcx)
+; WIN64-NEXT:    movq $0, 8(%rcx)
+; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    popq %rbx
 ; WIN64-NEXT:    popq %rdi
 ; WIN64-NEXT:    popq %rsi
 ; WIN64-NEXT:    retq
@@ -1642,44 +1725,47 @@ define i128 @urem_magic_is_add(i128 %x) nounwind {
 ; WIN64:       # %bb.0:
 ; WIN64-NEXT:    pushq %rsi
 ; WIN64-NEXT:    pushq %rdi
-; WIN64-NEXT:    movq %rdx, %r8
-; WIN64-NEXT:    movq $-1, %rsi
-; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    mulq %rsi
-; WIN64-NEXT:    movq %rax, %r9
-; WIN64-NEXT:    movq %rdx, %r10
-; WIN64-NEXT:    movq $-3, %rdi
-; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    pushq %rbx
+; WIN64-NEXT:    movq %rdx, %r9
+; WIN64-NEXT:    movq $-1, %rdi
+; WIN64-NEXT:    movq %rdx, %rax
 ; WIN64-NEXT:    mulq %rdi
+; WIN64-NEXT:    movq %rax, %r10
 ; WIN64-NEXT:    movq %rdx, %r11
-; WIN64-NEXT:    addq %r9, %r11
-; WIN64-NEXT:    adcq $0, %r10
-; WIN64-NEXT:    movq %r8, %rax
-; WIN64-NEXT:    mulq %rsi
-; WIN64-NEXT:    movq %rdx, %r9
-; WIN64-NEXT:    movq %rax, %rsi
+; WIN64-NEXT:    movq $-3, %rbx
+; WIN64-NEXT:    movq %r9, %rax
+; WIN64-NEXT:    mulq %rbx
+; WIN64-NEXT:    movq %rdx, %rsi
+; WIN64-NEXT:    addq %r10, %rsi
+; WIN64-NEXT:    adcq $0, %r11
 ; WIN64-NEXT:    movq %r8, %rax
 ; WIN64-NEXT:    mulq %rdi
-; WIN64-NEXT:    addq %r11, %rax
-; WIN64-NEXT:    adcq %r10, %rdx
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    addq %rsi, %rdx
-; WIN64-NEXT:    adcq $0, %r9
-; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    movq %rax, %rdi
+; WIN64-NEXT:    movq %r8, %rax
+; WIN64-NEXT:    mulq %rbx
+; WIN64-NEXT:    addq %rsi, %rax
+; WIN64-NEXT:    adcq %r11, %rdx
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    addq %rdi, %rdx
+; WIN64-NEXT:    adcq $0, %r10
+; WIN64-NEXT:    movq %r9, %rax
 ; WIN64-NEXT:    subq %rdx, %rax
-; WIN64-NEXT:    movq %r8, %r10
-; WIN64-NEXT:    sbbq %r9, %r10
-; WIN64-NEXT:    shrdq $1, %r10, %rax
-; WIN64-NEXT:    shrq %r10
+; WIN64-NEXT:    movq %r8, %r11
+; WIN64-NEXT:    sbbq %r10, %r11
+; WIN64-NEXT:    shrdq $1, %r11, %rax
+; WIN64-NEXT:    shrq %r11
 ; WIN64-NEXT:    addq %rdx, %rax
-; WIN64-NEXT:    adcq %r9, %r10
+; WIN64-NEXT:    adcq %r10, %r11
 ; WIN64-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
-; WIN64-NEXT:    andq %r10, %rax
-; WIN64-NEXT:    shrq $63, %r10
-; WIN64-NEXT:    subq %r10, %rcx
+; WIN64-NEXT:    andq %r11, %rax
+; WIN64-NEXT:    shrq $63, %r11
+; WIN64-NEXT:    subq %r11, %r9
 ; WIN64-NEXT:    sbbq %rax, %r8
+; WIN64-NEXT:    movq %r9, (%rcx)
+; WIN64-NEXT:    movq %r8, 8(%rcx)
 ; WIN64-NEXT:    movq %rcx, %rax
-; WIN64-NEXT:    movq %r8, %rdx
+; WIN64-NEXT:    popq %rbx
 ; WIN64-NEXT:    popq %rdi
 ; WIN64-NEXT:    popq %rsi
 ; WIN64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/fp128-cast.ll b/llvm/test/CodeGen/X86/fp128-cast.ll
index 36ec50efa70b38..bbbd72a0ed476c 100644
--- a/llvm/test/CodeGen/X86/fp128-cast.ll
+++ b/llvm/test/CodeGen/X86/fp128-cast.ll
@@ -518,13 +518,15 @@ define dso_local void @TestFPToSIF128_I128() nounwind {
 ;
 ; WIN64-SSE-LABEL: TestFPToSIF128_I128:
 ; WIN64-SSE:       # %bb.0: # %entry
-; WIN64-SSE-NEXT:    subq $56, %rsp
+; WIN64-SSE-NEXT:    subq $72, %rsp
 ; WIN64-SSE-NEXT:    movaps vf128(%rip), %xmm0
 ; WIN64-SSE-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
 ; WIN64-SSE-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-SSE-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
 ; WIN64-SSE-NEXT:    callq __fixtfti
+; WIN64-SSE-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm0
 ; WIN64-SSE-NEXT:    movaps %xmm0, vi128(%rip)
-; WIN64-SSE-NEXT:    addq $56, %rsp
+; WIN64-SSE-NEXT:    addq $72, %rsp
 ; WIN64-SSE-NEXT:    retq
 ;
 ; X86-LABEL: TestFPToSIF128_I128:
@@ -584,13 +586,15 @@ define dso_local void @TestFPToUIF128_U128() nounwind {
 ;
 ; WIN64-SSE-LABEL: TestFPToUIF128_U128:
 ; WIN64-SSE:       # %bb.0: # %entry
-; WIN64-SSE-NEXT:    subq $56, %rsp
+; WIN64-SSE-NEXT:    subq $72, %rsp
 ; WIN64-SSE-NEXT:    movaps vf128(%rip), %xmm0
 ; WIN64-SSE-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
 ; WIN64-SSE-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-SSE-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
 ; WIN64-SSE-NEXT:    callq __fixunstfti
+; WIN64-SSE-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm0
 ; WIN64-SSE-NEXT:    movaps %xmm0, vu128(%rip)
-; WIN64-SSE-NEXT:    addq $56, %rsp
+; WIN64-SSE-NEXT:    addq $72, %rsp
 ; WIN64-SSE-NEXT:    retq
 ;
 ; X86-LABEL: TestFPToUIF128_U128:
diff --git a/llvm/test/CodeGen/X86/fp128-return-calling-conv.ll b/llvm/test/CodeGen/X86/fp128-return-calling-conv.ll
index a7949479506628..a4930602bfe14d 100644
--- a/llvm/test/CodeGen/X86/fp128-return-calling-conv.ll
+++ b/llvm/test/CodeGen/X86/fp128-return-calling-conv.ll
@@ -1,15 +1,52 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-pc-windows-gnu  | FileCheck %s --check-prefixes=WIN
 ; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc | FileCheck %s --check-prefixes=WIN
+; RUN: llc < %s -mtriple=x86_64-unknown-uefi | FileCheck %s --check-prefixes=WIN
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefix=LINUX
 
 ; On Windows, for calling conventions that have GCC compatibility requirements,
-; follow the GCC convention of returning f128 via the stack (an sret).
-; Elsewhere, and for other calling conventions, return it in XMM0.
+; follow the GCC convention of returning f128 and i128 via the stack (an sret).
+; Elsewhere, and for other calling conventions, f128 is returned in XMM0.
 ;
 ; NOTE: f128 arguments are passed indirectly on Windows, but in XMM registers
 ; elsewhere.
 
+define i128 @ret_i128_ccc() {
+; WIN-LABEL: ret_i128_ccc:
+; WIN:       # %bb.0:
+; WIN-NEXT:    movq %rcx, %rax
+; WIN-NEXT:    movq $0, 8(%rcx)
+; WIN-NEXT:    movq $42, (%rcx)
+; WIN-NEXT:    retq
+;
+; LINUX-LABEL: ret_i128_ccc:
+; LINUX:       # %bb.0:
+; LINUX-NEXT:    movl $42, %eax
+; LINUX-NEXT:    xorl %edx, %edx
+; LINUX-NEXT:    retq
+  ret i128 42
+}
+
+define { i128, i1 } @ret_i128_aggregate(ptr %p) {
+; WIN-LABEL: ret_i128_aggregate:
+; WIN:       # %bb.0:
+; WIN-NEXT:    movq %rcx, %rax
+; WIN-NEXT:    movaps (%rdx), %xmm0
+; WIN-NEXT:    movzbl 16(%rdx), %ecx
+; WIN-NEXT:    movb %cl, 16(%rax)
+; WIN-NEXT:    movaps %xmm0, (%rax)
+; WIN-NEXT:    retq
+;
+; LINUX-LABEL: ret_i128_aggregate:
+; LINUX:       # %bb.0:
+; LINUX-NEXT:    movq (%rdi), %rax
+; LINUX-NEXT:    movq 8(%rdi), %rdx
+; LINUX-NEXT:    movzbl 16(%rdi), %ecx
+; LINUX-NEXT:    retq
+  %v = load { i128, i1 }, ptr %p
+  ret { i128, i1 } %v
+}
+
 define fp128 @ret_ccc(ptr %p) {
 ; WIN-LABEL: ret_ccc:
 ; WIN:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/fp80-conv-libcalls.ll b/llvm/test/CodeGen/X86/fp80-conv-libcalls.ll
index 9a41d25feb0307..6d7defa167d37c 100644
--- a/llvm/test/CodeGen/X86/fp80-conv-libcalls.ll
+++ b/llvm/test/CodeGen/X86/fp80-conv-libcalls.ll
@@ -92,15 +92,19 @@ define i128 @test_fptosi(x86_fp80 %x) nounwind {
 ;
 ; MSVC-LABEL: test_fptosi:
 ; MSVC:       # %bb.0:
-; MSVC-NEXT:    subq $56, %rsp
-; MSVC-NEXT:    fldt (%rcx)
+; MSVC-NEXT:    pushq %rsi
+; MSVC-NEXT:    subq $64, %rsp
+; MSVC-NEXT:    movq %rcx, %rsi
+; MSVC-NEXT:    fldt (%rdx)
 ; MSVC-NEXT:    fstpt {{[0-9]+}}(%rsp)
 ; MSVC-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; MSVC-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
 ; MSVC-NEXT:    callq __fixxfti
-; MSVC-NEXT:    movq %xmm0, %rax
-; MSVC-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; MSVC-NEXT:    movq %xmm0, %rdx
-; MSVC-NEXT:    addq $56, %rsp
+; MSVC-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm0
+; MSVC-NEXT:    movaps %xmm0, (%rsi)
+; MSVC-NEXT:    movq %rsi, %rax
+; MSVC-NEXT:    addq $64, %rsp
+; MSVC-NEXT:    popq %rsi
 ; MSVC-NEXT:    retq
   %r = fptosi x86_fp80 %x to i128
   ret i128 %r
diff --git a/llvm/test/CodeGen/X86/fptosi-f16-wide.ll b/llvm/test/CodeGen/X86/fptosi-f16-wide.ll
index 49131fb1ada9bd..d41c8c2c88afb9 100644
--- a/llvm/test/CodeGen/X86/fptosi-f16-wide.ll
+++ b/llvm/test/CodeGen/X86/fptosi-f16-wide.ll
@@ -34,19 +34,29 @@ define i128 @fptosi_f16_to_i128(half %x) nounwind {
 ;
 ; WIN-SSE2-LABEL: fptosi_f16_to_i128:
 ; WIN-SSE2:       # %bb.0:
-; WIN-SSE2-NEXT:    subq $40, %rsp
+; WIN-SSE2-NEXT:    pushq %rsi
+; WIN-SSE2-NEXT:    subq $32, %rsp
+; WIN-SSE2-NEXT:    movaps %xmm1, %xmm0
+; WIN-SSE2-NEXT:    movq %rcx, %rsi
 ; WIN-SSE2-NEXT:    callq __extendhfsf2
 ; WIN-SSE2-NEXT:    cvttss2si %xmm0, %rax
-; WIN-SSE2-NEXT:    movq %rax, %rdx
-; WIN-SSE2-NEXT:    sarq $63, %rdx
-; WIN-SSE2-NEXT:    addq $40, %rsp
+; WIN-SSE2-NEXT:    movq %rax, %rcx
+; WIN-SSE2-NEXT:    sarq $63, %rcx
+; WIN-SSE2-NEXT:    movq %rcx, 8(%rsi)
+; WIN-SSE2-NEXT:    movq %rax, (%rsi)
+; WIN-SSE2-NEXT:    movq %rsi, %rax
+; WIN-SSE2-NEXT:    addq $32, %rsp
+; WIN-SSE2-NEXT:    popq %rsi
 ; WIN-SSE2-NEXT:    retq
 ;
 ; WIN-FP16-LABEL: fptosi_f16_to_i128:
 ; WIN-FP16:       # %bb.0:
-; WIN-FP16-NEXT:    vcvttsh2si %xmm0, %rax
-; WIN-FP16-NEXT:    movq %rax, %rdx
+; WIN-FP16-NEXT:    movq %rcx, %rax
+; WIN-FP16-NEXT:    vcvttsh2si %xmm1, %rcx
+; WIN-FP16-NEXT:    movq %rcx, %rdx
 ; WIN-FP16-NEXT:    sarq $63, %rdx
+; WIN-FP16-NEXT:    movq %rdx, 8(%rax)
+; WIN-FP16-NEXT:    movq %rcx, (%rax)
 ; WIN-FP16-NEXT:    retq
   %r = fptosi half %x to i128
   ret i128 %r
@@ -83,19 +93,29 @@ define i128 @fptosi_f16_to_i128_strict(half %x) nounwind strictfp {
 ;
 ; WIN-SSE2-LABEL: fptosi_f16_to_i128_strict:
 ; WIN-SSE2:       # %bb.0:
-; WIN-SSE2-NEXT:    subq $40, %rsp
+; WIN-SSE2-NEXT:    pushq %rsi
+; WIN-SSE2-NEXT:    subq $32, %rsp
+; WIN-SSE2-NEXT:    movaps %xmm1, %xmm0
+; WIN-SSE2-NEXT:    movq %rcx, %rsi
 ; WIN-SSE2-NEXT:    callq __extendhfsf2
 ; WIN-SSE2-NEXT:    cvttss2si %xmm0, %rax
-; WIN-SSE2-NEXT:    movq %rax, %rdx
-; WIN-SSE2-NEXT:    sarq $63, %rdx
-; WIN-SSE2-NEXT:    addq $40, %rsp
+; WIN-SSE2-NEXT:    movq %rax, %rcx
+; WIN-SSE2-NEXT:    sarq $63, %rcx
+; WIN-SSE2-NEXT:    movq %rcx, 8(%rsi)
+; WIN-SSE2-NEXT:    movq %rax, (%rsi)
+; WIN-SSE2-NEXT:    movq %rsi, %rax
+; WIN-SSE2-NEXT:    addq $32, %rsp
+; WIN-SSE2-NEXT:    popq %rsi
 ; WIN-SSE2-NEXT:    retq
 ;
 ; WIN-FP16-LABEL: fptosi_f16_to_i128_strict:
 ; WIN-FP16:       # %bb.0:
-; WIN-FP16-NEXT:    vcvttsh2si %xmm0, %rax
-; WIN-FP16-NEXT:    movq %rax, %rdx
+; WIN-FP16-NEXT:    movq %rcx, %rax
+; WIN-FP16-NEXT:    vcvttsh2si %xmm1, %rcx
+; WIN-FP16-NEXT:    movq %rcx, %rdx
 ; WIN-FP16-NEXT:    sarq $63, %rdx
+; WIN-FP16-NEXT:    movq %rdx, 8(%rax)
+; WIN-FP16-NEXT:    movq %rcx, (%rax)
 ; WIN-FP16-NEXT:    retq
   %r = call i128 @llvm.experimental.constrained.fptosi.i128.f16(half %x, metadata !"fpexcept.strict") strictfp
   ret i128 %r
diff --git a/llvm/test/CodeGen/X86/fptoui-f16-wide.ll b/llvm/test/CodeGen/X86/fptoui-f16-wide.ll
index e2edd6232d9dec..0d7f84af5cce70 100644
--- a/llvm/test/CodeGen/X86/fptoui-f16-wide.ll
+++ b/llvm/test/CodeGen/X86/fptoui-f16-wide.ll
@@ -31,17 +31,25 @@ define i128 @fptoui_f16_to_i128(half %x) nounwind {
 ;
 ; WIN-SSE2-LABEL: fptoui_f16_to_i128:
 ; WIN-SSE2:       # %bb.0:
-; WIN-SSE2-NEXT:    subq $40, %rsp
+; WIN-SSE2-NEXT:    pushq %rsi
+; WIN-SSE2-NEXT:    subq $32, %rsp
+; WIN-SSE2-NEXT:    movaps %xmm1, %xmm0
+; WIN-SSE2-NEXT:    movq %rcx, %rsi
 ; WIN-SSE2-NEXT:    callq __extendhfsf2
 ; WIN-SSE2-NEXT:    cvttss2si %xmm0, %rax
-; WIN-SSE2-NEXT:    xorl %edx, %edx
-; WIN-SSE2-NEXT:    addq $40, %rsp
+; WIN-SSE2-NEXT:    movq %rax, (%rsi)
+; WIN-SSE2-NEXT:    movq $0, 8(%rsi)
+; WIN-SSE2-NEXT:    movq %rsi, %rax
+; WIN-SSE2-NEXT:    addq $32, %rsp
+; WIN-SSE2-NEXT:    popq %rsi
 ; WIN-SSE2-NEXT:    retq
 ;
 ; WIN-FP16-LABEL: fptoui_f16_to_i128:
 ; WIN-FP16:       # %bb.0:
-; WIN-FP16-NEXT:    vcvttsh2si %xmm0, %rax
-; WIN-FP16-NEXT:    xorl %edx, %edx
+; WIN-FP16-NEXT:    movq %rcx, %rax
+; WIN-FP16-NEXT:    vcvttsh2si %xmm1, %rcx
+; WIN-FP16-NEXT:    movq %rcx, (%rax)
+; WIN-FP16-NEXT:    movq $0, 8(%rax)
 ; WIN-FP16-NEXT:    retq
   %r = fptoui half %x to i128
   ret i128 %r
@@ -75,17 +83,25 @@ define i128 @fptoui_f16_to_i128_strict(half %x) nounwind strictfp {
 ;
 ; WIN-SSE2-LABEL: fptoui_f16_to_i128_strict:
 ; WIN-SSE2:       # %bb.0:
-; WIN-SSE2-NEXT:    subq $40, %rsp
+; WIN-SSE2-NEXT:    pushq %rsi
+; WIN-SSE2-NEXT:    subq $32, %rsp
+; WIN-SSE2-NEXT:    movaps %xmm1, %xmm0
+; WIN-SSE2-NEXT:    movq %rcx, %rsi
 ; WIN-SSE2-NEXT:    callq __extendhfsf2
 ; WIN-SSE2-NEXT:    cvttss2si %xmm0, %rax
-; WIN-SSE2-NEXT:    xorl %edx, %edx
-; WIN-SSE2-NEXT:    addq $40, %rsp
+; WIN-SSE2-NEXT:    movq %rax, (%rsi)
+; WIN-SSE2-NEXT:    movq $0, 8(%rsi)
+; WIN-SSE2-NEXT:    movq %rsi, %rax
+; WIN-SSE2-NEXT:    addq $32, %rsp
+; WIN-SSE2-NEXT:    popq %rsi
 ; WIN-SSE2-NEXT:    retq
 ;
 ; WIN-FP16-LABEL: fptoui_f16_to_i128_strict:
 ; WIN-FP16:       # %bb.0:
-; WIN-FP16-NEXT:    vcvttsh2si %xmm0, %rax
-; WIN-FP16-NEXT:    xorl %edx, %edx
+; WIN-FP16-NEXT:    movq %rcx, %rax
+; WIN-FP16-NEXT:    vcvttsh2si %xmm1, %rcx
+; WIN-FP16-NEXT:    movq %rcx, (%rax)
+; WIN-FP16-NEXT:    movq $0, 8(%rax)
 ; WIN-FP16-NEXT:    retq
   %r = call i128 @llvm.experimental.constrained.fptoui.i128.f16(half %x, metadata !"fpexcept.strict") strictfp
   ret i128 %r
diff --git a/llvm/test/CodeGen/X86/i128-fp128-abi.ll b/llvm/test/CodeGen/X86/i128-fp128-abi.ll
index 77881225408dd1..77bdf3bc7ae4f4 100644
--- a/llvm/test/CodeGen/X86/i128-fp128-abi.ll
+++ b/llvm/test/CodeGen/X86/i128-fp128-abi.ll
@@ -197,8 +197,9 @@ define PrimTy @return(ptr %p) nounwind {
 ;
 ; CHECK-MSVC64-I128-LABEL: return:
 ; CHECK-MSVC64-I128:       # %bb.0:
-; CHECK-MSVC64-I128-NEXT:    movq (%rcx), %rax
-; CHECK-MSVC64-I128-NEXT:    movq 8(%rcx), %rdx
+; CHECK-MSVC64-I128-NEXT:    movq %rcx, %rax
+; CHECK-MSVC64-I128-NEXT:    movaps (%rdx), %xmm0
+; CHECK-MSVC64-I128-NEXT:    movaps %xmm0, (%rcx)
 ; CHECK-MSVC64-I128-NEXT:    retq
 ;
 ; CHECK-MINGW-F128-LABEL: return:
@@ -210,8 +211,9 @@ define PrimTy @return(ptr %p) nounwind {
 ;
 ; CHECK-MINGW-I128-LABEL: return:
 ; CHECK-MINGW-I128:       # %bb.0:
-; CHECK-MINGW-I128-NEXT:    movq (%rcx), %rax
-; CHECK-MINGW-I128-NEXT:    movq 8(%rcx), %rdx
+; CHECK-MINGW-I128-NEXT:    movq %rcx, %rax
+; CHECK-MINGW-I128-NEXT:    movaps (%rdx), %xmm0
+; CHECK-MINGW-I128-NEXT:    movaps %xmm0, (%rcx)
 ; CHECK-MINGW-I128-NEXT:    retq
 ;
 ; CHECK-X86-LABEL: return:
@@ -274,6 +276,8 @@ define PrimTy @first_arg(PrimTy %x) nounwind {
 ; CHECK-MSVC64-I128-LABEL: first_arg:
 ; CHECK-MSVC64-I128:       # %bb.0:
 ; CHECK-MSVC64-I128-NEXT:    movq %rcx, %rax
+; CHECK-MSVC64-I128-NEXT:    movq %r8, 8(%rcx)
+; CHECK-MSVC64-I128-NEXT:    movq %rdx, (%rcx)
 ; CHECK-MSVC64-I128-NEXT:    retq
 ;
 ; CHECK-MINGW-F128-LABEL: first_arg:
@@ -286,6 +290,8 @@ define PrimTy @first_arg(PrimTy %x) nounwind {
 ; CHECK-MINGW-I128-LABEL: first_arg:
 ; CHECK-MINGW-I128:       # %bb.0:
 ; CHECK-MINGW-I128-NEXT:    movq %rcx, %rax
+; CHECK-MINGW-I128-NEXT:    movq %r8, 8(%rcx)
+; CHECK-MINGW-I128-NEXT:    movq %rdx, (%rcx)
 ; CHECK-MINGW-I128-NEXT:    retq
 ;
 ; CHECK-X86-LABEL: first_arg:
@@ -354,8 +360,11 @@ define PrimTy @leading_args(i64 %_0, i64 %_1, i64 %_2, i64 %_3, PrimTy %x) nounw
 ;
 ; CHECK-MSVC64-I128-LABEL: leading_args:
 ; CHECK-MSVC64-I128:       # %bb.0:
-; CHECK-MSVC64-I128-NEXT:    movq 40(%rsp), %rax
-; CHECK-MSVC64-I128-NEXT:    movq 48(%rsp), %rdx
+; CHECK-MSVC64-I128-NEXT:    movq %rcx, %rax
+; CHECK-MSVC64-I128-NEXT:    movq 48(%rsp), %rcx
+; CHECK-MSVC64-I128-NEXT:    movq 56(%rsp), %rdx
+; CHECK-MSVC64-I128-NEXT:    movq %rdx, 8(%rax)
+; CHECK-MSVC64-I128-NEXT:    movq %rcx, (%rax)
 ; CHECK-MSVC64-I128-NEXT:    retq
 ;
 ; CHECK-MINGW-F128-LABEL: leading_args:
@@ -368,8 +377,11 @@ define PrimTy @leading_args(i64 %_0, i64 %_1, i64 %_2, i64 %_3, PrimTy %x) nounw
 ;
 ; CHECK-MINGW-I128-LABEL: leading_args:
 ; CHECK-MINGW-I128:       # %bb.0:
-; CHECK-MINGW-I128-NEXT:    movq 40(%rsp), %rax
-; CHECK-MINGW-I128-NEXT:    movq 48(%rsp), %rdx
+; CHECK-MINGW-I128-NEXT:    movq %rcx, %rax
+; CHECK-MINGW-I128-NEXT:    movq 48(%rsp), %rcx
+; CHECK-MINGW-I128-NEXT:    movq 56(%rsp), %rdx
+; CHECK-MINGW-I128-NEXT:    movq %rdx, 8(%rax)
+; CHECK-MINGW-I128-NEXT:    movq %rcx, (%rax)
 ; CHECK-MINGW-I128-NEXT:    retq
 ;
 ; CHECK-X86-LABEL: leading_args:
@@ -438,8 +450,9 @@ define PrimTy @many_leading_args(i64 %_0, i64 %_1, i64 %_2, i64 %_3, i64 %_4, Pr
 ;
 ; CHECK-MSVC64-I128-LABEL: many_leading_args:
 ; CHECK-MSVC64-I128:       # %bb.0:
-; CHECK-MSVC64-I128-NEXT:    movq 64(%rsp), %rax
-; CHECK-MSVC64-I128-NEXT:    movq 72(%rsp), %rdx
+; CHECK-MSVC64-I128-NEXT:    movq %rcx, %rax
+; CHECK-MSVC64-I128-NEXT:    movaps 72(%rsp), %xmm0
+; CHECK-MSVC64-I128-NEXT:    movaps %xmm0, (%rcx)
 ; CHECK-MSVC64-I128-NEXT:    retq
 ;
 ; CHECK-MINGW-F128-LABEL: many_leading_args:
@@ -452,8 +465,9 @@ define PrimTy @many_leading_args(i64 %_0, i64 %_1, i64 %_2, i64 %_3, i64 %_4, Pr
 ;
 ; CHECK-MINGW-I128-LABEL: many_leading_args:
 ; CHECK-MINGW-I128:       # %bb.0:
-; CHECK-MINGW-I128-NEXT:    movq 64(%rsp), %rax
-; CHECK-MINGW-I128-NEXT:    movq 72(%rsp), %rdx
+; CHECK-MINGW-I128-NEXT:    movq %rcx, %rax
+; CHECK-MINGW-I128-NEXT:    movaps 72(%rsp), %xmm0
+; CHECK-MINGW-I128-NEXT:    movaps %xmm0, (%rcx)
 ; CHECK-MINGW-I128-NEXT:    retq
 ;
 ; CHECK-X86-LABEL: many_leading_args:
@@ -520,8 +534,9 @@ define PrimTy @trailing_arg(i64 %_0, i64 %_1, i64 %_2, i64 %_3, i64 %_4, PrimTy
 ;
 ; CHECK-MSVC64-I128-LABEL: trailing_arg:
 ; CHECK-MSVC64-I128:       # %bb.0:
-; CHECK-MSVC64-I128-NEXT:    movq 48(%rsp), %rax
-; CHECK-MSVC64-I128-NEXT:    movq 56(%rsp), %rdx
+; CHECK-MSVC64-I128-NEXT:    movq %rcx, %rax
+; CHECK-MSVC64-I128-NEXT:    movaps 56(%rsp), %xmm0
+; CHECK-MSVC64-I128-NEXT:    movaps %xmm0, (%rcx)
 ; CHECK-MSVC64-I128-NEXT:    retq
 ;
 ; CHECK-MINGW-F128-LABEL: trailing_arg:
@@ -534,8 +549,9 @@ define PrimTy @trailing_arg(i64 %_0, i64 %_1, i64 %_2, i64 %_3, i64 %_4, PrimTy
 ;
 ; CHECK-MINGW-I128-LABEL: trailing_arg:
 ; CHECK-MINGW-I128:       # %bb.0:
-; CHECK-MINGW-I128-NEXT:    movq 48(%rsp), %rax
-; CHECK-MINGW-I128-NEXT:    movq 56(%rsp), %rdx
+; CHECK-MINGW-I128-NEXT:    movq %rcx, %rax
+; CHECK-MINGW-I128-NEXT:    movaps 56(%rsp), %xmm0
+; CHECK-MINGW-I128-NEXT:    movaps %xmm0, (%rcx)
 ; CHECK-MINGW-I128-NEXT:    retq
 ;
 ; CHECK-X86-LABEL: trailing_arg:
@@ -609,9 +625,12 @@ define void @call_first_arg(PrimTy %x) nounwind {
 ;
 ; CHECK-MSVC64-I128-LABEL: call_first_arg:
 ; CHECK-MSVC64-I128:       # %bb.0:
-; CHECK-MSVC64-I128-NEXT:    subq $40, %rsp
+; CHECK-MSVC64-I128-NEXT:    subq $56, %rsp
+; CHECK-MSVC64-I128-NEXT:    movq %rdx, %r8
+; CHECK-MSVC64-I128-NEXT:    movq %rcx, %rdx
+; CHECK-MSVC64-I128-NEXT:    leaq 32(%rsp), %rcx
 ; CHECK-MSVC64-I128-NEXT:    callq first_arg
-; CHECK-MSVC64-I128-NEXT:    addq $40, %rsp
+; CHECK-MSVC64-I128-NEXT:    addq $56, %rsp
 ; CHECK-MSVC64-I128-NEXT:    retq
 ;
 ; CHECK-MINGW-F128-LABEL: call_first_arg:
@@ -627,9 +646,12 @@ define void @call_first_arg(PrimTy %x) nounwind {
 ;
 ; CHECK-MINGW-I128-LABEL: call_first_arg:
 ; CHECK-MINGW-I128:       # %bb.0:
-; CHECK-MINGW-I128-NEXT:    subq $40, %rsp
+; CHECK-MINGW-I128-NEXT:    subq $56, %rsp
+; CHECK-MINGW-I128-NEXT:    movq %rdx, %r8
+; CHECK-MINGW-I128-NEXT:    movq %rcx, %rdx
+; CHECK-MINGW-I128-NEXT:    leaq 32(%rsp), %rcx
 ; CHECK-MINGW-I128-NEXT:    callq first_arg
-; CHECK-MINGW-I128-NEXT:    addq $40, %rsp
+; CHECK-MINGW-I128-NEXT:    addq $56, %rsp
 ; CHECK-MINGW-I128-NEXT:    retq
 ;
 ; CHECK-X86-LABEL: call_first_arg:
@@ -720,15 +742,16 @@ define void @call_leading_args(PrimTy %x) nounwind {
 ;
 ; CHECK-MSVC64-I128-LABEL: call_leading_args:
 ; CHECK-MSVC64-I128:       # %bb.0:
-; CHECK-MSVC64-I128-NEXT:    subq $56, %rsp
-; CHECK-MSVC64-I128-NEXT:    movq %rdx, 40(%rsp)
-; CHECK-MSVC64-I128-NEXT:    movq %rcx, 32(%rsp)
-; CHECK-MSVC64-I128-NEXT:    xorl %ecx, %ecx
+; CHECK-MSVC64-I128-NEXT:    subq $88, %rsp
+; CHECK-MSVC64-I128-NEXT:    movq %rdx, 48(%rsp)
+; CHECK-MSVC64-I128-NEXT:    movq %rcx, 40(%rsp)
+; CHECK-MSVC64-I128-NEXT:    movq $0, 32(%rsp)
+; CHECK-MSVC64-I128-NEXT:    leaq 64(%rsp), %rcx
 ; CHECK-MSVC64-I128-NEXT:    xorl %edx, %edx
 ; CHECK-MSVC64-I128-NEXT:    xorl %r8d, %r8d
 ; CHECK-MSVC64-I128-NEXT:    xorl %r9d, %r9d
 ; CHECK-MSVC64-I128-NEXT:    callq leading_args
-; CHECK-MSVC64-I128-NEXT:    addq $56, %rsp
+; CHECK-MSVC64-I128-NEXT:    addq $88, %rsp
 ; CHECK-MSVC64-I128-NEXT:    retq
 ;
 ; CHECK-MINGW-F128-LABEL: call_leading_args:
@@ -749,15 +772,16 @@ define void @call_leading_args(PrimTy %x) nounwind {
 ;
 ; CHECK-MINGW-I128-LABEL: call_leading_args:
 ; CHECK-MINGW-I128:       # %bb.0:
-; CHECK-MINGW-I128-NEXT:    subq $56, %rsp
-; CHECK-MINGW-I128-NEXT:    movq %rdx, 40(%rsp)
-; CHECK-MINGW-I128-NEXT:    movq %rcx, 32(%rsp)
-; CHECK-MINGW-I128-NEXT:    xorl %ecx, %ecx
+; CHECK-MINGW-I128-NEXT:    subq $88, %rsp
+; CHECK-MINGW-I128-NEXT:    movq %rdx, 48(%rsp)
+; CHECK-MINGW-I128-NEXT:    movq %rcx, 40(%rsp)
+; CHECK-MINGW-I128-NEXT:    movq $0, 32(%rsp)
+; CHECK-MINGW-I128-NEXT:    leaq 64(%rsp), %rcx
 ; CHECK-MINGW-I128-NEXT:    xorl %edx, %edx
 ; CHECK-MINGW-I128-NEXT:    xorl %r8d, %r8d
 ; CHECK-MINGW-I128-NEXT:    xorl %r9d, %r9d
 ; CHECK-MINGW-I128-NEXT:    callq leading_args
-; CHECK-MINGW-I128-NEXT:    addq $56, %rsp
+; CHECK-MINGW-I128-NEXT:    addq $88, %rsp
 ; CHECK-MINGW-I128-NEXT:    retq
 ;
 ; CHECK-X86-LABEL: call_leading_args:
@@ -875,17 +899,18 @@ define void @call_many_leading_args(PrimTy %x) nounwind {
 ;
 ; CHECK-MSVC64-I128-LABEL: call_many_leading_args:
 ; CHECK-MSVC64-I128:       # %bb.0:
-; CHECK-MSVC64-I128-NEXT:    subq $72, %rsp
-; CHECK-MSVC64-I128-NEXT:    movq %rdx, 56(%rsp)
-; CHECK-MSVC64-I128-NEXT:    movq %rcx, 48(%rsp)
+; CHECK-MSVC64-I128-NEXT:    subq $104, %rsp
+; CHECK-MSVC64-I128-NEXT:    movq %rdx, 64(%rsp)
+; CHECK-MSVC64-I128-NEXT:    movq %rcx, 56(%rsp)
+; CHECK-MSVC64-I128-NEXT:    movq $0, 48(%rsp)
 ; CHECK-MSVC64-I128-NEXT:    movq $0, 40(%rsp)
 ; CHECK-MSVC64-I128-NEXT:    movq $0, 32(%rsp)
-; CHECK-MSVC64-I128-NEXT:    xorl %ecx, %ecx
+; CHECK-MSVC64-I128-NEXT:    leaq 80(%rsp), %rcx
 ; CHECK-MSVC64-I128-NEXT:    xorl %edx, %edx
 ; CHECK-MSVC64-I128-NEXT:    xorl %r8d, %r8d
 ; CHECK-MSVC64-I128-NEXT:    xorl %r9d, %r9d
 ; CHECK-MSVC64-I128-NEXT:    callq many_leading_args
-; CHECK-MSVC64-I128-NEXT:    addq $72, %rsp
+; CHECK-MSVC64-I128-NEXT:    addq $104, %rsp
 ; CHECK-MSVC64-I128-NEXT:    retq
 ;
 ; CHECK-MINGW-F128-LABEL: call_many_leading_args:
@@ -910,17 +935,18 @@ define void @call_many_leading_args(PrimTy %x) nounwind {
 ;
 ; CHECK-MINGW-I128-LABEL: call_many_leading_args:
 ; CHECK-MINGW-I128:       # %bb.0:
-; CHECK-MINGW-I128-NEXT:    subq $72, %rsp
-; CHECK-MINGW-I128-NEXT:    movq %rdx, 56(%rsp)
-; CHECK-MINGW-I128-NEXT:    movq %rcx, 48(%rsp)
+; CHECK-MINGW-I128-NEXT:    subq $104, %rsp
+; CHECK-MINGW-I128-NEXT:    movq %rdx, 64(%rsp)
+; CHECK-MINGW-I128-NEXT:    movq %rcx, 56(%rsp)
+; CHECK-MINGW-I128-NEXT:    movq $0, 48(%rsp)
 ; CHECK-MINGW-I128-NEXT:    movq $0, 40(%rsp)
 ; CHECK-MINGW-I128-NEXT:    movq $0, 32(%rsp)
-; CHECK-MINGW-I128-NEXT:    xorl %ecx, %ecx
+; CHECK-MINGW-I128-NEXT:    leaq 80(%rsp), %rcx
 ; CHECK-MINGW-I128-NEXT:    xorl %edx, %edx
 ; CHECK-MINGW-I128-NEXT:    xorl %r8d, %r8d
 ; CHECK-MINGW-I128-NEXT:    xorl %r9d, %r9d
 ; CHECK-MINGW-I128-NEXT:    callq many_leading_args
-; CHECK-MINGW-I128-NEXT:    addq $72, %rsp
+; CHECK-MINGW-I128-NEXT:    addq $104, %rsp
 ; CHECK-MINGW-I128-NEXT:    retq
 ;
 ; CHECK-X86-LABEL: call_many_leading_args:
@@ -1035,15 +1061,16 @@ define void @call_trailing_arg(PrimTy %x) nounwind {
 ;
 ; CHECK-MSVC64-I128-LABEL: call_trailing_arg:
 ; CHECK-MSVC64-I128:       # %bb.0:
-; CHECK-MSVC64-I128-NEXT:    subq $56, %rsp
-; CHECK-MSVC64-I128-NEXT:    movq %rdx, 40(%rsp)
-; CHECK-MSVC64-I128-NEXT:    movq %rcx, 32(%rsp)
-; CHECK-MSVC64-I128-NEXT:    xorl %ecx, %ecx
+; CHECK-MSVC64-I128-NEXT:    subq $88, %rsp
+; CHECK-MSVC64-I128-NEXT:    movq %rdx, 48(%rsp)
+; CHECK-MSVC64-I128-NEXT:    movq %rcx, 40(%rsp)
+; CHECK-MSVC64-I128-NEXT:    movq $0, 32(%rsp)
+; CHECK-MSVC64-I128-NEXT:    leaq 64(%rsp), %rcx
 ; CHECK-MSVC64-I128-NEXT:    xorl %edx, %edx
 ; CHECK-MSVC64-I128-NEXT:    xorl %r8d, %r8d
 ; CHECK-MSVC64-I128-NEXT:    xorl %r9d, %r9d
 ; CHECK-MSVC64-I128-NEXT:    callq trailing_arg
-; CHECK-MSVC64-I128-NEXT:    addq $56, %rsp
+; CHECK-MSVC64-I128-NEXT:    addq $88, %rsp
 ; CHECK-MSVC64-I128-NEXT:    retq
 ;
 ; CHECK-MINGW-F128-LABEL: call_trailing_arg:
@@ -1064,15 +1091,16 @@ define void @call_trailing_arg(PrimTy %x) nounwind {
 ;
 ; CHECK-MINGW-I128-LABEL: call_trailing_arg:
 ; CHECK-MINGW-I128:       # %bb.0:
-; CHECK-MINGW-I128-NEXT:    subq $56, %rsp
-; CHECK-MINGW-I128-NEXT:    movq %rdx, 40(%rsp)
-; CHECK-MINGW-I128-NEXT:    movq %rcx, 32(%rsp)
-; CHECK-MINGW-I128-NEXT:    xorl %ecx, %ecx
+; CHECK-MINGW-I128-NEXT:    subq $88, %rsp
+; CHECK-MINGW-I128-NEXT:    movq %rdx, 48(%rsp)
+; CHECK-MINGW-I128-NEXT:    movq %rcx, 40(%rsp)
+; CHECK-MINGW-I128-NEXT:    movq $0, 32(%rsp)
+; CHECK-MINGW-I128-NEXT:    leaq 64(%rsp), %rcx
 ; CHECK-MINGW-I128-NEXT:    xorl %edx, %edx
 ; CHECK-MINGW-I128-NEXT:    xorl %r8d, %r8d
 ; CHECK-MINGW-I128-NEXT:    xorl %r9d, %r9d
 ; CHECK-MINGW-I128-NEXT:    callq trailing_arg
-; CHECK-MINGW-I128-NEXT:    addq $56, %rsp
+; CHECK-MINGW-I128-NEXT:    addq $88, %rsp
 ; CHECK-MINGW-I128-NEXT:    retq
 ;
 ; CHECK-X86-LABEL: call_trailing_arg:
diff --git a/llvm/test/CodeGen/X86/i128-fpconv-win64-strict.ll b/llvm/test/CodeGen/X86/i128-fpconv-win64-strict.ll
index fe506cc75cff69..9f8a7757b0c73c 100644
--- a/llvm/test/CodeGen/X86/i128-fpconv-win64-strict.ll
+++ b/llvm/test/CodeGen/X86/i128-fpconv-win64-strict.ll
@@ -6,10 +6,12 @@
 define i64 @double_to_i128(double %d) nounwind strictfp {
 ; WIN64-LABEL: double_to_i128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $40, %rsp
+; WIN64-NEXT:    subq $56, %rsp
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT:    movaps %xmm0, %xmm1
 ; WIN64-NEXT:    callq __fixdfti
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $40, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $56, %rsp
 ; WIN64-NEXT:    retq
   %1 = tail call i128 @llvm.experimental.constrained.fptosi.i128.f64(double %d, metadata !"fpexcept.strict")
   %2 = trunc i128 %1 to i64
@@ -19,10 +21,12 @@ define i64 @double_to_i128(double %d) nounwind strictfp {
 define i64 @double_to_ui128(double %d) nounwind strictfp {
 ; WIN64-LABEL: double_to_ui128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $40, %rsp
+; WIN64-NEXT:    subq $56, %rsp
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT:    movaps %xmm0, %xmm1
 ; WIN64-NEXT:    callq __fixunsdfti
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $40, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $56, %rsp
 ; WIN64-NEXT:    retq
   %1 = tail call i128 @llvm.experimental.constrained.fptoui.i128.f64(double %d, metadata !"fpexcept.strict")
   %2 = trunc i128 %1 to i64
@@ -32,10 +36,12 @@ define i64 @double_to_ui128(double %d) nounwind strictfp {
 define i64 @float_to_i128(float %d) nounwind strictfp {
 ; WIN64-LABEL: float_to_i128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $40, %rsp
+; WIN64-NEXT:    subq $56, %rsp
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT:    movaps %xmm0, %xmm1
 ; WIN64-NEXT:    callq __fixsfti
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $40, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $56, %rsp
 ; WIN64-NEXT:    retq
   %1 = tail call i128 @llvm.experimental.constrained.fptosi.i128.f32(float %d, metadata !"fpexcept.strict")
   %2 = trunc i128 %1 to i64
@@ -45,10 +51,12 @@ define i64 @float_to_i128(float %d) nounwind strictfp {
 define i64 @float_to_ui128(float %d) nounwind strictfp {
 ; WIN64-LABEL: float_to_ui128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $40, %rsp
+; WIN64-NEXT:    subq $56, %rsp
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT:    movaps %xmm0, %xmm1
 ; WIN64-NEXT:    callq __fixunssfti
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $40, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $56, %rsp
 ; WIN64-NEXT:    retq
   %1 = tail call i128 @llvm.experimental.constrained.fptoui.i128.f32(float %d, metadata !"fpexcept.strict")
   %2 = trunc i128 %1 to i64
@@ -58,14 +66,15 @@ define i64 @float_to_ui128(float %d) nounwind strictfp {
 define i64 @longdouble_to_i128(ptr nocapture readonly %0) nounwind strictfp {
 ; WIN64-LABEL: longdouble_to_i128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $56, %rsp
+; WIN64-NEXT:    subq $72, %rsp
 ; WIN64-NEXT:    fldt (%rcx)
 ; WIN64-NEXT:    fstpt {{[0-9]+}}(%rsp)
 ; WIN64-NEXT:    wait
 ; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
 ; WIN64-NEXT:    callq __fixxfti
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $56, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $72, %rsp
 ; WIN64-NEXT:    retq
   %2 = load x86_fp80, ptr %0, align 16
   %3 = tail call i128 @llvm.experimental.constrained.fptosi.i128.f80(x86_fp80 %2, metadata !"fpexcept.strict")
@@ -76,14 +85,15 @@ define i64 @longdouble_to_i128(ptr nocapture readonly %0) nounwind strictfp {
 define i64 @longdouble_to_ui128(ptr nocapture readonly %0) nounwind strictfp {
 ; WIN64-LABEL: longdouble_to_ui128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $56, %rsp
+; WIN64-NEXT:    subq $72, %rsp
 ; WIN64-NEXT:    fldt (%rcx)
 ; WIN64-NEXT:    fstpt {{[0-9]+}}(%rsp)
 ; WIN64-NEXT:    wait
 ; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
 ; WIN64-NEXT:    callq __fixunsxfti
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $56, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $72, %rsp
 ; WIN64-NEXT:    retq
   %2 = load x86_fp80, ptr %0, align 16
   %3 = tail call i128 @llvm.experimental.constrained.fptoui.i128.f80(x86_fp80 %2, metadata !"fpexcept.strict")
diff --git a/llvm/test/CodeGen/X86/i128-fpconv-win64.ll b/llvm/test/CodeGen/X86/i128-fpconv-win64.ll
index b036a3b0004bb7..4c7ab056a90b4e 100644
--- a/llvm/test/CodeGen/X86/i128-fpconv-win64.ll
+++ b/llvm/test/CodeGen/X86/i128-fpconv-win64.ll
@@ -6,10 +6,12 @@
 define i64 @double_to_i128(double %d) nounwind {
 ; WIN64-LABEL: double_to_i128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $40, %rsp
+; WIN64-NEXT:    subq $56, %rsp
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT:    movaps %xmm0, %xmm1
 ; WIN64-NEXT:    callq __fixdfti
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $40, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $56, %rsp
 ; WIN64-NEXT:    retq
   %1 = fptosi double %d to i128
   %2 = trunc i128 %1 to i64
@@ -19,10 +21,12 @@ define i64 @double_to_i128(double %d) nounwind {
 define i64 @double_to_ui128(double %d) nounwind {
 ; WIN64-LABEL: double_to_ui128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $40, %rsp
+; WIN64-NEXT:    subq $56, %rsp
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT:    movaps %xmm0, %xmm1
 ; WIN64-NEXT:    callq __fixunsdfti
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $40, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $56, %rsp
 ; WIN64-NEXT:    retq
   %1 = fptoui double %d to i128
   %2 = trunc i128 %1 to i64
@@ -32,10 +36,12 @@ define i64 @double_to_ui128(double %d) nounwind {
 define i64 @float_to_i128(float %d) nounwind {
 ; WIN64-LABEL: float_to_i128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $40, %rsp
+; WIN64-NEXT:    subq $56, %rsp
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT:    movaps %xmm0, %xmm1
 ; WIN64-NEXT:    callq __fixsfti
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $40, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $56, %rsp
 ; WIN64-NEXT:    retq
   %1 = fptosi float %d to i128
   %2 = trunc i128 %1 to i64
@@ -45,10 +51,12 @@ define i64 @float_to_i128(float %d) nounwind {
 define i64 @float_to_ui128(float %d) nounwind {
 ; WIN64-LABEL: float_to_ui128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $40, %rsp
+; WIN64-NEXT:    subq $56, %rsp
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT:    movaps %xmm0, %xmm1
 ; WIN64-NEXT:    callq __fixunssfti
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $40, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $56, %rsp
 ; WIN64-NEXT:    retq
   %1 = fptoui float %d to i128
   %2 = trunc i128 %1 to i64
@@ -58,13 +66,14 @@ define i64 @float_to_ui128(float %d) nounwind {
 define i64 @longdouble_to_i128(ptr nocapture readonly %0) nounwind {
 ; WIN64-LABEL: longdouble_to_i128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $56, %rsp
+; WIN64-NEXT:    subq $72, %rsp
 ; WIN64-NEXT:    fldt (%rcx)
 ; WIN64-NEXT:    fstpt {{[0-9]+}}(%rsp)
 ; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
 ; WIN64-NEXT:    callq __fixxfti
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $56, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $72, %rsp
 ; WIN64-NEXT:    retq
   %2 = load x86_fp80, ptr %0, align 16
   %3 = fptosi x86_fp80 %2 to i128
@@ -75,13 +84,14 @@ define i64 @longdouble_to_i128(ptr nocapture readonly %0) nounwind {
 define i64 @longdouble_to_ui128(ptr nocapture readonly %0) nounwind {
 ; WIN64-LABEL: longdouble_to_ui128:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    subq $56, %rsp
+; WIN64-NEXT:    subq $72, %rsp
 ; WIN64-NEXT:    fldt (%rcx)
 ; WIN64-NEXT:    fstpt {{[0-9]+}}(%rsp)
 ; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; WIN64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
 ; WIN64-NEXT:    callq __fixunsxfti
-; WIN64-NEXT:    movq %xmm0, %rax
-; WIN64-NEXT:    addq $56, %rsp
+; WIN64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; WIN64-NEXT:    addq $72, %rsp
 ; WIN64-NEXT:    retq
   %2 = load x86_fp80, ptr %0, align 16
   %3 = fptoui x86_fp80 %2 to i128
diff --git a/llvm/test/CodeGen/X86/i128-ret.ll b/llvm/test/CodeGen/X86/i128-ret.ll
index 6d208bdef4f343..773bc207e15975 100644
--- a/llvm/test/CodeGen/X86/i128-ret.ll
+++ b/llvm/test/CodeGen/X86/i128-ret.ll
@@ -1,7 +1,10 @@
-; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-win32 | FileCheck %s
-; CHECK: movq ([[A0:%rdi|%rcx]]), %rax
-; CHECK: movq 8([[A0]]), %rdx
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefix=SYSV
+; RUN: llc < %s -mtriple=x86_64-win32 | FileCheck %s --check-prefix=WIN64
+; SYSV: movq (%rdi), %rax
+; SYSV: movq 8(%rdi), %rdx
+; WIN64: movq %rcx, %rax
+; WIN64: movaps (%rdx), %xmm0
+; WIN64: movaps %xmm0, (%rcx)
 
 define i128 @test(ptr%P) {
         %A = load i128, ptr %P
diff --git a/llvm/test/CodeGen/X86/mul128.ll b/llvm/test/CodeGen/X86/mul128.ll
index 90d5fc0cd37707..87fefa6cd8d121 100644
--- a/llvm/test/CodeGen/X86/mul128.ll
+++ b/llvm/test/CodeGen/X86/mul128.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-windows-gnu | FileCheck %s --check-prefix=WIN64
 ; RUN: llc < %s -mtriple=i386-unknown | FileCheck %s --check-prefix=X86
 
 define i128 @foo(i128 %t, i128 %u) {
@@ -14,6 +15,20 @@ define i128 @foo(i128 %t, i128 %u) {
 ; X64-NEXT:    addq %r8, %rdx
 ; X64-NEXT:    retq
 ;
+; WIN64-LABEL: foo:
+; WIN64:       # %bb.0:
+; WIN64-NEXT:    movq %rdx, %r10
+; WIN64-NEXT:    imulq %r9, %r8
+; WIN64-NEXT:    movq %rdx, %rax
+; WIN64-NEXT:    mulq %r9
+; WIN64-NEXT:    imulq {{[0-9]+}}(%rsp), %r10
+; WIN64-NEXT:    addq %r8, %rdx
+; WIN64-NEXT:    addq %r10, %rdx
+; WIN64-NEXT:    movq %rdx, 8(%rcx)
+; WIN64-NEXT:    movq %rax, (%rcx)
+; WIN64-NEXT:    movq %rcx, %rax
+; WIN64-NEXT:    retq
+;
 ; X86-LABEL: foo:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %ebp
@@ -111,6 +126,16 @@ define void @PR13897() nounwind {
 ; X64-NEXT:    movq %rax, aaa(%rip)
 ; X64-NEXT:    retq
 ;
+; WIN64-LABEL: PR13897:
+; WIN64:       # %bb.0: # %"0x0"
+; WIN64-NEXT:    movq bbb(%rip), %rax
+; WIN64-NEXT:    movl %eax, %ecx
+; WIN64-NEXT:    shlq $32, %rax
+; WIN64-NEXT:    orq %rcx, %rax
+; WIN64-NEXT:    movq %rax, aaa+8(%rip)
+; WIN64-NEXT:    movq %rax, aaa(%rip)
+; WIN64-NEXT:    retq
+;
 ; X86-LABEL: PR13897:
 ; X86:       # %bb.0: # %"0x0"
 ; X86-NEXT:    movl bbb, %eax
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-non-integer.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-non-integer.ll
index 142ebefa5439a4..fc66280e7c5301 100644
--- a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-non-integer.ll
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-non-integer.ll
@@ -1,7 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=CHECK,CHECK64
-; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=CHECK,CHECK32
-; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=x86_64-linux-gnu -mattr=+cx16,-avx | FileCheck %s --check-prefixes=CX16
+; RUN: opt -S %s -passes="require<libcall-lowering-info>,atomic-expand" -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=CHECK,CHECK64
+; RUN: opt -S %s -passes="require<libcall-lowering-info>,atomic-expand" -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=CHECK,CHECK32
+; RUN: opt -S %s -passes="require<libcall-lowering-info>,atomic-expand" -mtriple=x86_64-linux-gnu -mattr=+cx16,-avx | FileCheck %s --check-prefixes=CX16
+; RUN: opt -S %s -passes="require<libcall-lowering-info>,atomic-expand" -mtriple=x86_64-windows-gnu | FileCheck %s --check-prefixes=WIN64
 
 ; This file tests AtomicExpand's non-integer atomic type conversions and
 ; X86-specific expansion choices. If X86 stops using this functionality, please
@@ -19,6 +20,12 @@ define float @float_load_expand(ptr %ptr) {
 ; CX16-NEXT:    [[TMP1:%.*]] = load atomic i32, ptr [[PTR]] unordered, align 4
 ; CX16-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
 ; CX16-NEXT:    ret float [[TMP2]]
+;
+; WIN64-LABEL: define float @float_load_expand(
+; WIN64-SAME: ptr [[PTR:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = load atomic i32, ptr [[PTR]] unordered, align 4
+; WIN64-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
+; WIN64-NEXT:    ret float [[TMP2]]
 ;
   %res = load atomic float, ptr %ptr unordered, align 4
   ret float %res
@@ -36,6 +43,12 @@ define float @float_load_expand_seq_cst(ptr %ptr) {
 ; CX16-NEXT:    [[TMP1:%.*]] = load atomic i32, ptr [[PTR]] seq_cst, align 4
 ; CX16-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
 ; CX16-NEXT:    ret float [[TMP2]]
+;
+; WIN64-LABEL: define float @float_load_expand_seq_cst(
+; WIN64-SAME: ptr [[PTR:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = load atomic i32, ptr [[PTR]] seq_cst, align 4
+; WIN64-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
+; WIN64-NEXT:    ret float [[TMP2]]
 ;
   %res = load atomic float, ptr %ptr seq_cst, align 4
   ret float %res
@@ -53,6 +66,12 @@ define float @float_load_expand_vol(ptr %ptr) {
 ; CX16-NEXT:    [[TMP1:%.*]] = load atomic volatile i32, ptr [[PTR]] unordered, align 4
 ; CX16-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
 ; CX16-NEXT:    ret float [[TMP2]]
+;
+; WIN64-LABEL: define float @float_load_expand_vol(
+; WIN64-SAME: ptr [[PTR:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = load atomic volatile i32, ptr [[PTR]] unordered, align 4
+; WIN64-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
+; WIN64-NEXT:    ret float [[TMP2]]
 ;
   %res = load atomic volatile float, ptr %ptr unordered, align 4
   ret float %res
@@ -70,6 +89,12 @@ define float @float_load_expand_addr1(ptr addrspace(1) %ptr) {
 ; CX16-NEXT:    [[TMP1:%.*]] = load atomic i32, ptr addrspace(1) [[PTR]] unordered, align 4
 ; CX16-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
 ; CX16-NEXT:    ret float [[TMP2]]
+;
+; WIN64-LABEL: define float @float_load_expand_addr1(
+; WIN64-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = load atomic i32, ptr addrspace(1) [[PTR]] unordered, align 4
+; WIN64-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
+; WIN64-NEXT:    ret float [[TMP2]]
 ;
   %res = load atomic float, ptr addrspace(1) %ptr unordered, align 4
   ret float %res
@@ -87,6 +112,12 @@ define void @float_store_expand(ptr %ptr, float %v) {
 ; CX16-NEXT:    [[TMP1:%.*]] = bitcast float [[V]] to i32
 ; CX16-NEXT:    store atomic i32 [[TMP1]], ptr [[PTR]] unordered, align 4
 ; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @float_store_expand(
+; WIN64-SAME: ptr [[PTR:%.*]], float [[V:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = bitcast float [[V]] to i32
+; WIN64-NEXT:    store atomic i32 [[TMP1]], ptr [[PTR]] unordered, align 4
+; WIN64-NEXT:    ret void
 ;
   store atomic float %v, ptr %ptr unordered, align 4
   ret void
@@ -104,6 +135,12 @@ define void @float_store_expand_seq_cst(ptr %ptr, float %v) {
 ; CX16-NEXT:    [[TMP1:%.*]] = bitcast float [[V]] to i32
 ; CX16-NEXT:    store atomic i32 [[TMP1]], ptr [[PTR]] seq_cst, align 4
 ; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @float_store_expand_seq_cst(
+; WIN64-SAME: ptr [[PTR:%.*]], float [[V:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = bitcast float [[V]] to i32
+; WIN64-NEXT:    store atomic i32 [[TMP1]], ptr [[PTR]] seq_cst, align 4
+; WIN64-NEXT:    ret void
 ;
   store atomic float %v, ptr %ptr seq_cst, align 4
   ret void
@@ -121,6 +158,12 @@ define void @float_store_expand_vol(ptr %ptr, float %v) {
 ; CX16-NEXT:    [[TMP1:%.*]] = bitcast float [[V]] to i32
 ; CX16-NEXT:    store atomic volatile i32 [[TMP1]], ptr [[PTR]] unordered, align 4
 ; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @float_store_expand_vol(
+; WIN64-SAME: ptr [[PTR:%.*]], float [[V:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = bitcast float [[V]] to i32
+; WIN64-NEXT:    store atomic volatile i32 [[TMP1]], ptr [[PTR]] unordered, align 4
+; WIN64-NEXT:    ret void
 ;
   store atomic volatile float %v, ptr %ptr unordered, align 4
   ret void
@@ -138,6 +181,12 @@ define void @float_store_expand_addr1(ptr addrspace(1) %ptr, float %v) {
 ; CX16-NEXT:    [[TMP1:%.*]] = bitcast float [[V]] to i32
 ; CX16-NEXT:    store atomic i32 [[TMP1]], ptr addrspace(1) [[PTR]] unordered, align 4
 ; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @float_store_expand_addr1(
+; WIN64-SAME: ptr addrspace(1) [[PTR:%.*]], float [[V:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = bitcast float [[V]] to i32
+; WIN64-NEXT:    store atomic i32 [[TMP1]], ptr addrspace(1) [[PTR]] unordered, align 4
+; WIN64-NEXT:    ret void
 ;
   store atomic float %v, ptr addrspace(1) %ptr unordered, align 4
   ret void
@@ -176,6 +225,17 @@ define void @pointer_cmpxchg_expand(ptr %ptr, ptr %v) {
 ; CX16-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
 ; CX16-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
 ; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @pointer_cmpxchg_expand(
+; WIN64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
+; WIN64-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 0, i64 [[TMP1]] seq_cst monotonic, align 8
+; WIN64-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
+; WIN64-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; WIN64-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; WIN64-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
+; WIN64-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
+; WIN64-NEXT:    ret void
 ;
   cmpxchg ptr %ptr, ptr null, ptr %v seq_cst monotonic
   ret void
@@ -214,6 +274,17 @@ define void @pointer_cmpxchg_expand2(ptr %ptr, ptr %v) {
 ; CX16-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
 ; CX16-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
 ; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @pointer_cmpxchg_expand2(
+; WIN64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
+; WIN64-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 0, i64 [[TMP1]] release monotonic, align 8
+; WIN64-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
+; WIN64-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; WIN64-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; WIN64-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
+; WIN64-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
+; WIN64-NEXT:    ret void
 ;
   cmpxchg ptr %ptr, ptr null, ptr %v release monotonic
   ret void
@@ -252,6 +323,17 @@ define void @pointer_cmpxchg_expand3(ptr %ptr, ptr %v) {
 ; CX16-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
 ; CX16-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
 ; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @pointer_cmpxchg_expand3(
+; WIN64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
+; WIN64-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 0, i64 [[TMP1]] seq_cst seq_cst, align 8
+; WIN64-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
+; WIN64-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; WIN64-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; WIN64-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
+; WIN64-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
+; WIN64-NEXT:    ret void
 ;
   cmpxchg ptr %ptr, ptr null, ptr %v seq_cst seq_cst
   ret void
@@ -290,6 +372,17 @@ define void @pointer_cmpxchg_expand4(ptr %ptr, ptr %v) {
 ; CX16-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
 ; CX16-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
 ; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @pointer_cmpxchg_expand4(
+; WIN64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
+; WIN64-NEXT:    [[TMP2:%.*]] = cmpxchg weak ptr [[PTR]], i64 0, i64 [[TMP1]] seq_cst seq_cst, align 8
+; WIN64-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
+; WIN64-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; WIN64-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; WIN64-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
+; WIN64-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
+; WIN64-NEXT:    ret void
 ;
   cmpxchg weak ptr %ptr, ptr null, ptr %v seq_cst seq_cst
   ret void
@@ -328,6 +421,17 @@ define void @pointer_cmpxchg_expand5(ptr %ptr, ptr %v) {
 ; CX16-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
 ; CX16-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
 ; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @pointer_cmpxchg_expand5(
+; WIN64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
+; WIN64-NEXT:    [[TMP2:%.*]] = cmpxchg volatile ptr [[PTR]], i64 0, i64 [[TMP1]] seq_cst seq_cst, align 8
+; WIN64-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
+; WIN64-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; WIN64-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; WIN64-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
+; WIN64-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
+; WIN64-NEXT:    ret void
 ;
   cmpxchg volatile ptr %ptr, ptr null, ptr %v seq_cst seq_cst
   ret void
@@ -366,6 +470,17 @@ define void @pointer_cmpxchg_expand6(ptr addrspace(1) %ptr, ptr addrspace(2) %v)
 ; CX16-NEXT:    [[TMP6:%.*]] = insertvalue { ptr addrspace(2), i1 } poison, ptr addrspace(2) [[TMP5]], 0
 ; CX16-NEXT:    [[TMP7:%.*]] = insertvalue { ptr addrspace(2), i1 } [[TMP6]], i1 [[TMP4]], 1
 ; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @pointer_cmpxchg_expand6(
+; WIN64-SAME: ptr addrspace(1) [[PTR:%.*]], ptr addrspace(2) [[V:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(2) [[V]] to i64
+; WIN64-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 0, i64 [[TMP1]] seq_cst seq_cst, align 8
+; WIN64-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
+; WIN64-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; WIN64-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr addrspace(2)
+; WIN64-NEXT:    [[TMP6:%.*]] = insertvalue { ptr addrspace(2), i1 } poison, ptr addrspace(2) [[TMP5]], 0
+; WIN64-NEXT:    [[TMP7:%.*]] = insertvalue { ptr addrspace(2), i1 } [[TMP6]], i1 [[TMP4]], 1
+; WIN64-NEXT:    ret void
 ;
   cmpxchg ptr addrspace(1) %ptr, ptr addrspace(2) null, ptr addrspace(2) %v seq_cst seq_cst
   ret void
@@ -388,6 +503,13 @@ define <2 x ptr> @atomic_vec2_ptr_align(ptr %x) nounwind {
 ; CX16-SAME: ptr [[X:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CX16-NEXT:    [[RET:%.*]] = load atomic <2 x ptr>, ptr [[X]] acquire, align 16
 ; CX16-NEXT:    ret <2 x ptr> [[RET]]
+;
+; WIN64-LABEL: define <2 x ptr> @atomic_vec2_ptr_align(
+; WIN64-SAME: ptr [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; WIN64-NEXT:    [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[X]], i32 2)
+; WIN64-NEXT:    [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <2 x i64>
+; WIN64-NEXT:    [[TMP3:%.*]] = inttoptr <2 x i64> [[TMP2]] to <2 x ptr>
+; WIN64-NEXT:    ret <2 x ptr> [[TMP3]]
 ;
   %ret = load atomic <2 x ptr>, ptr %x acquire, align 16
   ret <2 x ptr> %ret
@@ -414,6 +536,13 @@ define <4 x ptr addrspace(270)> @atomic_vec4_ptr_align(ptr %x) nounwind {
 ; CX16-SAME: ptr [[X:%.*]]) #[[ATTR1]] {
 ; CX16-NEXT:    [[RET:%.*]] = load atomic <4 x ptr addrspace(270)>, ptr [[X]] acquire, align 16
 ; CX16-NEXT:    ret <4 x ptr addrspace(270)> [[RET]]
+;
+; WIN64-LABEL: define <4 x ptr addrspace(270)> @atomic_vec4_ptr_align(
+; WIN64-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT:    [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[X]], i32 2)
+; WIN64-NEXT:    [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <4 x i32>
+; WIN64-NEXT:    [[TMP3:%.*]] = inttoptr <4 x i32> [[TMP2]] to <4 x ptr addrspace(270)>
+; WIN64-NEXT:    ret <4 x ptr addrspace(270)> [[TMP3]]
 ;
   %ret = load atomic <4 x ptr addrspace(270)>, ptr %x acquire, align 16
   ret <4 x ptr addrspace(270)> %ret
@@ -429,6 +558,11 @@ define <2 x i16> @atomic_vec2_i16(ptr %x) nounwind {
 ; CX16-SAME: ptr [[X:%.*]]) #[[ATTR1]] {
 ; CX16-NEXT:    [[RET:%.*]] = load atomic <2 x i16>, ptr [[X]] acquire, align 8
 ; CX16-NEXT:    ret <2 x i16> [[RET]]
+;
+; WIN64-LABEL: define <2 x i16> @atomic_vec2_i16(
+; WIN64-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT:    [[RET:%.*]] = load atomic <2 x i16>, ptr [[X]] acquire, align 8
+; WIN64-NEXT:    ret <2 x i16> [[RET]]
 ;
   %ret = load atomic <2 x i16>, ptr %x acquire, align 8
   ret <2 x i16> %ret
@@ -444,6 +578,11 @@ define <2 x half> @atomic_vec2_half(ptr %x) nounwind {
 ; CX16-SAME: ptr [[X:%.*]]) #[[ATTR1]] {
 ; CX16-NEXT:    [[TMP2:%.*]] = load atomic <2 x half>, ptr [[X]] acquire, align 8
 ; CX16-NEXT:    ret <2 x half> [[TMP2]]
+;
+; WIN64-LABEL: define <2 x half> @atomic_vec2_half(
+; WIN64-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT:    [[RET:%.*]] = load atomic <2 x half>, ptr [[X]] acquire, align 8
+; WIN64-NEXT:    ret <2 x half> [[RET]]
 ;
   %ret = load atomic <2 x half>, ptr %x acquire, align 8
   ret <2 x half> %ret
@@ -471,6 +610,12 @@ define <4 x i32> @atomic_vec4_i32(ptr %x) nounwind {
 ; CX16-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
 ; CX16-NEXT:    [[TMP2:%.*]] = bitcast i128 [[LOADED]] to <4 x i32>
 ; CX16-NEXT:    ret <4 x i32> [[TMP2]]
+;
+; WIN64-LABEL: define <4 x i32> @atomic_vec4_i32(
+; WIN64-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT:    [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[X]], i32 2)
+; WIN64-NEXT:    [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <4 x i32>
+; WIN64-NEXT:    ret <4 x i32> [[TMP2]]
 ;
   %ret = load atomic <4 x i32>, ptr %x acquire, align 16
   ret <4 x i32> %ret
@@ -498,6 +643,12 @@ define <4 x float> @atomic_vec4_float(ptr %x) nounwind {
 ; CX16-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
 ; CX16-NEXT:    [[TMP2:%.*]] = bitcast i128 [[LOADED]] to <4 x float>
 ; CX16-NEXT:    ret <4 x float> [[TMP2]]
+;
+; WIN64-LABEL: define <4 x float> @atomic_vec4_float(
+; WIN64-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT:    [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[X]], i32 2)
+; WIN64-NEXT:    [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <4 x float>
+; WIN64-NEXT:    ret <4 x float> [[TMP2]]
 ;
   %ret = load atomic <4 x float>, ptr %x acquire, align 16
   ret <4 x float> %ret
@@ -528,6 +679,12 @@ define <2 x i64> @load_v2i64_cmpxchg(ptr %p) {
 ; CX16-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
 ; CX16-NEXT:    [[TMP2:%.*]] = bitcast i128 [[LOADED]] to <2 x i64>
 ; CX16-NEXT:    ret <2 x i64> [[TMP2]]
+;
+; WIN64-LABEL: define <2 x i64> @load_v2i64_cmpxchg(
+; WIN64-SAME: ptr [[P:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[P]], i32 5)
+; WIN64-NEXT:    [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <2 x i64>
+; WIN64-NEXT:    ret <2 x i64> [[TMP2]]
 ;
   %r = load atomic <2 x i64>, ptr %p seq_cst, align 16
   ret <2 x i64> %r
@@ -555,6 +712,12 @@ define <4 x i32> @load_v4i32_cmpxchg(ptr %p) {
 ; CX16-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
 ; CX16-NEXT:    [[TMP2:%.*]] = bitcast i128 [[LOADED]] to <4 x i32>
 ; CX16-NEXT:    ret <4 x i32> [[TMP2]]
+;
+; WIN64-LABEL: define <4 x i32> @load_v4i32_cmpxchg(
+; WIN64-SAME: ptr [[P:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[P]], i32 5)
+; WIN64-NEXT:    [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <4 x i32>
+; WIN64-NEXT:    ret <4 x i32> [[TMP2]]
 ;
   %r = load atomic <4 x i32>, ptr %p seq_cst, align 16
   ret <4 x i32> %r
@@ -582,6 +745,12 @@ define <16 x i8> @load_v16i8_cmpxchg(ptr %p) {
 ; CX16-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
 ; CX16-NEXT:    [[TMP2:%.*]] = bitcast i128 [[LOADED]] to <16 x i8>
 ; CX16-NEXT:    ret <16 x i8> [[TMP2]]
+;
+; WIN64-LABEL: define <16 x i8> @load_v16i8_cmpxchg(
+; WIN64-SAME: ptr [[P:%.*]]) {
+; WIN64-NEXT:    [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[P]], i32 5)
+; WIN64-NEXT:    [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <16 x i8>
+; WIN64-NEXT:    ret <16 x i8> [[TMP2]]
 ;
   %r = load atomic <16 x i8>, ptr %p seq_cst, align 16
   ret <16 x i8> %r
@@ -610,6 +779,11 @@ define i128 @load_i128_volatile_syncscope(ptr %p) {
 ; CX16-NEXT:    [[TMP1:%.*]] = cmpxchg volatile ptr [[P]], i128 0, i128 0 syncscope("singlethread") seq_cst seq_cst, align 16
 ; CX16-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
 ; CX16-NEXT:    ret i128 [[LOADED]]
+;
+; WIN64-LABEL: define i128 @load_i128_volatile_syncscope(
+; WIN64-SAME: ptr [[P:%.*]]) {
+; WIN64-NEXT:    [[VALUE:%.*]] = call i128 @__atomic_load_16(ptr [[P]], i32 5)
+; WIN64-NEXT:    ret i128 [[VALUE]]
 ;
   %v = load atomic volatile i128, ptr %p syncscope("singlethread") seq_cst, align 16
   ret i128 %v
@@ -642,11 +816,120 @@ define void @store_i128_volatile_syncscope(ptr %p, i128 %x) {
 ; CX16-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
 ; CX16:       [[ATOMICRMW_END]]:
 ; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @store_i128_volatile_syncscope(
+; WIN64-SAME: ptr [[P:%.*]], i128 [[X:%.*]]) {
+; WIN64-NEXT:    [[VALUE:%.*]] = alloca i128, align 16
+; WIN64-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VALUE]])
+; WIN64-NEXT:    store i128 [[X]], ptr [[VALUE]], align 16
+; WIN64-NEXT:    call void @__atomic_store_16(ptr [[P]], ptr [[VALUE]], i32 5)
+; WIN64-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VALUE]])
+; WIN64-NEXT:    ret void
 ;
   store atomic volatile i128 %x, ptr %p syncscope("singlethread") seq_cst, align 16
   ret void
 }
 
+define i128 @xchg_i128(ptr %p, i128 %x) {
+; CHECK64-LABEL: define i128 @xchg_i128(
+; CHECK64-SAME: ptr [[P:%.*]], i128 [[X:%.*]]) {
+; CHECK64-NEXT:    [[TMP1:%.*]] = call i128 @__atomic_exchange_16(ptr [[P]], i128 [[X]], i32 5)
+; CHECK64-NEXT:    ret i128 [[TMP1]]
+;
+; CHECK32-LABEL: define i128 @xchg_i128(
+; CHECK32-SAME: ptr [[P:%.*]], i128 [[X:%.*]]) {
+; CHECK32-NEXT:    [[TMP1:%.*]] = alloca i128, align 16
+; CHECK32-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK32-NEXT:    store i128 [[X]], ptr [[TMP1]], align 16
+; CHECK32-NEXT:    [[TMP2:%.*]] = alloca i128, align 16
+; CHECK32-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP2]])
+; CHECK32-NEXT:    call void @__atomic_exchange(i32 16, ptr [[P]], ptr [[TMP1]], ptr [[TMP2]], i32 5)
+; CHECK32-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK32-NEXT:    [[TMP3:%.*]] = load i128, ptr [[TMP2]], align 16
+; CHECK32-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP2]])
+; CHECK32-NEXT:    ret i128 [[TMP3]]
+;
+; CX16-LABEL: define i128 @xchg_i128(
+; CX16-SAME: ptr [[P:%.*]], i128 [[X:%.*]]) #[[ATTR0]] {
+; CX16-NEXT:    [[TMP1:%.*]] = load i128, ptr [[P]], align 16
+; CX16-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CX16:       [[ATOMICRMW_START]]:
+; CX16-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CX16-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[P]], i128 [[LOADED]], i128 [[X]] seq_cst seq_cst, align 16
+; CX16-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1
+; CX16-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP2]], 0
+; CX16-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CX16:       [[ATOMICRMW_END]]:
+; CX16-NEXT:    ret i128 [[NEWLOADED]]
+;
+; WIN64-LABEL: define i128 @xchg_i128(
+; WIN64-SAME: ptr [[P:%.*]], i128 [[X:%.*]]) {
+; WIN64-NEXT:    [[VALUE:%.*]] = alloca i128, align 16
+; WIN64-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VALUE]])
+; WIN64-NEXT:    store i128 [[X]], ptr [[VALUE]], align 16
+; WIN64-NEXT:    [[RESULT:%.*]] = call i128 @__atomic_exchange_16(ptr [[P]], ptr [[VALUE]], i32 5)
+; WIN64-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VALUE]])
+; WIN64-NEXT:    ret i128 [[RESULT]]
+;
+  %result = atomicrmw xchg ptr %p, i128 %x seq_cst, align 16
+  ret i128 %result
+}
+
+define { i128, i1 } @cmpxchg_i128(ptr %p, i128 %expected, i128 %desired) {
+; CHECK64-LABEL: define { i128, i1 } @cmpxchg_i128(
+; CHECK64-SAME: ptr [[P:%.*]], i128 [[EXPECTED:%.*]], i128 [[DESIRED:%.*]]) {
+; CHECK64-NEXT:    [[TMP1:%.*]] = alloca i128, align 16
+; CHECK64-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK64-NEXT:    store i128 [[EXPECTED]], ptr [[TMP1]], align 16
+; CHECK64-NEXT:    [[TMP2:%.*]] = call zeroext i1 @__atomic_compare_exchange_16(ptr [[P]], ptr [[TMP1]], i128 [[DESIRED]], i32 5, i32 5)
+; CHECK64-NEXT:    [[TMP3:%.*]] = load i128, ptr [[TMP1]], align 16
+; CHECK64-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK64-NEXT:    [[TMP4:%.*]] = insertvalue { i128, i1 } poison, i128 [[TMP3]], 0
+; CHECK64-NEXT:    [[TMP5:%.*]] = insertvalue { i128, i1 } [[TMP4]], i1 [[TMP2]], 1
+; CHECK64-NEXT:    ret { i128, i1 } [[TMP5]]
+;
+; CHECK32-LABEL: define { i128, i1 } @cmpxchg_i128(
+; CHECK32-SAME: ptr [[P:%.*]], i128 [[EXPECTED:%.*]], i128 [[DESIRED:%.*]]) {
+; CHECK32-NEXT:    [[TMP1:%.*]] = alloca i128, align 16
+; CHECK32-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK32-NEXT:    store i128 [[EXPECTED]], ptr [[TMP1]], align 16
+; CHECK32-NEXT:    [[TMP2:%.*]] = alloca i128, align 16
+; CHECK32-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP2]])
+; CHECK32-NEXT:    store i128 [[DESIRED]], ptr [[TMP2]], align 16
+; CHECK32-NEXT:    [[TMP3:%.*]] = call zeroext i1 @__atomic_compare_exchange(i32 16, ptr [[P]], ptr [[TMP1]], ptr [[TMP2]], i32 5, i32 5)
+; CHECK32-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP2]])
+; CHECK32-NEXT:    [[TMP4:%.*]] = load i128, ptr [[TMP1]], align 16
+; CHECK32-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK32-NEXT:    [[TMP5:%.*]] = insertvalue { i128, i1 } poison, i128 [[TMP4]], 0
+; CHECK32-NEXT:    [[TMP6:%.*]] = insertvalue { i128, i1 } [[TMP5]], i1 [[TMP3]], 1
+; CHECK32-NEXT:    ret { i128, i1 } [[TMP6]]
+;
+; CX16-LABEL: define { i128, i1 } @cmpxchg_i128(
+; CX16-SAME: ptr [[P:%.*]], i128 [[EXPECTED:%.*]], i128 [[DESIRED:%.*]]) #[[ATTR0]] {
+; CX16-NEXT:    [[RESULT:%.*]] = cmpxchg ptr [[P]], i128 [[EXPECTED]], i128 [[DESIRED]] seq_cst seq_cst, align 16
+; CX16-NEXT:    ret { i128, i1 } [[RESULT]]
+;
+; WIN64-LABEL: define { i128, i1 } @cmpxchg_i128(
+; WIN64-SAME: ptr [[P:%.*]], i128 [[EXPECTED:%.*]], i128 [[DESIRED:%.*]]) {
+; WIN64-NEXT:    [[EXPECTED_ADDR:%.*]] = alloca i128, align 16
+; WIN64-NEXT:    call void @llvm.lifetime.start.p0(ptr [[EXPECTED_ADDR]])
+; WIN64-NEXT:    store i128 [[EXPECTED]], ptr [[EXPECTED_ADDR]], align 16
+; WIN64-NEXT:    [[DESIRED_ADDR:%.*]] = alloca i128, align 16
+; WIN64-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DESIRED_ADDR]])
+; WIN64-NEXT:    store i128 [[DESIRED]], ptr [[DESIRED_ADDR]], align 16
+; WIN64-NEXT:    [[SUCCESS:%.*]] = call zeroext i1 @__atomic_compare_exchange_16(ptr [[P]], ptr [[EXPECTED_ADDR]], ptr [[DESIRED_ADDR]], i32 5, i32 5)
+; WIN64-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DESIRED_ADDR]])
+; WIN64-NEXT:    [[OLD:%.*]] = load i128, ptr [[EXPECTED_ADDR]], align 16
+; WIN64-NEXT:    call void @llvm.lifetime.end.p0(ptr [[EXPECTED_ADDR]])
+; WIN64-NEXT:    [[WITH_OLD:%.*]] = insertvalue { i128, i1 } poison, i128 [[OLD]], 0
+; WIN64-NEXT:    [[RESULT:%.*]] = insertvalue { i128, i1 } [[WITH_OLD]], i1 [[SUCCESS]], 1
+; WIN64-NEXT:    ret { i128, i1 } [[RESULT]]
+;
+  %result = cmpxchg ptr %p, i128 %expected, i128 %desired seq_cst seq_cst,
+  align 16
+  ret { i128, i1 } %result
+}
+
 define void @store_atomic_vec2_ptr_align(ptr %x, <2 x ptr> %v) nounwind {
 ; CHECK64-LABEL: define void @store_atomic_vec2_ptr_align(
 ; CHECK64-SAME: ptr [[X:%.*]], <2 x ptr> [[V:%.*]]) #[[ATTR0]] {
@@ -659,6 +942,22 @@ define void @store_atomic_vec2_ptr_align(ptr %x, <2 x ptr> %v) nounwind {
 ; CHECK32-SAME: ptr [[X:%.*]], <2 x ptr> [[V:%.*]]) #[[ATTR0]] {
 ; CHECK32-NEXT:    store atomic <2 x ptr> [[V]], ptr [[X]] release, align 16
 ; CHECK32-NEXT:    ret void
+;
+; CX16-LABEL: define void @store_atomic_vec2_ptr_align(
+; CX16-SAME: ptr [[X:%.*]], <2 x ptr> [[V:%.*]]) #[[ATTR1]] {
+; CX16-NEXT:    store atomic <2 x ptr> [[V]], ptr [[X]] release, align 16
+; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @store_atomic_vec2_ptr_align(
+; WIN64-SAME: ptr [[X:%.*]], <2 x ptr> [[V:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT:    [[TMP1:%.*]] = ptrtoint <2 x ptr> [[V]] to <2 x i64>
+; WIN64-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to i128
+; WIN64-NEXT:    [[TMP3:%.*]] = alloca i128, align 16
+; WIN64-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP3]])
+; WIN64-NEXT:    store i128 [[TMP2]], ptr [[TMP3]], align 16
+; WIN64-NEXT:    call void @__atomic_store_16(ptr [[X]], ptr [[TMP3]], i32 3)
+; WIN64-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP3]])
+; WIN64-NEXT:    ret void
 ;
   store atomic <2 x ptr> %v, ptr %x release, align 16
   ret void
@@ -680,6 +979,22 @@ define void @store_atomic_vec4_ptr270_align(ptr %x, <4 x ptr addrspace(270)> %v)
 ; CHECK32-NEXT:    call void @__atomic_store(i32 16, ptr [[X]], ptr [[TMP1]], i32 3)
 ; CHECK32-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
 ; CHECK32-NEXT:    ret void
+;
+; CX16-LABEL: define void @store_atomic_vec4_ptr270_align(
+; CX16-SAME: ptr [[X:%.*]], <4 x ptr addrspace(270)> [[V:%.*]]) #[[ATTR1]] {
+; CX16-NEXT:    store atomic <4 x ptr addrspace(270)> [[V]], ptr [[X]] release, align 16
+; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @store_atomic_vec4_ptr270_align(
+; WIN64-SAME: ptr [[X:%.*]], <4 x ptr addrspace(270)> [[V:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT:    [[TMP1:%.*]] = ptrtoint <4 x ptr addrspace(270)> [[V]] to <4 x i32>
+; WIN64-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; WIN64-NEXT:    [[TMP3:%.*]] = alloca i128, align 16
+; WIN64-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP3]])
+; WIN64-NEXT:    store i128 [[TMP2]], ptr [[TMP3]], align 16
+; WIN64-NEXT:    call void @__atomic_store_16(ptr [[X]], ptr [[TMP3]], i32 3)
+; WIN64-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP3]])
+; WIN64-NEXT:    ret void
 ;
   store atomic <4 x ptr addrspace(270)> %v, ptr %x release, align 16
   ret void
@@ -690,6 +1005,16 @@ define void @store_atomic_vec2_i16(ptr %x, <2 x i16> %v) nounwind {
 ; CHECK-SAME: ptr [[X:%.*]], <2 x i16> [[V:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    store atomic <2 x i16> [[V]], ptr [[X]] release, align 8
 ; CHECK-NEXT:    ret void
+;
+; CX16-LABEL: define void @store_atomic_vec2_i16(
+; CX16-SAME: ptr [[X:%.*]], <2 x i16> [[V:%.*]]) #[[ATTR1]] {
+; CX16-NEXT:    store atomic <2 x i16> [[V]], ptr [[X]] release, align 8
+; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @store_atomic_vec2_i16(
+; WIN64-SAME: ptr [[X:%.*]], <2 x i16> [[V:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT:    store atomic <2 x i16> [[V]], ptr [[X]] release, align 8
+; WIN64-NEXT:    ret void
 ;
   store atomic <2 x i16> %v, ptr %x release, align 8
   ret void
@@ -700,6 +1025,16 @@ define void @store_atomic_vec2_half(ptr %x, <2 x half> %v) nounwind {
 ; CHECK-SAME: ptr [[X:%.*]], <2 x half> [[V:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    store atomic <2 x half> [[V]], ptr [[X]] release, align 8
 ; CHECK-NEXT:    ret void
+;
+; CX16-LABEL: define void @store_atomic_vec2_half(
+; CX16-SAME: ptr [[X:%.*]], <2 x half> [[V:%.*]]) #[[ATTR1]] {
+; CX16-NEXT:    store atomic <2 x half> [[V]], ptr [[X]] release, align 8
+; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @store_atomic_vec2_half(
+; WIN64-SAME: ptr [[X:%.*]], <2 x half> [[V:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT:    store atomic <2 x half> [[V]], ptr [[X]] release, align 8
+; WIN64-NEXT:    ret void
 ;
   store atomic <2 x half> %v, ptr %x release, align 8
   ret void
@@ -720,6 +1055,32 @@ define void @store_atomic_vec4_i32(ptr %x, <4 x i32> %v) nounwind {
 ; CHECK32-NEXT:    call void @__atomic_store(i32 16, ptr [[X]], ptr [[TMP1]], i32 3)
 ; CHECK32-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
 ; CHECK32-NEXT:    ret void
+;
+; CX16-LABEL: define void @store_atomic_vec4_i32(
+; CX16-SAME: ptr [[X:%.*]], <4 x i32> [[V:%.*]]) #[[ATTR1]] {
+; CX16-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[X]], align 16
+; CX16-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CX16:       [[ATOMICRMW_START]]:
+; CX16-NEXT:    [[LOADED:%.*]] = phi <4 x i32> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CX16-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[V]] to i128
+; CX16-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[LOADED]] to i128
+; CX16-NEXT:    [[TMP4:%.*]] = cmpxchg ptr [[X]], i128 [[TMP3]], i128 [[TMP2]] release monotonic, align 16
+; CX16-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP4]], 1
+; CX16-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i128, i1 } [[TMP4]], 0
+; CX16-NEXT:    [[TMP5]] = bitcast i128 [[NEWLOADED]] to <4 x i32>
+; CX16-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CX16:       [[ATOMICRMW_END]]:
+; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @store_atomic_vec4_i32(
+; WIN64-SAME: ptr [[X:%.*]], <4 x i32> [[V:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V]] to i128
+; WIN64-NEXT:    [[TMP2:%.*]] = alloca i128, align 16
+; WIN64-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP2]])
+; WIN64-NEXT:    store i128 [[TMP1]], ptr [[TMP2]], align 16
+; WIN64-NEXT:    call void @__atomic_store_16(ptr [[X]], ptr [[TMP2]], i32 3)
+; WIN64-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP2]])
+; WIN64-NEXT:    ret void
 ;
   store atomic <4 x i32> %v, ptr %x release, align 16
   ret void
@@ -740,6 +1101,32 @@ define void @store_atomic_vec4_float(ptr %x, <4 x float> %v) nounwind {
 ; CHECK32-NEXT:    call void @__atomic_store(i32 16, ptr [[X]], ptr [[TMP1]], i32 3)
 ; CHECK32-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
 ; CHECK32-NEXT:    ret void
+;
+; CX16-LABEL: define void @store_atomic_vec4_float(
+; CX16-SAME: ptr [[X:%.*]], <4 x float> [[V:%.*]]) #[[ATTR1]] {
+; CX16-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[X]], align 16
+; CX16-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CX16:       [[ATOMICRMW_START]]:
+; CX16-NEXT:    [[LOADED:%.*]] = phi <4 x float> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]
+; CX16-NEXT:    [[TMP2:%.*]] = bitcast <4 x float> [[V]] to i128
+; CX16-NEXT:    [[TMP3:%.*]] = bitcast <4 x float> [[LOADED]] to i128
+; CX16-NEXT:    [[TMP4:%.*]] = cmpxchg ptr [[X]], i128 [[TMP3]], i128 [[TMP2]] release monotonic, align 16
+; CX16-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP4]], 1
+; CX16-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i128, i1 } [[TMP4]], 0
+; CX16-NEXT:    [[TMP5]] = bitcast i128 [[NEWLOADED]] to <4 x float>
+; CX16-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CX16:       [[ATOMICRMW_END]]:
+; CX16-NEXT:    ret void
+;
+; WIN64-LABEL: define void @store_atomic_vec4_float(
+; WIN64-SAME: ptr [[X:%.*]], <4 x float> [[V:%.*]]) #[[ATTR0]] {
+; WIN64-NEXT:    [[TMP1:%.*]] = bitcast <4 x float> [[V]] to i128
+; WIN64-NEXT:    [[TMP2:%.*]] = alloca i128, align 16
+; WIN64-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP2]])
+; WIN64-NEXT:    store i128 [[TMP1]], ptr [[TMP2]], align 16
+; WIN64-NEXT:    call void @__atomic_store_16(ptr [[X]], ptr [[TMP2]], i32 3)
+; WIN64-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP2]])
+; WIN64-NEXT:    ret void
 ;
   store atomic <4 x float> %v, ptr %x release, align 16
   ret void



More information about the cfe-commits mailing list