[llvm] [X86][APX] Implement Windows APX ABI callee-saved registers per Microsoft spec (PR #200391)
Timur Golubovich via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 4 07:49:01 PDT 2026
https://github.com/timurgol007 updated https://github.com/llvm/llvm-project/pull/200391
>From 7929c70f9d752d25b30e8b44e4f4cfa8baadbb3e Mon Sep 17 00:00:00 2001
From: Timur Golubovich <timur.golubovich at intel.com>
Date: Wed, 29 Apr 2026 09:53:15 +0200
Subject: [PATCH 1/6] Implement Windows APX ABI callee-saved registers per
Microsoft spec
Add CSR_Win64_APX and CSR_Win64_APX_NoSSE: r30/r31 are non-volatile on
Win64. Both extend standard Win64 CSRs with R30 and R31; NoSSE variant
drops XMMs. Update getCalleeSavedRegs and getCallPreservedMask to use
Win64 APX CSR lists.
---
llvm/lib/Target/X86/X86CallingConv.td | 6 +
llvm/lib/Target/X86/X86RegisterInfo.cpp | 40 +-
llvm/lib/Target/X86/X86RegisterInfo.td | 16 +-
.../CodeGen/X86/apx/memfold-no-physreg.ll | 138 +--
llvm/test/CodeGen/X86/apx/mul-i1024.ll | 838 +++++++++---------
llvm/test/CodeGen/X86/apx/setjmp-win64-abi.ll | 237 +++++
llvm/test/CodeGen/X86/apx/win64-abi.ll | 168 ++++
llvm/test/CodeGen/X86/apx/win64cc-abi.ll | 164 ++++
8 files changed, 1106 insertions(+), 501 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/apx/setjmp-win64-abi.ll
create mode 100644 llvm/test/CodeGen/X86/apx/win64-abi.ll
create mode 100644 llvm/test/CodeGen/X86/apx/win64cc-abi.ll
diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td
index 369c1af7eb488..441b565e4bc03 100644
--- a/llvm/lib/Target/X86/X86CallingConv.td
+++ b/llvm/lib/Target/X86/X86CallingConv.td
@@ -1135,9 +1135,15 @@ def CSR_32EHRet : CalleeSavedRegs<(add EAX, EDX, CSR_32)>;
def CSR_64EHRet : CalleeSavedRegs<(add RAX, RDX, CSR_64)>;
def CSR_Win64_NoSSE : CalleeSavedRegs<(add RBX, RBP, RDI, RSI, R12, R13, R14, R15)>;
+// When APX support is present, in Win64 ABI R30 and R31 are callee-saved
+// https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170#callercallee-saved-registers
+def CSR_Win64_EGPR : CalleeSavedRegs<(add R30, R31)>;
+def CSR_Win64_NoSSE_APX : CalleeSavedRegs<(add CSR_Win64_NoSSE, CSR_Win64_EGPR)>;
def CSR_Win64 : CalleeSavedRegs<(add CSR_Win64_NoSSE,
(sequence "XMM%u", 6, 15))>;
+def CSR_Win64_APX : CalleeSavedRegs<(add CSR_Win64_NoSSE_APX,
+ (sequence "XMM%u", 6, 15))>;
def CSR_Win64_SwiftError : CalleeSavedRegs<(sub CSR_Win64, R12)>;
def CSR_Win64_SwiftTail : CalleeSavedRegs<(sub CSR_Win64, R13, R14)>;
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index c92e20ad7ef13..7ec5d11667bf7 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -27,6 +27,7 @@
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TileShapeInfo.h"
#include "llvm/CodeGen/VirtRegMap.h"
+#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Type.h"
#include "llvm/MC/MCContext.h"
@@ -244,6 +245,7 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
bool HasSSE = Subtarget.hasSSE1();
bool HasAVX = Subtarget.hasAVX();
bool HasAVX512 = Subtarget.hasAVX512();
+ bool HasEGPR = Subtarget.hasEGPR();
bool CallsEHReturn = MF->callsEHReturn();
CallingConv::ID CC = F.getCallingConv();
@@ -315,9 +317,9 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
return CSR_64_MostRegs_SaveList;
break;
case CallingConv::Win64:
- if (!HasSSE)
- return CSR_Win64_NoSSE_SaveList;
- return CSR_Win64_SaveList;
+ if (HasEGPR)
+ return HasSSE ? CSR_Win64_APX_SaveList : CSR_Win64_NoSSE_APX_SaveList;
+ return HasSSE ? CSR_Win64_SaveList : CSR_Win64_NoSSE_SaveList;
case CallingConv::SwiftTail:
if (!Is64Bit)
return CSR_32_SaveList;
@@ -355,8 +357,11 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
return IsWin64 ? CSR_Win64_SwiftError_SaveList
: CSR_64_SwiftError_SaveList;
- if (IsWin64 || IsUEFI64)
+ if (IsWin64 || IsUEFI64) {
+ if (HasEGPR)
+ return HasSSE ? CSR_Win64_APX_SaveList : CSR_Win64_NoSSE_APX_SaveList;
return HasSSE ? CSR_Win64_SaveList : CSR_Win64_NoSSE_SaveList;
+ }
if (CallsEHReturn)
return CSR_64EHRet_SaveList;
return CSR_64_SaveList;
@@ -446,7 +451,7 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
return CSR_64_MostRegs_RegMask;
break;
case CallingConv::Win64:
- return CSR_Win64_RegMask;
+ return CSR_Win64_APX_RegMask;
case CallingConv::SwiftTail:
if (!Is64Bit)
return CSR_32_RegMask;
@@ -484,7 +489,7 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
if (IsSwiftCC)
return IsWin64 ? CSR_Win64_SwiftError_RegMask : CSR_64_SwiftError_RegMask;
- return (IsWin64 || IsUEFI64) ? CSR_Win64_RegMask : CSR_64_RegMask;
+ return (IsWin64 || IsUEFI64) ? CSR_Win64_APX_RegMask : CSR_64_RegMask;
}
return CSR_32_RegMask;
@@ -614,6 +619,29 @@ BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
if (!Is64Bit || !MF.getSubtarget<X86Subtarget>().hasEGPR())
Reserved.set(X86::R16, X86::R31WH + 1);
+ // There is a problem with allocation R30/R31 registers in Win64 APX ABI
+ // functions that have setjmp as unwinder won't be able to restore them:
+ // https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170#setjmplongjmp
+ if (MF.exposesReturnsTwice()) {
+ unsigned NumReservedCSRs = 0;
+ for (const MCPhysReg &Reg : CSR_Win64_EGPR_SaveList)
+ if (isCalleeSavedPhysReg(Reg, MF)) {
+ ++NumReservedCSRs;
+ for (const MCPhysReg &SubReg : subregs_inclusive(Reg))
+ Reserved.set(SubReg);
+ }
+ if (NumReservedCSRs && MF.size() > 50 &&
+ !MF.getRegInfo().reservedRegsFrozen()) {
+ const Function &F = MF.getFunction();
+ F.getContext().diagnose(DiagnosticInfoResourceLimit(
+ F,
+ "callee-saved registers reserved due to setjmp; "
+ "this may impact performance in large functions (" +
+ Twine(NumReservedCSRs) + " registers excluded from allocation)",
+ MF.size(), 50, DS_Warning));
+ }
+ }
+
if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) {
for (MCRegAliasIterator AI(X86::R14, this, true); AI.isValid(); ++AI)
Reserved.set(*AI);
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.td b/llvm/lib/Target/X86/X86RegisterInfo.td
index 692e42ae5e752..bafea74f34403 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.td
+++ b/llvm/lib/Target/X86/X86RegisterInfo.td
@@ -545,8 +545,8 @@ def SSP : X86Reg<"ssp", 0>;
def GR8 : RegisterClass<"X86", [i8], 8,
(add AL, CL, DL, AH, CH, DH, BL, BH, SIL, DIL, BPL, SPL,
R8B, R9B, R10B, R11B, R16B, R17B, R18B, R19B, R22B,
- R23B, R24B, R25B, R26B, R27B, R30B, R31B, R14B,
- R15B, R12B, R13B, R20B, R21B, R28B, R29B)> {
+ R23B, R24B, R25B, R26B, R27B, R14B, R15B, R12B,
+ R13B, R20B, R21B, R28B, R29B, R30B, R31B)> {
let AltOrders = [(sub GR8, AH, BH, CH, DH)];
let AltOrderSelect = [{
return MF.getSubtarget<X86Subtarget>().is64Bit();
@@ -562,8 +562,8 @@ def GRH8 : RegisterClass<"X86", [i8], 8,
def GR16 : RegisterClass<"X86", [i16], 16,
(add AX, CX, DX, SI, DI, BX, BP, SP, R8W, R9W, R10W,
R11W, R16W, R17W, R18W, R19W, R22W, R23W, R24W,
- R25W, R26W, R27W, R30W, R31W, R14W, R15W, R12W,
- R13W, R20W, R21W, R28W, R29W)>;
+ R25W, R26W, R27W, R14W, R15W, R12W, R13W, R20W,
+ R21W, R28W, R29W, R30W, R31W)>;
let isAllocatable = 0 in
def GRH16 : RegisterClass<"X86", [i16], 16,
@@ -574,8 +574,8 @@ def GRH16 : RegisterClass<"X86", [i16], 16,
def GR32 : RegisterClass<"X86", [i32], 32,
(add EAX, ECX, EDX, ESI, EDI, EBX, EBP, ESP, R8D, R9D,
R10D, R11D, R16D, R17D, R18D, R19D, R22D, R23D,
- R24D, R25D, R26D, R27D, R30D, R31D, R14D, R15D,
- R12D, R13D, R20D, R21D, R28D, R29D)>;
+ R24D, R25D, R26D, R27D, R14D, R15D, R12D, R13D,
+ R20D, R21D, R28D, R29D, R30D, R31D)>;
// GR64 - 64-bit GPRs. This oddly includes RIP, which isn't accurate, since
// RIP isn't really a register and it can't be used anywhere except in an
@@ -584,8 +584,8 @@ def GR32 : RegisterClass<"X86", [i32], 32,
// tests because of the inclusion of RIP in this register class.
def GR64 : RegisterClass<"X86", [i64], 64,
(add RAX, RCX, RDX, RSI, RDI, R8, R9, R10, R11, R16, R17,
- R18, R19, R22, R23, R24, R25, R26, R27, R30, R31, RBX,
- R14, R15, R12, R13, R20, R21, R28, R29, RBP, RSP, RIP)>;
+ R18, R19, R22, R23, R24, R25, R26, R27, RBX, R14, R15,
+ R12, R13, R20, R21, R28, R29, R30, R31, RBP, RSP, RIP)>;
// GR64PLTSafe - 64-bit GPRs without R10, R11, RSP and RIP. Could be used when
// emitting code for intrinsics, which use implict input registers.
diff --git a/llvm/test/CodeGen/X86/apx/memfold-no-physreg.ll b/llvm/test/CodeGen/X86/apx/memfold-no-physreg.ll
index 740b321165b63..b85e26f5c1894 100644
--- a/llvm/test/CodeGen/X86/apx/memfold-no-physreg.ll
+++ b/llvm/test/CodeGen/X86/apx/memfold-no-physreg.ll
@@ -5,6 +5,8 @@ define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr
; CHECK-LABEL: foo:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: pushq %rbp
+; CHECK-NEXT: pushq %r31
+; CHECK-NEXT: pushq %r30
; CHECK-NEXT: pushq %r15
; CHECK-NEXT: pushq %r14
; CHECK-NEXT: pushq %r13
@@ -12,36 +14,36 @@ define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr
; CHECK-NEXT: pushq %rsi
; CHECK-NEXT: pushq %rdi
; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: subq $56, %rsp
+; CHECK-NEXT: subq $40, %rsp
; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rbp
; CHECK-NEXT: andq $-32, %rsp
; CHECK-NEXT: movq %rsp, %rbx
-; CHECK-NEXT: movq 312(%rbp), %rax
+; CHECK-NEXT: movq 328(%rbp), %rax
; CHECK-NEXT: vpmovsxwd (%rax), %ymm0
; CHECK-NEXT: vpslld $31, %ymm0, %ymm0
; CHECK-NEXT: vptestmd %ymm0, %ymm0, %k2
-; CHECK-NEXT: movq 304(%rbp), %r29
-; CHECK-NEXT: movzbl 296(%rbp), %r26d
-; CHECK-NEXT: movzbl 288(%rbp), %r30d
-; CHECK-NEXT: movq 280(%rbp), %r31
-; CHECK-NEXT: movq 272(%rbp), %r20
-; CHECK-NEXT: movq 264(%rbp), %r21
-; CHECK-NEXT: movq 240(%rbp), %r9
-; CHECK-NEXT: movq 232(%rbp), %r10
-; CHECK-NEXT: movq 224(%rbp), %r11
-; CHECK-NEXT: movq 216(%rbp), %r16
-; CHECK-NEXT: movq 208(%rbp), %r17
-; CHECK-NEXT: movq 192(%rbp), %r18
-; CHECK-NEXT: movq 184(%rbp), %r19
-; CHECK-NEXT: movq 176(%rbp), %r22
-; CHECK-NEXT: movq 168(%rbp), %r23
-; CHECK-NEXT: movq 160(%rbp), %r24
-; CHECK-NEXT: movq 152(%rbp), %r25
-; CHECK-NEXT: movq 144(%rbp), %r14
-; CHECK-NEXT: movq 136(%rbp), %r15
-; CHECK-NEXT: movq 128(%rbp), %r12
-; CHECK-NEXT: movq 120(%rbp), %r13
-; CHECK-NEXT: movq 112(%rbp), %rsi
+; CHECK-NEXT: movq 320(%rbp), %r29
+; CHECK-NEXT: movzbl 312(%rbp), %r24d
+; CHECK-NEXT: movzbl 304(%rbp), %r26d
+; CHECK-NEXT: movq 296(%rbp), %r27
+; CHECK-NEXT: movq 288(%rbp), %r20
+; CHECK-NEXT: movq 280(%rbp), %r21
+; CHECK-NEXT: movq 256(%rbp), %r9
+; CHECK-NEXT: movq 248(%rbp), %r10
+; CHECK-NEXT: movq 240(%rbp), %r11
+; CHECK-NEXT: movq 232(%rbp), %r16
+; CHECK-NEXT: movq 224(%rbp), %r17
+; CHECK-NEXT: movq 208(%rbp), %r18
+; CHECK-NEXT: movq 200(%rbp), %r19
+; CHECK-NEXT: movq 192(%rbp), %r22
+; CHECK-NEXT: movq 184(%rbp), %r23
+; CHECK-NEXT: movq 176(%rbp), %r12
+; CHECK-NEXT: movq 168(%rbp), %r13
+; CHECK-NEXT: movq 160(%rbp), %r30
+; CHECK-NEXT: movq 152(%rbp), %r31
+; CHECK-NEXT: movq 144(%rbp), %rsi
+; CHECK-NEXT: movq 136(%rbp), %rdi
+; CHECK-NEXT: movq 128(%rbp), %r14
; CHECK-NEXT: leaq 15(,%r21,4), %rdx
; CHECK-NEXT: andq $-16, %rdx
; CHECK-NEXT: movq %rdx, %rax
@@ -53,7 +55,7 @@ define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr
; CHECK-NEXT: subq %rax, %rsp
; CHECK-NEXT: andq $-32, %rsp
; CHECK-NEXT: movl %ecx, %r8d
-; CHECK-NEXT: xorl %edi, %edi
+; CHECK-NEXT: xorl %r15d, %r15d
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: kmovw %k2, 14(%rbx) # 2-byte Spill
; CHECK-NEXT: movq %r8, 24(%rbx) # 8-byte Spill
@@ -61,16 +63,16 @@ define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB0_6: # %loop_outer.backedge
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
-; CHECK-NEXT: subq %r21, %rdi
+; CHECK-NEXT: subq %r21, %r15
; CHECK-NEXT: .LBB0_1: # %loop_outer
; CHECK-NEXT: # =>This Loop Header: Depth=1
; CHECK-NEXT: # Child Loop BB0_8 Depth 2
-; CHECK-NEXT: testq %r31, %r31
+; CHECK-NEXT: testq %r27, %r27
; CHECK-NEXT: ctestgq {dfv=} %r20, %r20
; CHECK-NEXT: jns .LBB0_10
; CHECK-NEXT: # %bb.2: # %then_branch
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
-; CHECK-NEXT: testb $1, %r30b
+; CHECK-NEXT: testb $1, %r26b
; CHECK-NEXT: je .LBB0_3
; CHECK-NEXT: # %bb.5: # %early_exit
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
@@ -79,9 +81,9 @@ define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB0_10: # %else_branch
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
-; CHECK-NEXT: leaq -1(%rax), %r27
-; CHECK-NEXT: testb $1, %r26b
-; CHECK-NEXT: movq %r27, 16(%rbx) # 8-byte Spill
+; CHECK-NEXT: leaq -1(%rax), %r25
+; CHECK-NEXT: testb $1, %r24b
+; CHECK-NEXT: movq %r25, 16(%rbx) # 8-byte Spill
; CHECK-NEXT: je .LBB0_11
; CHECK-NEXT: # %bb.7: # %memset_path
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
@@ -90,59 +92,57 @@ define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr
; CHECK-NEXT: xorl %edx, %edx
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: callq memset
-; CHECK-NEXT: movzbl 288(%rbp), %r30d
+; CHECK-NEXT: movzbl 304(%rbp), %r26d
; CHECK-NEXT: addq $32, %rsp
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB0_8: # %inner_loop
; CHECK-NEXT: # Parent Loop BB0_1 Depth=1
; CHECK-NEXT: # => This Inner Loop Header: Depth=2
-; CHECK-NEXT: testb $1, %r30b
+; CHECK-NEXT: testb $1, %r26b
; CHECK-NEXT: jne .LBB0_8
; CHECK-NEXT: # %bb.9: # in Loop: Header=BB0_1 Depth=1
; CHECK-NEXT: kmovw 14(%rbx), %k2 # 2-byte Reload
-; CHECK-NEXT: movq 304(%rbp), %r29
-; CHECK-NEXT: movq 240(%rbp), %r9
-; CHECK-NEXT: movq 232(%rbp), %r10
-; CHECK-NEXT: movq 224(%rbp), %r11
-; CHECK-NEXT: movq 216(%rbp), %r16
-; CHECK-NEXT: movq 208(%rbp), %r17
-; CHECK-NEXT: movq 192(%rbp), %r18
-; CHECK-NEXT: movq 184(%rbp), %r19
-; CHECK-NEXT: movq 176(%rbp), %r22
-; CHECK-NEXT: movq 168(%rbp), %r23
-; CHECK-NEXT: movq 160(%rbp), %r24
-; CHECK-NEXT: movq 152(%rbp), %r25
-; CHECK-NEXT: movzbl 296(%rbp), %r26d
-; CHECK-NEXT: movq 280(%rbp), %r31
-; CHECK-NEXT: movq 272(%rbp), %r20
-; CHECK-NEXT: movq 264(%rbp), %r21
+; CHECK-NEXT: movq 320(%rbp), %r29
+; CHECK-NEXT: movq 256(%rbp), %r9
+; CHECK-NEXT: movq 248(%rbp), %r10
+; CHECK-NEXT: movq 240(%rbp), %r11
+; CHECK-NEXT: movq 232(%rbp), %r16
+; CHECK-NEXT: movq 224(%rbp), %r17
+; CHECK-NEXT: movq 208(%rbp), %r18
+; CHECK-NEXT: movq 200(%rbp), %r19
+; CHECK-NEXT: movq 192(%rbp), %r22
+; CHECK-NEXT: movq 184(%rbp), %r23
+; CHECK-NEXT: movzbl 312(%rbp), %r24d
+; CHECK-NEXT: movq 296(%rbp), %r27
+; CHECK-NEXT: movq 288(%rbp), %r20
+; CHECK-NEXT: movq 280(%rbp), %r21
; CHECK-NEXT: movq 24(%rbx), %r8 # 8-byte Reload
; CHECK-NEXT: movq 16(%rbx), %rax # 8-byte Reload
-; CHECK-NEXT: subq %r21, %rdi
+; CHECK-NEXT: subq %r21, %r15
; CHECK-NEXT: jmp .LBB0_1
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB0_11: # %compute_path
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
-; CHECK-NEXT: testb $1, %r30b
+; CHECK-NEXT: testb $1, %r26b
; CHECK-NEXT: jne .LBB0_12
; CHECK-NEXT: # %bb.14: # %loop_exit
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
-; CHECK-NEXT: movq 256(%rbp), %rcx
+; CHECK-NEXT: movq 272(%rbp), %rcx
; CHECK-NEXT: addq %rax, %rcx
-; CHECK-NEXT: andl $1, %r21d, %r27d
+; CHECK-NEXT: andl $1, %r21d, %r25d
; CHECK-NEXT: andl $1, %r8d, %edx
; CHECK-NEXT: movq %r20, %rax
-; CHECK-NEXT: movq %r31, %r20
-; CHECK-NEXT: movl %r26d, %r31d
-; CHECK-NEXT: movq 320(%rbp), %r26
-; CHECK-NEXT: movq 200(%rbp), %r28
+; CHECK-NEXT: movq %r27, %r20
+; CHECK-NEXT: movl %r24d, %r27d
+; CHECK-NEXT: movq 336(%rbp), %r24
+; CHECK-NEXT: movq 216(%rbp), %r28
; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; CHECK-NEXT: vmovdqu32 %ymm0, (%r28,%r26) {%k2}
-; CHECK-NEXT: movl %r31d, %r26d
-; CHECK-NEXT: movq %r20, %r31
+; CHECK-NEXT: vmovdqu32 %ymm0, (%r28,%r24) {%k2}
+; CHECK-NEXT: movl %r27d, %r24d
+; CHECK-NEXT: movq %r20, %r27
; CHECK-NEXT: movq %rax, %r20
; CHECK-NEXT: vmovups %ymm0, (%rcx) {%k2}
-; CHECK-NEXT: cmpl %edx, %r27d
+; CHECK-NEXT: cmpl %edx, %r25d
; CHECK-NEXT: movq 16(%rbx), %rax # 8-byte Reload
; CHECK-NEXT: je .LBB0_6
; CHECK-NEXT: # %bb.15:
@@ -157,22 +157,22 @@ define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr
; CHECK-NEXT: vmovdqu (%r9), %ymm1
; CHECK-NEXT: vpord (%r18), %ymm1, %ymm1
; CHECK-NEXT: vmovups %ymm0, (%r23) {%k2}
-; CHECK-NEXT: vmovups %ymm0, (%r24) {%k2}
-; CHECK-NEXT: vmovups (%r25,%rdi), %ymm2
-; CHECK-NEXT: vcmpltps (%r14,%rdi), %ymm2, %k1
+; CHECK-NEXT: vmovups %ymm0, (%r12) {%k2}
+; CHECK-NEXT: vmovups (%r13,%r15), %ymm2
+; CHECK-NEXT: vcmpltps (%r30,%r15), %ymm2, %k1
; CHECK-NEXT: vptestnmd %ymm1, %ymm1, %k1 {%k1}
; CHECK-NEXT: vmovups (%r10), %ymm1 {%k1} {z}
; CHECK-NEXT: vmovups %ymm1, (%r22) {%k2}
-; CHECK-NEXT: vmovdqu32 %ymm0, (%r13) {%k2}
+; CHECK-NEXT: vmovdqu32 %ymm0, (%rdi) {%k2}
; CHECK-NEXT: vmovdqu32 %ymm0, (%r16) {%k2}
; CHECK-NEXT: vmovups %ymm0, (%r19) {%k2}
-; CHECK-NEXT: vmovups (%r15), %ymm1 {%k2} {z}
+; CHECK-NEXT: vmovups (%r31), %ymm1 {%k2} {z}
; CHECK-NEXT: vmovups %ymm1, (%r17,%r29,4) {%k2}
; CHECK-NEXT: vmovups %ymm0, (%r11) {%k2}
-; CHECK-NEXT: vmovups %ymm0, (%r12) {%k2}
; CHECK-NEXT: vmovups %ymm0, (%rsi) {%k2}
+; CHECK-NEXT: vmovups %ymm0, (%r14) {%k2}
; CHECK-NEXT: incq %rax
-; CHECK-NEXT: addq $4, %rdi
+; CHECK-NEXT: addq $4, %r15
; CHECK-NEXT: jmp .LBB0_4
; CHECK-NEXT: .LBB0_12:
; CHECK-NEXT: xorl %eax, %eax
@@ -185,6 +185,8 @@ define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr
; CHECK-NEXT: popq %r13
; CHECK-NEXT: popq %r14
; CHECK-NEXT: popq %r15
+; CHECK-NEXT: popq %r30
+; CHECK-NEXT: popq %r31
; CHECK-NEXT: popq %rbp
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/apx/mul-i1024.ll b/llvm/test/CodeGen/X86/apx/mul-i1024.ll
index 0ca77c303de3d..101c5a4015f07 100644
--- a/llvm/test/CodeGen/X86/apx/mul-i1024.ll
+++ b/llvm/test/CodeGen/X86/apx/mul-i1024.ll
@@ -16,7 +16,7 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %rdi, %r26
; EGPR-NEXT: movq (%rdi), %r13
; EGPR-NEXT: movq 8(%rdi), %r18
-; EGPR-NEXT: movq 24(%rdi), %r21
+; EGPR-NEXT: movq 24(%rdi), %r29
; EGPR-NEXT: movq 16(%rdi), %r17
; EGPR-NEXT: movq 40(%rdi), %rdi
; EGPR-NEXT: movq 32(%r26), %r10
@@ -25,14 +25,14 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NEXT: movq 24(%rsi), %r25
; EGPR-NEXT: movq 16(%rsi), %r11
-; EGPR-NEXT: movq (%rsi), %r31
+; EGPR-NEXT: movq (%rsi), %r21
; EGPR-NEXT: movq 8(%rsi), %r14
; EGPR-NEXT: movq %r12, %rax
-; EGPR-NEXT: mulq %r31
+; EGPR-NEXT: mulq %r21
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, %r19
; EGPR-NEXT: movq %r15, %rax
-; EGPR-NEXT: mulq %r31
+; EGPR-NEXT: mulq %r21
; EGPR-NEXT: movq %rdx, %r9
; EGPR-NEXT: movq %rax, %r16
; EGPR-NEXT: addq %r8, %r16
@@ -52,11 +52,11 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: addq %r22, %r16
; EGPR-NEXT: adcq %rcx, %r9
; EGPR-NEXT: movq %r10, %rax
-; EGPR-NEXT: mulq %r31
+; EGPR-NEXT: mulq %r21
; EGPR-NEXT: movq %rdx, %r22
; EGPR-NEXT: movq %rax, %r27
; EGPR-NEXT: movq %rdi, %rax
-; EGPR-NEXT: mulq %r31
+; EGPR-NEXT: mulq %r21
; EGPR-NEXT: movq %rdx, %r23
; EGPR-NEXT: movq %rax, %r24
; EGPR-NEXT: addq %r22, %r24
@@ -64,8 +64,8 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %r10, %rax
; EGPR-NEXT: mulq %r14
; EGPR-NEXT: movq %rdx, %r22
-; EGPR-NEXT: movq %rax, %r20
-; EGPR-NEXT: addq %r24, %r20
+; EGPR-NEXT: movq %rax, %r28
+; EGPR-NEXT: addq %r24, %r28
; EGPR-NEXT: adcq %r23, %r22
; EGPR-NEXT: setb %al
; EGPR-NEXT: movzbl %al, %ecx
@@ -83,7 +83,7 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %r10, %rax
; EGPR-NEXT: mulq %r11
; EGPR-NEXT: movq %rdx, %r8
-; EGPR-NEXT: movq %rax, %r28
+; EGPR-NEXT: movq %rax, %r30
; EGPR-NEXT: movq %rdi, %rax
; EGPR-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NEXT: mulq %r11
@@ -94,23 +94,23 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %r10, %rax
; EGPR-NEXT: mulq %r25
; EGPR-NEXT: movq %rdx, %rbx
-; EGPR-NEXT: movq %rax, %r29
-; EGPR-NEXT: addq %r22, %r29
+; EGPR-NEXT: movq %rax, %r31
+; EGPR-NEXT: addq %r22, %r31
; EGPR-NEXT: adcq %r19, %rbx
; EGPR-NEXT: setb %al
; EGPR-NEXT: movzbl %al, %ecx
; EGPR-NEXT: movq %rdi, %rax
; EGPR-NEXT: mulq %r25
-; EGPR-NEXT: movq %rdx, %r30
+; EGPR-NEXT: movq %rdx, %r20
; EGPR-NEXT: movq %rax, %r8
; EGPR-NEXT: addq %rbx, %r8
-; EGPR-NEXT: adcq %rcx, %r30
-; EGPR-NEXT: addq %r24, %r28
-; EGPR-NEXT: adcq %r23, %r29
+; EGPR-NEXT: adcq %rcx, %r20
+; EGPR-NEXT: addq %r24, %r30
+; EGPR-NEXT: adcq %r23, %r31
; EGPR-NEXT: adcq $0, %r8
-; EGPR-NEXT: adcq $0, %r30
+; EGPR-NEXT: adcq $0, %r20
; EGPR-NEXT: addq %r16, %r8
-; EGPR-NEXT: adcq %r9, %r30
+; EGPR-NEXT: adcq %r9, %r20
; EGPR-NEXT: setb %al
; EGPR-NEXT: movzbl %al, %ecx
; EGPR-NEXT: movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
@@ -141,15 +141,15 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: adcq %r10, %r23
; EGPR-NEXT: addq %r8, %rsi
; EGPR-NEXT: movq %rsi, %r19
-; EGPR-NEXT: adcq %r30, %rdi
+; EGPR-NEXT: adcq %r20, %rdi
; EGPR-NEXT: adcq %rcx, %r24
; EGPR-NEXT: adcq $0, %r23
; EGPR-NEXT: movq %r17, %rax
-; EGPR-NEXT: mulq %r31
+; EGPR-NEXT: mulq %r21
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, %rbx
-; EGPR-NEXT: movq %r21, %rax
-; EGPR-NEXT: mulq %r31
+; EGPR-NEXT: movq %r29, %rax
+; EGPR-NEXT: mulq %r21
; EGPR-NEXT: movq %rdx, %r9
; EGPR-NEXT: movq %rax, %r16
; EGPR-NEXT: addq %r8, %r16
@@ -157,12 +157,12 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %r17, %rax
; EGPR-NEXT: mulq %r14
; EGPR-NEXT: movq %rdx, %r8
-; EGPR-NEXT: movq %rax, %r30
-; EGPR-NEXT: addq %r16, %r30
+; EGPR-NEXT: movq %rax, %r20
+; EGPR-NEXT: addq %r16, %r20
; EGPR-NEXT: adcq %r9, %r8
; EGPR-NEXT: setb %al
; EGPR-NEXT: movzbl %al, %ecx
-; EGPR-NEXT: movq %r21, %rax
+; EGPR-NEXT: movq %r29, %rax
; EGPR-NEXT: mulq %r14
; EGPR-NEXT: movq %r14, %rsi
; EGPR-NEXT: movq %rdx, %r9
@@ -170,11 +170,11 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: addq %r8, %r16
; EGPR-NEXT: adcq %rcx, %r9
; EGPR-NEXT: movq %r13, %rax
-; EGPR-NEXT: mulq %r31
+; EGPR-NEXT: mulq %r21
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NEXT: movq %r18, %rax
-; EGPR-NEXT: mulq %r31
+; EGPR-NEXT: mulq %r21
; EGPR-NEXT: movq %rdx, %r14
; EGPR-NEXT: movq %rax, %r15
; EGPR-NEXT: addq %r8, %r15
@@ -195,18 +195,18 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movzbl %cl, %eax
; EGPR-NEXT: adcq %rax, %r8
; EGPR-NEXT: addq %rbx, %r15
-; EGPR-NEXT: adcq %r30, %r8
+; EGPR-NEXT: adcq %r20, %r8
; EGPR-NEXT: adcq $0, %r16
; EGPR-NEXT: adcq $0, %r9
; EGPR-NEXT: movq %r13, %rax
; EGPR-NEXT: mulq %r11
-; EGPR-NEXT: movq %rdx, %r30
+; EGPR-NEXT: movq %rdx, %r20
; EGPR-NEXT: movq %rax, %rsi
; EGPR-NEXT: movq %r18, %rax
; EGPR-NEXT: mulq %r11
; EGPR-NEXT: movq %rdx, %rbx
; EGPR-NEXT: movq %rax, %r14
-; EGPR-NEXT: addq %r30, %r14
+; EGPR-NEXT: addq %r20, %r14
; EGPR-NEXT: adcq $0, %rbx
; EGPR-NEXT: movq %r13, %rax
; EGPR-NEXT: mulq %r25
@@ -218,17 +218,17 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %r18, %rax
; EGPR-NEXT: mulq %r25
; EGPR-NEXT: movq %rdx, %r14
-; EGPR-NEXT: movq %rax, %r30
-; EGPR-NEXT: addq %r12, %r30
+; EGPR-NEXT: movq %rax, %r20
+; EGPR-NEXT: addq %r12, %r20
; EGPR-NEXT: movzbl %cl, %eax
; EGPR-NEXT: adcq %rax, %r14
; EGPR-NEXT: addq %r15, %rsi
; EGPR-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NEXT: adcq %r8, %r10
; EGPR-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NEXT: adcq $0, %r30
+; EGPR-NEXT: adcq $0, %r20
; EGPR-NEXT: adcq $0, %r14
-; EGPR-NEXT: addq %r16, %r30
+; EGPR-NEXT: addq %r16, %r20
; EGPR-NEXT: adcq %r9, %r14
; EGPR-NEXT: setb %cl
; EGPR-NEXT: movq %r17, %rax
@@ -236,7 +236,7 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: mulq %r11
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, %rbx
-; EGPR-NEXT: movq %r21, %rax
+; EGPR-NEXT: movq %r29, %rax
; EGPR-NEXT: mulq %r11
; EGPR-NEXT: movq %rdx, %r9
; EGPR-NEXT: movq %rax, %r16
@@ -249,14 +249,14 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: addq %r16, %r15
; EGPR-NEXT: adcq %r9, %r8
; EGPR-NEXT: setb %r9b
-; EGPR-NEXT: movq %r21, %rax
+; EGPR-NEXT: movq %r29, %rax
; EGPR-NEXT: mulq %r25
; EGPR-NEXT: movq %rdx, %r12
; EGPR-NEXT: movq %rax, %rbp
; EGPR-NEXT: addq %r8, %rbp
; EGPR-NEXT: movzbl %r9b, %eax
; EGPR-NEXT: adcq %rax, %r12
-; EGPR-NEXT: addq %r30, %rbx
+; EGPR-NEXT: addq %r20, %rbx
; EGPR-NEXT: adcq %r14, %r15
; EGPR-NEXT: movzbl %cl, %eax
; EGPR-NEXT: adcq %rax, %rbp
@@ -264,9 +264,9 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: addq %r27, %rbx
; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; EGPR-NEXT: movq 32(%rsi), %r27
-; EGPR-NEXT: adcq %r20, %r15
-; EGPR-NEXT: adcq %r28, %rbp
-; EGPR-NEXT: adcq %r29, %r12
+; EGPR-NEXT: adcq %r28, %r15
+; EGPR-NEXT: adcq %r30, %rbp
+; EGPR-NEXT: adcq %r31, %r12
; EGPR-NEXT: adcq $0, %r19
; EGPR-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NEXT: adcq $0, %rdi
@@ -275,8 +275,8 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %r17, %rax
; EGPR-NEXT: mulq %r27
; EGPR-NEXT: movq %rdx, %r8
-; EGPR-NEXT: movq %rax, %r20
-; EGPR-NEXT: movq %r21, %rax
+; EGPR-NEXT: movq %rax, %r28
+; EGPR-NEXT: movq %r29, %rax
; EGPR-NEXT: mulq %r27
; EGPR-NEXT: movq %rdx, %r9
; EGPR-NEXT: movq %rax, %r16
@@ -286,11 +286,11 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %r17, %rax
; EGPR-NEXT: mulq %rcx
; EGPR-NEXT: movq %rdx, %r8
-; EGPR-NEXT: movq %rax, %r30
-; EGPR-NEXT: addq %r16, %r30
+; EGPR-NEXT: movq %rax, %r20
+; EGPR-NEXT: addq %r16, %r20
; EGPR-NEXT: adcq %r9, %r8
; EGPR-NEXT: setb %r10b
-; EGPR-NEXT: movq %r21, %rax
+; EGPR-NEXT: movq %r29, %rax
; EGPR-NEXT: mulq %rcx
; EGPR-NEXT: movq %rdx, %r9
; EGPR-NEXT: movq %rax, %r16
@@ -303,93 +303,93 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %rax, %r19
; EGPR-NEXT: movq %r18, %rax
; EGPR-NEXT: mulq %r27
-; EGPR-NEXT: movq %rdx, %r28
-; EGPR-NEXT: movq %rax, %r29
-; EGPR-NEXT: addq %r8, %r29
-; EGPR-NEXT: adcq $0, %r28
+; EGPR-NEXT: movq %rdx, %r30
+; EGPR-NEXT: movq %rax, %r31
+; EGPR-NEXT: addq %r8, %r31
+; EGPR-NEXT: adcq $0, %r30
; EGPR-NEXT: movq %r13, %rax
; EGPR-NEXT: mulq %rcx
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, %r22
-; EGPR-NEXT: addq %r29, %r22
-; EGPR-NEXT: adcq %r28, %r8
+; EGPR-NEXT: addq %r31, %r22
+; EGPR-NEXT: adcq %r30, %r8
; EGPR-NEXT: setb %r10b
; EGPR-NEXT: movq %r18, %rax
; EGPR-NEXT: mulq %rcx
-; EGPR-NEXT: movq %rdx, %r28
-; EGPR-NEXT: movq %rax, %r29
-; EGPR-NEXT: addq %r8, %r29
+; EGPR-NEXT: movq %rdx, %r30
+; EGPR-NEXT: movq %rax, %r31
+; EGPR-NEXT: addq %r8, %r31
; EGPR-NEXT: movzbl %r10b, %eax
-; EGPR-NEXT: adcq %rax, %r28
-; EGPR-NEXT: addq %r20, %r29
-; EGPR-NEXT: adcq %r30, %r28
+; EGPR-NEXT: adcq %rax, %r30
+; EGPR-NEXT: addq %r28, %r31
+; EGPR-NEXT: adcq %r20, %r30
; EGPR-NEXT: adcq $0, %r16
; EGPR-NEXT: adcq $0, %r9
-; EGPR-NEXT: movq 48(%rsi), %r20
+; EGPR-NEXT: movq 48(%rsi), %r28
; EGPR-NEXT: movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NEXT: movq %r13, %rax
-; EGPR-NEXT: mulq %r20
+; EGPR-NEXT: mulq %r28
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, %r11
; EGPR-NEXT: movq %r18, %rax
; EGPR-NEXT: movq %r18, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NEXT: mulq %r20
-; EGPR-NEXT: movq %rdx, %r30
+; EGPR-NEXT: mulq %r28
+; EGPR-NEXT: movq %rdx, %r20
; EGPR-NEXT: movq %rax, %r14
; EGPR-NEXT: addq %r8, %r14
-; EGPR-NEXT: adcq $0, %r30
+; EGPR-NEXT: adcq $0, %r20
; EGPR-NEXT: movq 56(%rsi), %r10
; EGPR-NEXT: movq %r13, %rax
; EGPR-NEXT: mulq %r10
; EGPR-NEXT: movq %rdx, %r13
; EGPR-NEXT: addq %r14, %rax
; EGPR-NEXT: movq %rax, %r14
-; EGPR-NEXT: adcq %r30, %r13
+; EGPR-NEXT: adcq %r20, %r13
; EGPR-NEXT: setb %sil
; EGPR-NEXT: movq %r18, %rax
; EGPR-NEXT: mulq %r10
-; EGPR-NEXT: movq %rdx, %r30
+; EGPR-NEXT: movq %rdx, %r20
; EGPR-NEXT: movq %rax, %r8
; EGPR-NEXT: addq %r13, %r8
; EGPR-NEXT: movzbl %sil, %eax
-; EGPR-NEXT: adcq %rax, %r30
-; EGPR-NEXT: addq %r29, %r11
-; EGPR-NEXT: adcq %r28, %r14
+; EGPR-NEXT: adcq %rax, %r20
+; EGPR-NEXT: addq %r31, %r11
+; EGPR-NEXT: adcq %r30, %r14
; EGPR-NEXT: adcq $0, %r8
-; EGPR-NEXT: adcq $0, %r30
+; EGPR-NEXT: adcq $0, %r20
; EGPR-NEXT: addq %r16, %r8
-; EGPR-NEXT: adcq %r9, %r30
+; EGPR-NEXT: adcq %r9, %r20
; EGPR-NEXT: setb %r18b
; EGPR-NEXT: movq %r17, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NEXT: movq %r17, %rax
-; EGPR-NEXT: mulq %r20
+; EGPR-NEXT: mulq %r28
; EGPR-NEXT: movq %rdx, %r9
-; EGPR-NEXT: movq %rax, %r28
-; EGPR-NEXT: movq %r21, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NEXT: movq %r21, %rax
-; EGPR-NEXT: mulq %r20
+; EGPR-NEXT: movq %rax, %r30
+; EGPR-NEXT: movq %r29, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; EGPR-NEXT: movq %r29, %rax
+; EGPR-NEXT: mulq %r28
; EGPR-NEXT: movq %rdx, %r16
-; EGPR-NEXT: movq %rax, %r29
-; EGPR-NEXT: addq %r9, %r29
+; EGPR-NEXT: movq %rax, %r31
+; EGPR-NEXT: addq %r9, %r31
; EGPR-NEXT: adcq $0, %r16
; EGPR-NEXT: movq %r17, %rax
; EGPR-NEXT: mulq %r10
; EGPR-NEXT: movq %rdx, %r9
; EGPR-NEXT: movq %rax, %r17
-; EGPR-NEXT: addq %r29, %r17
+; EGPR-NEXT: addq %r31, %r17
; EGPR-NEXT: adcq %r16, %r9
; EGPR-NEXT: setb %r16b
-; EGPR-NEXT: movq %r21, %rax
+; EGPR-NEXT: movq %r29, %rax
; EGPR-NEXT: mulq %r10
; EGPR-NEXT: movq %rdx, %r13
-; EGPR-NEXT: movq %rax, %r29
-; EGPR-NEXT: addq %r9, %r29
+; EGPR-NEXT: movq %rax, %r31
+; EGPR-NEXT: addq %r9, %r31
; EGPR-NEXT: movzbl %r16b, %eax
; EGPR-NEXT: adcq %rax, %r13
-; EGPR-NEXT: addq %r8, %r28
-; EGPR-NEXT: adcq %r30, %r17
+; EGPR-NEXT: addq %r8, %r30
+; EGPR-NEXT: adcq %r20, %r17
; EGPR-NEXT: movzbl %r18b, %eax
-; EGPR-NEXT: adcq %rax, %r29
+; EGPR-NEXT: adcq %rax, %r31
; EGPR-NEXT: adcq $0, %r13
; EGPR-NEXT: addq %rbx, %r19
; EGPR-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
@@ -399,13 +399,13 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NEXT: adcq %r12, %r14
; EGPR-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NEXT: adcq $0, %r28
+; EGPR-NEXT: adcq $0, %r30
; EGPR-NEXT: adcq $0, %r17
-; EGPR-NEXT: adcq $0, %r29
+; EGPR-NEXT: adcq $0, %r31
; EGPR-NEXT: adcq $0, %r13
-; EGPR-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r28 # 8-byte Folded Reload
+; EGPR-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r30 # 8-byte Folded Reload
; EGPR-NEXT: adcq %rdi, %r17
-; EGPR-NEXT: adcq %r24, %r29
+; EGPR-NEXT: adcq %r24, %r31
; EGPR-NEXT: adcq %r23, %r13
; EGPR-NEXT: setb %r15b
; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
@@ -421,7 +421,7 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: addq %r8, %r16
; EGPR-NEXT: adcq $0, %r9
; EGPR-NEXT: movq %rsi, %rax
-; EGPR-NEXT: movq %rsi, %r21
+; EGPR-NEXT: movq %rsi, %r29
; EGPR-NEXT: mulq %rcx
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, %r22
@@ -459,20 +459,20 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %rsi, %r23
; EGPR-NEXT: mulq %rcx
; EGPR-NEXT: movq %rdx, %r24
-; EGPR-NEXT: movq %rax, %r30
-; EGPR-NEXT: addq %r8, %r30
+; EGPR-NEXT: movq %rax, %r20
+; EGPR-NEXT: addq %r8, %r20
; EGPR-NEXT: movzbl %r18b, %eax
; EGPR-NEXT: adcq %rax, %r24
-; EGPR-NEXT: addq %r19, %r30
+; EGPR-NEXT: addq %r19, %r20
; EGPR-NEXT: adcq %r22, %r24
; EGPR-NEXT: adcq $0, %r16
; EGPR-NEXT: adcq $0, %r9
; EGPR-NEXT: movq %rbx, %rax
-; EGPR-NEXT: mulq %r20
+; EGPR-NEXT: mulq %r28
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, %rsi
; EGPR-NEXT: movq %r23, %rax
-; EGPR-NEXT: mulq %r20
+; EGPR-NEXT: mulq %r28
; EGPR-NEXT: movq %rdx, %r19
; EGPR-NEXT: movq %rax, %r22
; EGPR-NEXT: addq %r8, %r22
@@ -491,24 +491,24 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: addq %rbx, %r8
; EGPR-NEXT: movzbl %r18b, %eax
; EGPR-NEXT: adcq %rax, %r23
-; EGPR-NEXT: addq %r30, %rsi
+; EGPR-NEXT: addq %r20, %rsi
; EGPR-NEXT: adcq %r24, %r22
; EGPR-NEXT: adcq $0, %r8
; EGPR-NEXT: adcq $0, %r23
; EGPR-NEXT: addq %r16, %r8
; EGPR-NEXT: adcq %r9, %r23
; EGPR-NEXT: setb %r18b
-; EGPR-NEXT: movq %r21, %rax
-; EGPR-NEXT: mulq %r20
+; EGPR-NEXT: movq %r29, %rax
+; EGPR-NEXT: mulq %r28
; EGPR-NEXT: movq %rdx, %r9
; EGPR-NEXT: movq %rax, %r24
; EGPR-NEXT: movq %r14, %rax
-; EGPR-NEXT: mulq %r20
+; EGPR-NEXT: mulq %r28
; EGPR-NEXT: movq %rdx, %r16
; EGPR-NEXT: movq %rax, %r19
; EGPR-NEXT: addq %r9, %r19
; EGPR-NEXT: adcq $0, %r16
-; EGPR-NEXT: movq %r21, %rax
+; EGPR-NEXT: movq %r29, %rax
; EGPR-NEXT: mulq %r10
; EGPR-NEXT: movq %rdx, %r9
; EGPR-NEXT: addq %r19, %rax
@@ -527,11 +527,11 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movzbl %r18b, %eax
; EGPR-NEXT: adcq %rax, %r12
; EGPR-NEXT: adcq $0, %rbp
-; EGPR-NEXT: addq %r28, %rdi
+; EGPR-NEXT: addq %r30, %rdi
; EGPR-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NEXT: adcq %r17, %r11
; EGPR-NEXT: movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NEXT: adcq %r29, %rsi
+; EGPR-NEXT: adcq %r31, %rsi
; EGPR-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NEXT: adcq %r13, %r22
; EGPR-NEXT: movq %r22, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
@@ -554,84 +554,84 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %rax, %r16
; EGPR-NEXT: addq %r8, %r16
; EGPR-NEXT: adcq $0, %r9
-; EGPR-NEXT: movq 72(%r26), %r28
+; EGPR-NEXT: movq 72(%r26), %r30
; EGPR-NEXT: movq %rdi, %rax
-; EGPR-NEXT: mulq %r28
+; EGPR-NEXT: mulq %r30
; EGPR-NEXT: movq %rdx, %r8
-; EGPR-NEXT: movq %rax, %r30
-; EGPR-NEXT: addq %r16, %r30
+; EGPR-NEXT: movq %rax, %r20
+; EGPR-NEXT: addq %r16, %r20
; EGPR-NEXT: adcq %r9, %r8
; EGPR-NEXT: setb %r18b
; EGPR-NEXT: movq %r25, %rax
-; EGPR-NEXT: mulq %r28
+; EGPR-NEXT: mulq %r30
; EGPR-NEXT: movq %rdx, %r9
; EGPR-NEXT: movq %rax, %r16
; EGPR-NEXT: addq %r8, %r16
; EGPR-NEXT: movzbl %r18b, %eax
; EGPR-NEXT: adcq %rax, %r9
-; EGPR-NEXT: movq %r31, %rax
+; EGPR-NEXT: movq %r21, %rax
; EGPR-NEXT: mulq %r23
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
; EGPR-NEXT: movq %r11, %rax
; EGPR-NEXT: mulq %r23
-; EGPR-NEXT: movq %rdx, %r29
+; EGPR-NEXT: movq %rdx, %r31
; EGPR-NEXT: movq %rax, %rbx
; EGPR-NEXT: addq %r8, %rbx
-; EGPR-NEXT: adcq $0, %r29
-; EGPR-NEXT: movq %r31, %rax
-; EGPR-NEXT: mulq %r28
+; EGPR-NEXT: adcq $0, %r31
+; EGPR-NEXT: movq %r21, %rax
+; EGPR-NEXT: mulq %r30
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: addq %rbx, %rax
; EGPR-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NEXT: adcq %r29, %r8
+; EGPR-NEXT: adcq %r31, %r8
; EGPR-NEXT: setb %r18b
; EGPR-NEXT: movq %r11, %rax
-; EGPR-NEXT: mulq %r28
-; EGPR-NEXT: movq %rdx, %r29
+; EGPR-NEXT: mulq %r30
+; EGPR-NEXT: movq %rdx, %r31
; EGPR-NEXT: movq %rax, %rbx
; EGPR-NEXT: addq %r8, %rbx
; EGPR-NEXT: movzbl %r18b, %eax
-; EGPR-NEXT: adcq %rax, %r29
+; EGPR-NEXT: adcq %rax, %r31
; EGPR-NEXT: addq %r24, %rbx
-; EGPR-NEXT: adcq %r30, %r29
+; EGPR-NEXT: adcq %r20, %r31
; EGPR-NEXT: adcq $0, %r16
; EGPR-NEXT: adcq $0, %r9
; EGPR-NEXT: movq 80(%r26), %r13
-; EGPR-NEXT: movq %r31, %rax
+; EGPR-NEXT: movq %r21, %rax
; EGPR-NEXT: mulq %r13
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, %rsi
; EGPR-NEXT: movq %r11, %rax
; EGPR-NEXT: mulq %r13
-; EGPR-NEXT: movq %rdx, %r30
+; EGPR-NEXT: movq %rdx, %r20
; EGPR-NEXT: movq %rax, %r14
; EGPR-NEXT: addq %r8, %r14
-; EGPR-NEXT: adcq $0, %r30
+; EGPR-NEXT: adcq $0, %r20
; EGPR-NEXT: movq 88(%r26), %r18
-; EGPR-NEXT: movq %r31, %rax
+; EGPR-NEXT: movq %r21, %rax
; EGPR-NEXT: mulq %r18
; EGPR-NEXT: movq %rdx, %r15
; EGPR-NEXT: movq %rax, %r24
; EGPR-NEXT: addq %r14, %r24
-; EGPR-NEXT: adcq %r30, %r15
+; EGPR-NEXT: adcq %r20, %r15
; EGPR-NEXT: setb %r14b
; EGPR-NEXT: movq %r11, %rax
; EGPR-NEXT: mulq %r18
-; EGPR-NEXT: movq %rdx, %r30
+; EGPR-NEXT: movq %rdx, %r20
; EGPR-NEXT: movq %rax, %r8
; EGPR-NEXT: addq %r15, %r8
; EGPR-NEXT: movzbl %r14b, %eax
-; EGPR-NEXT: adcq %rax, %r30
+; EGPR-NEXT: adcq %rax, %r20
; EGPR-NEXT: addq %rbx, %rsi
; EGPR-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NEXT: adcq %r29, %r24
+; EGPR-NEXT: adcq %r31, %r24
; EGPR-NEXT: adcq $0, %r8
-; EGPR-NEXT: adcq $0, %r30
+; EGPR-NEXT: adcq $0, %r20
; EGPR-NEXT: addq %r16, %r8
-; EGPR-NEXT: adcq %r9, %r30
-; EGPR-NEXT: setb %r29b
+; EGPR-NEXT: adcq %r9, %r20
+; EGPR-NEXT: setb %r31b
; EGPR-NEXT: movq %rdi, %rax
; EGPR-NEXT: mulq %r13
; EGPR-NEXT: movq %rdx, %r9
@@ -657,8 +657,8 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movzbl %r16b, %eax
; EGPR-NEXT: adcq %rax, %r14
; EGPR-NEXT: addq %r8, %rsi
-; EGPR-NEXT: adcq %r30, %rbx
-; EGPR-NEXT: movzbl %r29b, %eax
+; EGPR-NEXT: adcq %r20, %rbx
+; EGPR-NEXT: movzbl %r31b, %eax
; EGPR-NEXT: adcq %rax, %r15
; EGPR-NEXT: adcq $0, %r14
; EGPR-NEXT: imulq %r27, %r18
@@ -668,95 +668,95 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: addq %r18, %rdx
; EGPR-NEXT: imulq %rcx, %r13
; EGPR-NEXT: addq %rdx, %r13
-; EGPR-NEXT: movq %r20, %r9
-; EGPR-NEXT: imulq %r28, %r9
-; EGPR-NEXT: movq %r20, %rax
+; EGPR-NEXT: movq %r28, %r9
+; EGPR-NEXT: imulq %r30, %r9
+; EGPR-NEXT: movq %r28, %rax
; EGPR-NEXT: mulq %r23
-; EGPR-NEXT: movq %rax, %r30
+; EGPR-NEXT: movq %rax, %r20
; EGPR-NEXT: addq %r9, %rdx
; EGPR-NEXT: imulq %r23, %r10
; EGPR-NEXT: addq %rdx, %r10
-; EGPR-NEXT: addq %r8, %r30
+; EGPR-NEXT: addq %r8, %r20
; EGPR-NEXT: adcq %r13, %r10
; EGPR-NEXT: movq %r23, %rax
; EGPR-NEXT: mulq %r27
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, %r9
-; EGPR-NEXT: movq %r28, %rax
+; EGPR-NEXT: movq %r30, %rax
; EGPR-NEXT: mulq %r27
; EGPR-NEXT: movq %rdx, %r27
-; EGPR-NEXT: movq %rax, %r20
-; EGPR-NEXT: addq %r8, %r20
+; EGPR-NEXT: movq %rax, %r28
+; EGPR-NEXT: addq %r8, %r28
; EGPR-NEXT: adcq $0, %r27
; EGPR-NEXT: movq %r23, %rax
; EGPR-NEXT: mulq %rcx
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, %r16
-; EGPR-NEXT: addq %r20, %r16
+; EGPR-NEXT: addq %r28, %r16
; EGPR-NEXT: adcq %r27, %r8
; EGPR-NEXT: setb %r18b
-; EGPR-NEXT: movq %r28, %rax
+; EGPR-NEXT: movq %r30, %rax
; EGPR-NEXT: mulq %rcx
; EGPR-NEXT: movq %rdx, %r23
-; EGPR-NEXT: movq %rax, %r20
-; EGPR-NEXT: addq %r8, %r20
+; EGPR-NEXT: movq %rax, %r28
+; EGPR-NEXT: addq %r8, %r28
; EGPR-NEXT: movzbl %r18b, %eax
; EGPR-NEXT: adcq %rax, %r23
-; EGPR-NEXT: addq %r30, %r20
+; EGPR-NEXT: addq %r20, %r28
; EGPR-NEXT: adcq %r10, %r23
; EGPR-NEXT: movq 112(%r26), %rcx
-; EGPR-NEXT: movq %r31, %rax
+; EGPR-NEXT: movq %r21, %rax
; EGPR-NEXT: mulq %rcx
; EGPR-NEXT: movq %rax, %r8
; EGPR-NEXT: imulq %r11, %rcx
; EGPR-NEXT: addq %rdx, %rcx
; EGPR-NEXT: movq 120(%r26), %rax
-; EGPR-NEXT: imulq %r31, %rax
+; EGPR-NEXT: imulq %r21, %rax
; EGPR-NEXT: addq %rax, %rcx
; EGPR-NEXT: movq 96(%r26), %r27
-; EGPR-NEXT: movq 104(%r26), %r30
+; EGPR-NEXT: movq 104(%r26), %r20
; EGPR-NEXT: movq %rdi, %rax
-; EGPR-NEXT: imulq %r30, %rdi
+; EGPR-NEXT: imulq %r20, %rdi
; EGPR-NEXT: mulq %r27
-; EGPR-NEXT: movq %rax, %r21
+; EGPR-NEXT: movq %rax, %r29
; EGPR-NEXT: addq %rdi, %rdx
; EGPR-NEXT: imulq %r27, %r25
; EGPR-NEXT: addq %rdx, %r25
-; EGPR-NEXT: addq %r8, %r21
+; EGPR-NEXT: addq %r8, %r29
; EGPR-NEXT: adcq %rcx, %r25
; EGPR-NEXT: movq %r27, %rax
-; EGPR-NEXT: mulq %r31
+; EGPR-NEXT: mulq %r21
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, %r22
-; EGPR-NEXT: movq %r30, %rax
-; EGPR-NEXT: mulq %r31
-; EGPR-NEXT: movq %rdx, %r31
-; EGPR-NEXT: movq %rax, %r28
-; EGPR-NEXT: addq %r8, %r28
-; EGPR-NEXT: adcq $0, %r31
+; EGPR-NEXT: movq %r20, %rax
+; EGPR-NEXT: mulq %r21
+; EGPR-NEXT: movq %rdx, %r21
+; EGPR-NEXT: movq %rax, %r30
+; EGPR-NEXT: addq %r8, %r30
+; EGPR-NEXT: adcq $0, %r21
; EGPR-NEXT: movq %r27, %rax
; EGPR-NEXT: mulq %r11
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, %r27
-; EGPR-NEXT: addq %r28, %r27
-; EGPR-NEXT: adcq %r31, %r8
+; EGPR-NEXT: addq %r30, %r27
+; EGPR-NEXT: adcq %r21, %r8
; EGPR-NEXT: setb %cl
-; EGPR-NEXT: movq %r30, %rax
+; EGPR-NEXT: movq %r20, %rax
; EGPR-NEXT: mulq %r11
; EGPR-NEXT: movq %rdx, %r26
-; EGPR-NEXT: movq %rax, %r31
-; EGPR-NEXT: addq %r8, %r31
+; EGPR-NEXT: movq %rax, %r21
+; EGPR-NEXT: addq %r8, %r21
; EGPR-NEXT: movzbl %cl, %eax
; EGPR-NEXT: adcq %rax, %r26
-; EGPR-NEXT: addq %r21, %r31
+; EGPR-NEXT: addq %r29, %r21
; EGPR-NEXT: adcq %r25, %r26
; EGPR-NEXT: addq %r9, %r22
; EGPR-NEXT: adcq %r16, %r27
-; EGPR-NEXT: adcq %r20, %r31
+; EGPR-NEXT: adcq %r28, %r21
; EGPR-NEXT: adcq %r23, %r26
; EGPR-NEXT: addq %rsi, %r22
; EGPR-NEXT: adcq %rbx, %r27
-; EGPR-NEXT: adcq %r15, %r31
+; EGPR-NEXT: adcq %r15, %r21
; EGPR-NEXT: adcq %r14, %r26
; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
; EGPR-NEXT: movq 80(%r11), %rbx
@@ -765,8 +765,8 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: mulq %r19
; EGPR-NEXT: movq %rax, %r23
; EGPR-NEXT: movq %rdx, %r8
-; EGPR-NEXT: movq 88(%r11), %r20
-; EGPR-NEXT: movq %r20, %rax
+; EGPR-NEXT: movq 88(%r11), %r28
+; EGPR-NEXT: movq %r28, %rax
; EGPR-NEXT: mulq %r19
; EGPR-NEXT: movq %rdx, %r9
; EGPR-NEXT: movq %rax, %r16
@@ -776,11 +776,11 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r17 # 8-byte Reload
; EGPR-NEXT: mulq %r17
; EGPR-NEXT: movq %rdx, %r8
-; EGPR-NEXT: movq %rax, %r30
-; EGPR-NEXT: addq %r16, %r30
+; EGPR-NEXT: movq %rax, %r20
+; EGPR-NEXT: addq %r16, %r20
; EGPR-NEXT: adcq %r9, %r8
; EGPR-NEXT: setb %cl
-; EGPR-NEXT: movq %r20, %rax
+; EGPR-NEXT: movq %r28, %rax
; EGPR-NEXT: mulq %r17
; EGPR-NEXT: movq %rdx, %r9
; EGPR-NEXT: movq %rax, %r16
@@ -795,66 +795,66 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq 72(%r11), %r14
; EGPR-NEXT: movq %r14, %rax
; EGPR-NEXT: mulq %r19
-; EGPR-NEXT: movq %rdx, %r28
-; EGPR-NEXT: movq %rax, %r29
-; EGPR-NEXT: addq %r8, %r29
-; EGPR-NEXT: adcq $0, %r28
+; EGPR-NEXT: movq %rdx, %r30
+; EGPR-NEXT: movq %rax, %r31
+; EGPR-NEXT: addq %r8, %r31
+; EGPR-NEXT: adcq $0, %r30
; EGPR-NEXT: movq %r15, %rax
; EGPR-NEXT: mulq %r17
; EGPR-NEXT: movq %rdx, %r8
-; EGPR-NEXT: movq %rax, %r21
-; EGPR-NEXT: addq %r29, %r21
-; EGPR-NEXT: adcq %r28, %r8
+; EGPR-NEXT: movq %rax, %r29
+; EGPR-NEXT: addq %r31, %r29
+; EGPR-NEXT: adcq %r30, %r8
; EGPR-NEXT: setb %cl
; EGPR-NEXT: movq %r14, %rax
; EGPR-NEXT: mulq %r17
-; EGPR-NEXT: movq %rdx, %r29
+; EGPR-NEXT: movq %rdx, %r31
; EGPR-NEXT: movq %rax, %r13
; EGPR-NEXT: addq %r8, %r13
; EGPR-NEXT: movzbl %cl, %eax
-; EGPR-NEXT: adcq %rax, %r29
+; EGPR-NEXT: adcq %rax, %r31
; EGPR-NEXT: addq %r23, %r13
-; EGPR-NEXT: adcq %r30, %r29
+; EGPR-NEXT: adcq %r20, %r31
; EGPR-NEXT: adcq $0, %r16
; EGPR-NEXT: adcq $0, %r9
; EGPR-NEXT: movq %r15, %rax
; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; EGPR-NEXT: mulq %rdi
; EGPR-NEXT: movq %rdx, %r8
-; EGPR-NEXT: movq %rax, %r28
+; EGPR-NEXT: movq %rax, %r30
; EGPR-NEXT: movq %r14, %rax
; EGPR-NEXT: mulq %rdi
-; EGPR-NEXT: movq %rdx, %r30
+; EGPR-NEXT: movq %rdx, %r20
; EGPR-NEXT: movq %rax, %rcx
; EGPR-NEXT: addq %r8, %rcx
-; EGPR-NEXT: adcq $0, %r30
+; EGPR-NEXT: adcq $0, %r20
; EGPR-NEXT: movq %r15, %rax
; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r18 # 8-byte Reload
; EGPR-NEXT: mulq %r18
; EGPR-NEXT: movq %rdx, %r10
; EGPR-NEXT: movq %rax, %r23
; EGPR-NEXT: addq %rcx, %r23
-; EGPR-NEXT: adcq %r30, %r10
+; EGPR-NEXT: adcq %r20, %r10
; EGPR-NEXT: setb %cl
; EGPR-NEXT: movq %r14, %rax
; EGPR-NEXT: mulq %r18
-; EGPR-NEXT: movq %rdx, %r30
+; EGPR-NEXT: movq %rdx, %r20
; EGPR-NEXT: movq %rax, %r8
; EGPR-NEXT: addq %r10, %r8
; EGPR-NEXT: movzbl %cl, %eax
-; EGPR-NEXT: adcq %rax, %r30
-; EGPR-NEXT: addq %r13, %r28
-; EGPR-NEXT: adcq %r29, %r23
+; EGPR-NEXT: adcq %rax, %r20
+; EGPR-NEXT: addq %r13, %r30
+; EGPR-NEXT: adcq %r31, %r23
; EGPR-NEXT: adcq $0, %r8
-; EGPR-NEXT: adcq $0, %r30
+; EGPR-NEXT: adcq $0, %r20
; EGPR-NEXT: addq %r16, %r8
-; EGPR-NEXT: adcq %r9, %r30
+; EGPR-NEXT: adcq %r9, %r20
; EGPR-NEXT: setb %sil
; EGPR-NEXT: movq %rbx, %rax
; EGPR-NEXT: mulq %rdi
; EGPR-NEXT: movq %rdx, %rcx
-; EGPR-NEXT: movq %rax, %r29
-; EGPR-NEXT: movq %r20, %rax
+; EGPR-NEXT: movq %rax, %r31
+; EGPR-NEXT: movq %r28, %rax
; EGPR-NEXT: mulq %rdi
; EGPR-NEXT: movq %rdx, %r9
; EGPR-NEXT: movq %rax, %r10
@@ -867,15 +867,15 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: addq %r10, %r13
; EGPR-NEXT: adcq %r9, %rcx
; EGPR-NEXT: setb %r10b
-; EGPR-NEXT: movq %r20, %rax
+; EGPR-NEXT: movq %r28, %rax
; EGPR-NEXT: mulq %r18
; EGPR-NEXT: movq %rdx, %r16
; EGPR-NEXT: movq %rax, %r9
; EGPR-NEXT: addq %rcx, %r9
; EGPR-NEXT: movzbl %r10b, %eax
; EGPR-NEXT: adcq %rax, %r16
-; EGPR-NEXT: addq %r8, %r29
-; EGPR-NEXT: adcq %r30, %r13
+; EGPR-NEXT: addq %r8, %r31
+; EGPR-NEXT: adcq %r20, %r13
; EGPR-NEXT: movzbl %sil, %eax
; EGPR-NEXT: adcq %rax, %r9
; EGPR-NEXT: adcq $0, %r16
@@ -885,9 +885,9 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: mulq %rdi
; EGPR-NEXT: movq %rax, %r8
; EGPR-NEXT: addq %r18, %rdx
-; EGPR-NEXT: movq 104(%r11), %r30
+; EGPR-NEXT: movq 104(%r11), %r20
; EGPR-NEXT: movq %rdi, %rax
-; EGPR-NEXT: imulq %r30, %rax
+; EGPR-NEXT: imulq %r20, %rax
; EGPR-NEXT: addq %rdx, %rax
; EGPR-NEXT: movq %rax, %r10
; EGPR-NEXT: movq 112(%r11), %rax
@@ -912,14 +912,14 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: addq %r8, %r10
; EGPR-NEXT: adcq $0, %rcx
; EGPR-NEXT: movq %r19, %rax
-; EGPR-NEXT: mulq %r30
+; EGPR-NEXT: mulq %r20
; EGPR-NEXT: movq %rdx, %r8
; EGPR-NEXT: movq %rax, %r11
; EGPR-NEXT: addq %r10, %r11
; EGPR-NEXT: adcq %rcx, %r8
; EGPR-NEXT: setb %cl
; EGPR-NEXT: movq %r17, %rax
-; EGPR-NEXT: mulq %r30
+; EGPR-NEXT: mulq %r20
; EGPR-NEXT: movq %rdx, %r10
; EGPR-NEXT: movq %rax, %r17
; EGPR-NEXT: addq %r8, %r17
@@ -944,12 +944,12 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq %rbx, %rax
; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
; EGPR-NEXT: mulq %r8
-; EGPR-NEXT: movq %rax, %r30
+; EGPR-NEXT: movq %rax, %r20
; EGPR-NEXT: addq %rdi, %rdx
-; EGPR-NEXT: imulq %r8, %r20
-; EGPR-NEXT: addq %rdx, %r20
-; EGPR-NEXT: addq %rcx, %r30
-; EGPR-NEXT: adcq %r18, %r20
+; EGPR-NEXT: imulq %r8, %r28
+; EGPR-NEXT: addq %rdx, %r28
+; EGPR-NEXT: addq %rcx, %r20
+; EGPR-NEXT: adcq %r18, %r28
; EGPR-NEXT: movq %r8, %rax
; EGPR-NEXT: movq %r8, %rdi
; EGPR-NEXT: mulq %r15
@@ -973,27 +973,27 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: addq %rcx, %rax
; EGPR-NEXT: movzbl %dil, %ecx
; EGPR-NEXT: adcq %rcx, %rdx
-; EGPR-NEXT: addq %r30, %rax
-; EGPR-NEXT: adcq %r20, %rdx
+; EGPR-NEXT: addq %r20, %rax
+; EGPR-NEXT: adcq %r28, %rdx
; EGPR-NEXT: addq %rsi, %r8
; EGPR-NEXT: adcq %r11, %r18
; EGPR-NEXT: adcq %r17, %rax
; EGPR-NEXT: adcq %r10, %rdx
-; EGPR-NEXT: addq %r29, %r8
+; EGPR-NEXT: addq %r31, %r8
; EGPR-NEXT: adcq %r13, %r18
; EGPR-NEXT: adcq %r9, %rax
; EGPR-NEXT: adcq %r16, %rdx
; EGPR-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r25 # 8-byte Folded Reload
-; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r21 # 8-byte Folded Reload
-; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r28 # 8-byte Folded Reload
+; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r29 # 8-byte Folded Reload
+; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r30 # 8-byte Folded Reload
; EGPR-NEXT: adcq %r24, %r23
; EGPR-NEXT: adcq %r22, %r8
; EGPR-NEXT: adcq %r27, %r18
-; EGPR-NEXT: adcq %r31, %rax
+; EGPR-NEXT: adcq %r21, %rax
; EGPR-NEXT: adcq %r26, %rdx
; EGPR-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r25 # 8-byte Folded Reload
-; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r21 # 8-byte Folded Reload
-; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r28 # 8-byte Folded Reload
+; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r29 # 8-byte Folded Reload
+; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r30 # 8-byte Folded Reload
; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r23 # 8-byte Folded Reload
; EGPR-NEXT: adcq (%rsp), %r8 # 8-byte Folded Reload
; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r18 # 8-byte Folded Reload
@@ -1017,8 +1017,8 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; EGPR-NEXT: movq %rsi, 56(%rcx)
; EGPR-NEXT: movq %r25, 64(%rcx)
-; EGPR-NEXT: movq %r21, 72(%rcx)
-; EGPR-NEXT: movq %r28, 80(%rcx)
+; EGPR-NEXT: movq %r29, 72(%rcx)
+; EGPR-NEXT: movq %r30, 80(%rcx)
; EGPR-NEXT: movq %r23, 88(%rcx)
; EGPR-NEXT: movq %r8, 96(%rcx)
; EGPR-NEXT: movq %r18, 104(%rcx)
@@ -1073,35 +1073,35 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: movzbl %al, %r8d
; EGPR-NDD-NEXT: movq %r18, %rax
; EGPR-NDD-NEXT: mulq %r23
-; EGPR-NDD-NEXT: addq %rcx, %rax, %r31
+; EGPR-NDD-NEXT: addq %rcx, %rax, %r21
; EGPR-NDD-NEXT: adcq %rdx, %r8
; EGPR-NDD-NEXT: movq %r16, %rax
; EGPR-NDD-NEXT: mulq %r24
-; EGPR-NDD-NEXT: movq %rdx, %r30
+; EGPR-NDD-NEXT: movq %rdx, %r20
; EGPR-NDD-NEXT: movq %rax, %r27
; EGPR-NDD-NEXT: movq %rdi, %rax
; EGPR-NDD-NEXT: mulq %r24
-; EGPR-NDD-NEXT: addq %r30, %rax, %rcx
-; EGPR-NDD-NEXT: adcq $0, %rdx, %r30
+; EGPR-NDD-NEXT: addq %r20, %rax, %rcx
+; EGPR-NDD-NEXT: adcq $0, %rdx, %r20
; EGPR-NDD-NEXT: movq %r16, %rax
; EGPR-NDD-NEXT: mulq %r23
; EGPR-NDD-NEXT: addq %rax, %rcx
-; EGPR-NDD-NEXT: adcq %rdx, %r30
+; EGPR-NDD-NEXT: adcq %rdx, %r20
; EGPR-NDD-NEXT: setb %al
-; EGPR-NDD-NEXT: movzbl %al, %r20d
+; EGPR-NDD-NEXT: movzbl %al, %r28d
; EGPR-NDD-NEXT: movq %rdi, %rax
; EGPR-NDD-NEXT: mulq %r23
-; EGPR-NDD-NEXT: addq %r30, %rax
-; EGPR-NDD-NEXT: adcq %r20, %rdx
-; EGPR-NDD-NEXT: addq %rax, %r19, %r20
-; EGPR-NDD-NEXT: adcq %rdx, %rsi, %r21
-; EGPR-NDD-NEXT: adcq $0, %r31
+; EGPR-NDD-NEXT: addq %r20, %rax
+; EGPR-NDD-NEXT: adcq %r28, %rdx
+; EGPR-NDD-NEXT: addq %rax, %r19, %r28
+; EGPR-NDD-NEXT: adcq %rdx, %rsi, %r29
+; EGPR-NDD-NEXT: adcq $0, %r21
; EGPR-NDD-NEXT: adcq $0, %r8
; EGPR-NDD-NEXT: movq %r16, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: movq %r16, %rax
; EGPR-NDD-NEXT: mulq %r26
; EGPR-NDD-NEXT: movq %rdx, %r19
-; EGPR-NDD-NEXT: movq %rax, %r30
+; EGPR-NDD-NEXT: movq %rax, %r20
; EGPR-NDD-NEXT: movq %rdi, %rax
; EGPR-NDD-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: mulq %r26
@@ -1112,47 +1112,47 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: addq %rax, %r19
; EGPR-NDD-NEXT: adcq %rdx, %rsi
; EGPR-NDD-NEXT: setb %al
-; EGPR-NDD-NEXT: movzbl %al, %r28d
+; EGPR-NDD-NEXT: movzbl %al, %r30d
; EGPR-NDD-NEXT: movq %rdi, %rax
; EGPR-NDD-NEXT: mulq %r14
; EGPR-NDD-NEXT: addq %rsi, %rax
-; EGPR-NDD-NEXT: adcq %r28, %rdx
-; EGPR-NDD-NEXT: addq %r20, %r30, %rsi
-; EGPR-NDD-NEXT: adcq %r21, %r19, %r20
+; EGPR-NDD-NEXT: adcq %r30, %rdx
+; EGPR-NDD-NEXT: addq %r28, %r20, %rsi
+; EGPR-NDD-NEXT: adcq %r29, %r19, %r28
; EGPR-NDD-NEXT: adcq $0, %rax
; EGPR-NDD-NEXT: adcq $0, %rdx
-; EGPR-NDD-NEXT: addq %rax, %r31
+; EGPR-NDD-NEXT: addq %rax, %r21
; EGPR-NDD-NEXT: adcq %rdx, %r8
; EGPR-NDD-NEXT: setb %al
-; EGPR-NDD-NEXT: movzbl %al, %r29d
+; EGPR-NDD-NEXT: movzbl %al, %r31d
; EGPR-NDD-NEXT: movq %r25, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: movq %r25, %rax
; EGPR-NDD-NEXT: mulq %r26
; EGPR-NDD-NEXT: movq %rdx, %r19
-; EGPR-NDD-NEXT: movq %rax, %r30
+; EGPR-NDD-NEXT: movq %rax, %r20
; EGPR-NDD-NEXT: movq %r18, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: movq %r18, %rax
; EGPR-NDD-NEXT: mulq %r26
; EGPR-NDD-NEXT: addq %rax, %r19
-; EGPR-NDD-NEXT: adcq $0, %rdx, %r21
+; EGPR-NDD-NEXT: adcq $0, %rdx, %r29
; EGPR-NDD-NEXT: movq %r25, %rax
; EGPR-NDD-NEXT: mulq %r14
; EGPR-NDD-NEXT: addq %rax, %r19
-; EGPR-NDD-NEXT: adcq %rdx, %r21
+; EGPR-NDD-NEXT: adcq %rdx, %r29
; EGPR-NDD-NEXT: setb %al
-; EGPR-NDD-NEXT: movzbl %al, %r28d
+; EGPR-NDD-NEXT: movzbl %al, %r30d
; EGPR-NDD-NEXT: movq %r18, %rax
; EGPR-NDD-NEXT: mulq %r14
-; EGPR-NDD-NEXT: addq %r21, %rax
-; EGPR-NDD-NEXT: adcq %r28, %rdx
-; EGPR-NDD-NEXT: addq %r31, %r30, %r21
-; EGPR-NDD-NEXT: adcq %r8, %r19, %r28
-; EGPR-NDD-NEXT: adcq %rax, %r29
+; EGPR-NDD-NEXT: addq %r29, %rax
+; EGPR-NDD-NEXT: adcq %r30, %rdx
+; EGPR-NDD-NEXT: addq %r21, %r20, %r29
+; EGPR-NDD-NEXT: adcq %r8, %r19, %r30
+; EGPR-NDD-NEXT: adcq %rax, %r31
; EGPR-NDD-NEXT: adcq $0, %rdx, %rdi
; EGPR-NDD-NEXT: movq %r10, %rax
; EGPR-NDD-NEXT: mulq %r24
; EGPR-NDD-NEXT: movq %rdx, %r19
-; EGPR-NDD-NEXT: movq %rax, %r30
+; EGPR-NDD-NEXT: movq %rax, %r20
; EGPR-NDD-NEXT: movq %r9, %rax
; EGPR-NDD-NEXT: mulq %r24
; EGPR-NDD-NEXT: addq %rax, %r19
@@ -1162,45 +1162,45 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: addq %rax, %r19
; EGPR-NDD-NEXT: adcq %rdx, %r8
; EGPR-NDD-NEXT: setb %al
-; EGPR-NDD-NEXT: movzbl %al, %r31d
+; EGPR-NDD-NEXT: movzbl %al, %r21d
; EGPR-NDD-NEXT: movq %r9, %rax
; EGPR-NDD-NEXT: mulq %r23
; EGPR-NDD-NEXT: addq %rax, %r8
-; EGPR-NDD-NEXT: adcq %r31, %rdx, %rbx
+; EGPR-NDD-NEXT: adcq %r21, %rdx, %rbx
; EGPR-NDD-NEXT: movq %r17, %rax
; EGPR-NDD-NEXT: mulq %r24
-; EGPR-NDD-NEXT: movq %rdx, %r31
+; EGPR-NDD-NEXT: movq %rdx, %r21
; EGPR-NDD-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: movq %r11, %rax
; EGPR-NDD-NEXT: mulq %r24
-; EGPR-NDD-NEXT: addq %rax, %r31
+; EGPR-NDD-NEXT: addq %rax, %r21
; EGPR-NDD-NEXT: adcq $0, %rdx, %r12
; EGPR-NDD-NEXT: movq %r17, %rax
; EGPR-NDD-NEXT: mulq %r23
-; EGPR-NDD-NEXT: addq %r31, %rax
+; EGPR-NDD-NEXT: addq %r21, %rax
; EGPR-NDD-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: adcq %rdx, %r12
-; EGPR-NDD-NEXT: setb %r31b
+; EGPR-NDD-NEXT: setb %r21b
; EGPR-NDD-NEXT: movq %r11, %rax
; EGPR-NDD-NEXT: mulq %r23
; EGPR-NDD-NEXT: addq %r12, %rax
-; EGPR-NDD-NEXT: movzbl %r31b, %r31d
-; EGPR-NDD-NEXT: adcq %r31, %rdx
-; EGPR-NDD-NEXT: addq %rax, %r30, %r12
+; EGPR-NDD-NEXT: movzbl %r21b, %r21d
+; EGPR-NDD-NEXT: adcq %r21, %rdx
+; EGPR-NDD-NEXT: addq %rax, %r20, %r12
; EGPR-NDD-NEXT: adcq %rdx, %r19
; EGPR-NDD-NEXT: adcq $0, %r8
; EGPR-NDD-NEXT: adcq $0, %rbx
; EGPR-NDD-NEXT: movq %r17, %rax
; EGPR-NDD-NEXT: mulq %r26
-; EGPR-NDD-NEXT: movq %rdx, %r30
-; EGPR-NDD-NEXT: movq %rax, %r31
+; EGPR-NDD-NEXT: movq %rdx, %r20
+; EGPR-NDD-NEXT: movq %rax, %r21
; EGPR-NDD-NEXT: movq %r11, %rax
; EGPR-NDD-NEXT: mulq %r26
-; EGPR-NDD-NEXT: addq %rax, %r30
+; EGPR-NDD-NEXT: addq %rax, %r20
; EGPR-NDD-NEXT: adcq $0, %rdx, %r13
; EGPR-NDD-NEXT: movq %r17, %rax
; EGPR-NDD-NEXT: mulq %r14
-; EGPR-NDD-NEXT: addq %rax, %r30
+; EGPR-NDD-NEXT: addq %rax, %r20
; EGPR-NDD-NEXT: adcq %rdx, %r13
; EGPR-NDD-NEXT: setb %bpl
; EGPR-NDD-NEXT: movq %r11, %rax
@@ -1208,9 +1208,9 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: addq %r13, %rax
; EGPR-NDD-NEXT: movzbl %bpl, %r13d
; EGPR-NDD-NEXT: adcq %r13, %rdx
-; EGPR-NDD-NEXT: addq %r12, %r31
-; EGPR-NDD-NEXT: movq %r31, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NDD-NEXT: adcq %r30, %r19
+; EGPR-NDD-NEXT: addq %r12, %r21
+; EGPR-NDD-NEXT: movq %r21, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; EGPR-NDD-NEXT: adcq %r20, %r19
; EGPR-NDD-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: adcq $0, %rax
; EGPR-NDD-NEXT: adcq $0, %rdx
@@ -1220,15 +1220,15 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: movq %r10, %r16
; EGPR-NDD-NEXT: movq %r10, %rax
; EGPR-NDD-NEXT: mulq %r26
-; EGPR-NDD-NEXT: movq %rdx, %r30
-; EGPR-NDD-NEXT: movq %rax, %r31
+; EGPR-NDD-NEXT: movq %rdx, %r20
+; EGPR-NDD-NEXT: movq %rax, %r21
; EGPR-NDD-NEXT: movq %r9, %rax
; EGPR-NDD-NEXT: mulq %r26
-; EGPR-NDD-NEXT: addq %rax, %r30
+; EGPR-NDD-NEXT: addq %rax, %r20
; EGPR-NDD-NEXT: adcq $0, %rdx, %r12
; EGPR-NDD-NEXT: movq %r10, %rax
; EGPR-NDD-NEXT: mulq %r14
-; EGPR-NDD-NEXT: addq %rax, %r30
+; EGPR-NDD-NEXT: addq %rax, %r20
; EGPR-NDD-NEXT: adcq %rdx, %r12
; EGPR-NDD-NEXT: setb %bpl
; EGPR-NDD-NEXT: movq %r9, %rax
@@ -1236,35 +1236,35 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: addq %r12, %rax
; EGPR-NDD-NEXT: movzbl %bpl, %r12d
; EGPR-NDD-NEXT: adcq %r12, %rdx
-; EGPR-NDD-NEXT: addq %r31, %r8
-; EGPR-NDD-NEXT: adcq %r30, %rbx
+; EGPR-NDD-NEXT: addq %r21, %r8
+; EGPR-NDD-NEXT: adcq %r20, %rbx
; EGPR-NDD-NEXT: movzbl %r19b, %r19d
; EGPR-NDD-NEXT: adcq %r19, %rax
; EGPR-NDD-NEXT: adcq $0, %rdx
; EGPR-NDD-NEXT: addq %r8, %r27, %r12
-; EGPR-NDD-NEXT: movq 32(%r15), %r30
+; EGPR-NDD-NEXT: movq 32(%r15), %r20
; EGPR-NDD-NEXT: adcq %rbx, %rcx, %r13
; EGPR-NDD-NEXT: adcq %rax, %rsi, %rbp
-; EGPR-NDD-NEXT: adcq %rdx, %r20, %rbx
-; EGPR-NDD-NEXT: adcq $0, %r21
-; EGPR-NDD-NEXT: movq %r21, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NDD-NEXT: adcq $0, %r28
+; EGPR-NDD-NEXT: adcq %rdx, %r28, %rbx
; EGPR-NDD-NEXT: adcq $0, %r29
+; EGPR-NDD-NEXT: movq %r29, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; EGPR-NDD-NEXT: adcq $0, %r30
+; EGPR-NDD-NEXT: adcq $0, %r31
; EGPR-NDD-NEXT: adcq $0, %rdi
; EGPR-NDD-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: movq %r10, %rax
-; EGPR-NDD-NEXT: mulq %r30
+; EGPR-NDD-NEXT: mulq %r20
; EGPR-NDD-NEXT: movq %rdx, %r27
-; EGPR-NDD-NEXT: movq %rax, %r31
+; EGPR-NDD-NEXT: movq %rax, %r21
; EGPR-NDD-NEXT: movq %r9, %r19
; EGPR-NDD-NEXT: movq %r9, %rax
-; EGPR-NDD-NEXT: mulq %r30
+; EGPR-NDD-NEXT: mulq %r20
; EGPR-NDD-NEXT: addq %rax, %r27
; EGPR-NDD-NEXT: adcq $0, %rdx, %rcx
; EGPR-NDD-NEXT: movq 40(%r15), %r18
; EGPR-NDD-NEXT: movq %r10, %rax
; EGPR-NDD-NEXT: mulq %r18
-; EGPR-NDD-NEXT: addq %r27, %rax, %r21
+; EGPR-NDD-NEXT: addq %r27, %rax, %r29
; EGPR-NDD-NEXT: adcq %rdx, %rcx
; EGPR-NDD-NEXT: setb %r8b
; EGPR-NDD-NEXT: movq %r9, %rax
@@ -1273,26 +1273,26 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: movzbl %r8b, %eax
; EGPR-NDD-NEXT: adcq %rax, %rdx, %rsi
; EGPR-NDD-NEXT: movq %r17, %rax
-; EGPR-NDD-NEXT: mulq %r30
-; EGPR-NDD-NEXT: movq %rdx, %r20
+; EGPR-NDD-NEXT: mulq %r20
+; EGPR-NDD-NEXT: movq %rdx, %r28
; EGPR-NDD-NEXT: movq %rax, %r27
; EGPR-NDD-NEXT: movq %r11, %r10
; EGPR-NDD-NEXT: movq %r11, %rax
-; EGPR-NDD-NEXT: mulq %r30
-; EGPR-NDD-NEXT: addq %r20, %rax, %r8
-; EGPR-NDD-NEXT: adcq $0, %rdx, %r20
+; EGPR-NDD-NEXT: mulq %r20
+; EGPR-NDD-NEXT: addq %r28, %rax, %r8
+; EGPR-NDD-NEXT: adcq $0, %rdx, %r28
; EGPR-NDD-NEXT: movq %r17, %rax
; EGPR-NDD-NEXT: mulq %r18
; EGPR-NDD-NEXT: addq %r8, %rax, %r25
-; EGPR-NDD-NEXT: adcq %rdx, %r20
+; EGPR-NDD-NEXT: adcq %rdx, %r28
; EGPR-NDD-NEXT: setb %cl
; EGPR-NDD-NEXT: movq %r11, %rax
; EGPR-NDD-NEXT: mulq %r18
-; EGPR-NDD-NEXT: addq %r20, %rax
+; EGPR-NDD-NEXT: addq %r28, %rax
; EGPR-NDD-NEXT: movzbl %cl, %ecx
; EGPR-NDD-NEXT: adcq %rdx, %rcx
-; EGPR-NDD-NEXT: addq %rax, %r31
-; EGPR-NDD-NEXT: adcq %rcx, %r21, %r8
+; EGPR-NDD-NEXT: addq %rax, %r21
+; EGPR-NDD-NEXT: adcq %rcx, %r29, %r8
; EGPR-NDD-NEXT: adcq $0, %rdi
; EGPR-NDD-NEXT: adcq $0, %rsi, %r9
; EGPR-NDD-NEXT: movq 48(%r15), %r11
@@ -1300,17 +1300,17 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: movq %r17, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: movq %r17, %rax
; EGPR-NDD-NEXT: mulq %r11
-; EGPR-NDD-NEXT: movq %rdx, %r20
-; EGPR-NDD-NEXT: movq %rax, %r21
+; EGPR-NDD-NEXT: movq %rdx, %r28
+; EGPR-NDD-NEXT: movq %rax, %r29
; EGPR-NDD-NEXT: movq %r10, %rax
; EGPR-NDD-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: mulq %r11
-; EGPR-NDD-NEXT: addq %rax, %r20
+; EGPR-NDD-NEXT: addq %rax, %r28
; EGPR-NDD-NEXT: adcq $0, %rdx, %rcx
; EGPR-NDD-NEXT: movq 56(%r15), %r17
; EGPR-NDD-NEXT: movq %rsi, %rax
; EGPR-NDD-NEXT: mulq %r17
-; EGPR-NDD-NEXT: addq %rax, %r20
+; EGPR-NDD-NEXT: addq %rax, %r28
; EGPR-NDD-NEXT: adcq %rdx, %rcx
; EGPR-NDD-NEXT: setb %sil
; EGPR-NDD-NEXT: movq %r10, %rax
@@ -1318,8 +1318,8 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: addq %rcx, %rax
; EGPR-NDD-NEXT: movzbl %sil, %ecx
; EGPR-NDD-NEXT: adcq %rdx, %rcx
-; EGPR-NDD-NEXT: addq %r21, %r31
-; EGPR-NDD-NEXT: adcq %r8, %r20, %r10
+; EGPR-NDD-NEXT: addq %r29, %r21
+; EGPR-NDD-NEXT: adcq %r8, %r28, %r10
; EGPR-NDD-NEXT: adcq $0, %rax
; EGPR-NDD-NEXT: adcq $0, %rcx
; EGPR-NDD-NEXT: addq %rax, %rdi
@@ -1328,16 +1328,16 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: movq %r16, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: movq %r16, %rax
; EGPR-NDD-NEXT: mulq %r11
-; EGPR-NDD-NEXT: movq %rdx, %r20
-; EGPR-NDD-NEXT: movq %rax, %r21
+; EGPR-NDD-NEXT: movq %rdx, %r28
+; EGPR-NDD-NEXT: movq %rax, %r29
; EGPR-NDD-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: movq %r19, %rax
; EGPR-NDD-NEXT: mulq %r11
-; EGPR-NDD-NEXT: addq %rax, %r20
+; EGPR-NDD-NEXT: addq %rax, %r28
; EGPR-NDD-NEXT: adcq $0, %rdx, %r9
; EGPR-NDD-NEXT: movq %r16, %rax
; EGPR-NDD-NEXT: mulq %r17
-; EGPR-NDD-NEXT: addq %rax, %r20
+; EGPR-NDD-NEXT: addq %rax, %r28
; EGPR-NDD-NEXT: adcq %rdx, %r9
; EGPR-NDD-NEXT: setb %cl
; EGPR-NDD-NEXT: movq %r19, %rax
@@ -1345,8 +1345,8 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: addq %r9, %rax
; EGPR-NDD-NEXT: movzbl %cl, %ecx
; EGPR-NDD-NEXT: adcq %rdx, %rcx
-; EGPR-NDD-NEXT: addq %r21, %rdi
-; EGPR-NDD-NEXT: adcq %r20, %r8
+; EGPR-NDD-NEXT: addq %r29, %rdi
+; EGPR-NDD-NEXT: adcq %r28, %r8
; EGPR-NDD-NEXT: movzbl %sil, %edx
; EGPR-NDD-NEXT: adcq %rdx, %rax
; EGPR-NDD-NEXT: adcq $0, %rcx
@@ -1354,8 +1354,8 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: movq %r27, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: adcq %r13, %r25, %r19
; EGPR-NDD-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NDD-NEXT: adcq %rbp, %r31
-; EGPR-NDD-NEXT: movq %r31, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; EGPR-NDD-NEXT: adcq %rbp, %r21
+; EGPR-NDD-NEXT: movq %r21, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: adcq %rbx, %r10
; EGPR-NDD-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: adcq $0, %rdi
@@ -1363,18 +1363,18 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: adcq $0, %rax
; EGPR-NDD-NEXT: adcq $0, %rcx
; EGPR-NDD-NEXT: addq %rdi, {{[-0-9]+}}(%r{{[sb]}}p), %r19 # 8-byte Folded Reload
-; EGPR-NDD-NEXT: adcq %r8, %r28
-; EGPR-NDD-NEXT: adcq %rax, %r29
+; EGPR-NDD-NEXT: adcq %r8, %r30
+; EGPR-NDD-NEXT: adcq %rax, %r31
; EGPR-NDD-NEXT: adcq %rcx, {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
; EGPR-NDD-NEXT: setb %r8b
; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
; EGPR-NDD-NEXT: movq %r13, %rax
-; EGPR-NDD-NEXT: mulq %r30
+; EGPR-NDD-NEXT: mulq %r20
; EGPR-NDD-NEXT: movq %rdx, %r27
-; EGPR-NDD-NEXT: movq %rax, %r20
+; EGPR-NDD-NEXT: movq %rax, %r28
; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
; EGPR-NDD-NEXT: movq %r10, %rax
-; EGPR-NDD-NEXT: mulq %r30
+; EGPR-NDD-NEXT: mulq %r20
; EGPR-NDD-NEXT: addq %rax, %r27
; EGPR-NDD-NEXT: adcq $0, %rdx, %rsi
; EGPR-NDD-NEXT: movq %r13, %rax
@@ -1390,64 +1390,64 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: adcq %rax, %rdx, %r9
; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r25 # 8-byte Reload
; EGPR-NDD-NEXT: movq %r25, %rax
-; EGPR-NDD-NEXT: mulq %r30
-; EGPR-NDD-NEXT: movq %rdx, %r21
+; EGPR-NDD-NEXT: mulq %r20
+; EGPR-NDD-NEXT: movq %rdx, %r29
; EGPR-NDD-NEXT: movq %rax, %r27
; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
; EGPR-NDD-NEXT: movq %r12, %rax
-; EGPR-NDD-NEXT: mulq %r30
-; EGPR-NDD-NEXT: addq %rax, %r21
+; EGPR-NDD-NEXT: mulq %r20
+; EGPR-NDD-NEXT: addq %rax, %r29
; EGPR-NDD-NEXT: adcq $0, %rdx, %r10
; EGPR-NDD-NEXT: movq %r25, %rax
; EGPR-NDD-NEXT: mulq %r18
-; EGPR-NDD-NEXT: addq %r21, %rax, %rbx
+; EGPR-NDD-NEXT: addq %r29, %rax, %rbx
; EGPR-NDD-NEXT: adcq %rdx, %r10
-; EGPR-NDD-NEXT: setb %r31b
+; EGPR-NDD-NEXT: setb %r21b
; EGPR-NDD-NEXT: movq %r12, %rax
; EGPR-NDD-NEXT: mulq %r18
; EGPR-NDD-NEXT: addq %r10, %rax
-; EGPR-NDD-NEXT: movzbl %r31b, %r10d
+; EGPR-NDD-NEXT: movzbl %r21b, %r10d
; EGPR-NDD-NEXT: adcq %r10, %rdx
-; EGPR-NDD-NEXT: addq %rax, %r20, %r10
+; EGPR-NDD-NEXT: addq %rax, %r28, %r10
; EGPR-NDD-NEXT: adcq %rdx, %rdi
; EGPR-NDD-NEXT: adcq $0, %rsi
; EGPR-NDD-NEXT: adcq $0, %r9
; EGPR-NDD-NEXT: movq %r25, %rax
; EGPR-NDD-NEXT: mulq %r11
-; EGPR-NDD-NEXT: movq %rdx, %r20
-; EGPR-NDD-NEXT: movq %rax, %r21
+; EGPR-NDD-NEXT: movq %rdx, %r28
+; EGPR-NDD-NEXT: movq %rax, %r29
; EGPR-NDD-NEXT: movq %r12, %rax
; EGPR-NDD-NEXT: mulq %r11
-; EGPR-NDD-NEXT: addq %rax, %r20
-; EGPR-NDD-NEXT: adcq $0, %rdx, %r31
+; EGPR-NDD-NEXT: addq %rax, %r28
+; EGPR-NDD-NEXT: adcq $0, %rdx, %r21
; EGPR-NDD-NEXT: movq %r25, %rax
; EGPR-NDD-NEXT: mulq %r17
-; EGPR-NDD-NEXT: addq %rax, %r20
-; EGPR-NDD-NEXT: adcq %rdx, %r31
+; EGPR-NDD-NEXT: addq %rax, %r28
+; EGPR-NDD-NEXT: adcq %rdx, %r21
; EGPR-NDD-NEXT: setb %bpl
; EGPR-NDD-NEXT: movq %r12, %rax
; EGPR-NDD-NEXT: mulq %r17
-; EGPR-NDD-NEXT: addq %r31, %rax
-; EGPR-NDD-NEXT: movzbl %bpl, %r31d
-; EGPR-NDD-NEXT: adcq %r31, %rdx
-; EGPR-NDD-NEXT: addq %r21, %r10
-; EGPR-NDD-NEXT: adcq %r20, %rdi
+; EGPR-NDD-NEXT: addq %r21, %rax
+; EGPR-NDD-NEXT: movzbl %bpl, %r21d
+; EGPR-NDD-NEXT: adcq %r21, %rdx
+; EGPR-NDD-NEXT: addq %r29, %r10
+; EGPR-NDD-NEXT: adcq %r28, %rdi
; EGPR-NDD-NEXT: adcq $0, %rax
; EGPR-NDD-NEXT: adcq $0, %rdx
; EGPR-NDD-NEXT: addq %rax, %rsi
; EGPR-NDD-NEXT: adcq %rdx, %r9
-; EGPR-NDD-NEXT: setb %r31b
+; EGPR-NDD-NEXT: setb %r21b
; EGPR-NDD-NEXT: movq %r13, %rax
; EGPR-NDD-NEXT: mulq %r11
-; EGPR-NDD-NEXT: movq %rdx, %r20
-; EGPR-NDD-NEXT: movq %rax, %r21
+; EGPR-NDD-NEXT: movq %rdx, %r28
+; EGPR-NDD-NEXT: movq %rax, %r29
; EGPR-NDD-NEXT: movq %r16, %rax
; EGPR-NDD-NEXT: mulq %r11
-; EGPR-NDD-NEXT: addq %rax, %r20
+; EGPR-NDD-NEXT: addq %rax, %r28
; EGPR-NDD-NEXT: adcq $0, %rdx, %r12
; EGPR-NDD-NEXT: movq %r13, %rax
; EGPR-NDD-NEXT: mulq %r17
-; EGPR-NDD-NEXT: addq %rax, %r20
+; EGPR-NDD-NEXT: addq %rax, %r28
; EGPR-NDD-NEXT: adcq %rdx, %r12
; EGPR-NDD-NEXT: setb %bpl
; EGPR-NDD-NEXT: movq %r16, %rax
@@ -1455,16 +1455,16 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: addq %r12, %rax
; EGPR-NDD-NEXT: movzbl %bpl, %r12d
; EGPR-NDD-NEXT: adcq %r12, %rdx
-; EGPR-NDD-NEXT: addq %r21, %rsi
-; EGPR-NDD-NEXT: adcq %r20, %r9
-; EGPR-NDD-NEXT: movzbl %r31b, %r31d
-; EGPR-NDD-NEXT: adcq %r31, %rax
+; EGPR-NDD-NEXT: addq %r29, %rsi
+; EGPR-NDD-NEXT: adcq %r28, %r9
+; EGPR-NDD-NEXT: movzbl %r21b, %r21d
+; EGPR-NDD-NEXT: adcq %r21, %rax
; EGPR-NDD-NEXT: adcq $0, %rdx
; EGPR-NDD-NEXT: addq %r27, %r19
; EGPR-NDD-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NDD-NEXT: adcq %rbx, %r28
-; EGPR-NDD-NEXT: movq %r28, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NDD-NEXT: adcq %r29, %r10
+; EGPR-NDD-NEXT: adcq %rbx, %r30
+; EGPR-NDD-NEXT: movq %r30, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; EGPR-NDD-NEXT: adcq %r31, %r10
; EGPR-NDD-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: adcq %rdi, %rcx
; EGPR-NDD-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
@@ -1477,62 +1477,62 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: adcq $0, %rdx
; EGPR-NDD-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NDD-NEXT: movq 64(%r22), %r20
+; EGPR-NDD-NEXT: movq 64(%r22), %r28
; EGPR-NDD-NEXT: movq %r26, %rax
-; EGPR-NDD-NEXT: mulq %r20
+; EGPR-NDD-NEXT: mulq %r28
; EGPR-NDD-NEXT: movq %rdx, %r27
-; EGPR-NDD-NEXT: movq %rax, %r28
+; EGPR-NDD-NEXT: movq %rax, %r30
; EGPR-NDD-NEXT: movq %r14, %rax
-; EGPR-NDD-NEXT: mulq %r20
+; EGPR-NDD-NEXT: mulq %r28
; EGPR-NDD-NEXT: addq %rax, %r27
; EGPR-NDD-NEXT: adcq $0, %rdx, %rcx
-; EGPR-NDD-NEXT: movq 72(%r22), %r21
+; EGPR-NDD-NEXT: movq 72(%r22), %r29
; EGPR-NDD-NEXT: movq %r26, %rax
-; EGPR-NDD-NEXT: mulq %r21
+; EGPR-NDD-NEXT: mulq %r29
; EGPR-NDD-NEXT: addq %rax, %r27
; EGPR-NDD-NEXT: adcq %rdx, %rcx
; EGPR-NDD-NEXT: setb %sil
; EGPR-NDD-NEXT: movq %r14, %rax
-; EGPR-NDD-NEXT: mulq %r21
+; EGPR-NDD-NEXT: mulq %r29
; EGPR-NDD-NEXT: addq %rax, %rcx
; EGPR-NDD-NEXT: movzbl %sil, %eax
; EGPR-NDD-NEXT: adcq %rax, %rdx, %rsi
; EGPR-NDD-NEXT: movq %r24, %rax
-; EGPR-NDD-NEXT: mulq %r20
-; EGPR-NDD-NEXT: movq %rdx, %r29
+; EGPR-NDD-NEXT: mulq %r28
+; EGPR-NDD-NEXT: movq %rdx, %r31
; EGPR-NDD-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: movq %r23, %rax
-; EGPR-NDD-NEXT: mulq %r20
-; EGPR-NDD-NEXT: addq %rax, %r29
+; EGPR-NDD-NEXT: mulq %r28
+; EGPR-NDD-NEXT: addq %rax, %r31
; EGPR-NDD-NEXT: adcq $0, %rdx, %rdi
; EGPR-NDD-NEXT: movq %r24, %rax
-; EGPR-NDD-NEXT: mulq %r21
-; EGPR-NDD-NEXT: addq %r29, %rax
+; EGPR-NDD-NEXT: mulq %r29
+; EGPR-NDD-NEXT: addq %r31, %rax
; EGPR-NDD-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; EGPR-NDD-NEXT: adcq %rdx, %rdi
; EGPR-NDD-NEXT: setb %r8b
; EGPR-NDD-NEXT: movq %r23, %rax
-; EGPR-NDD-NEXT: mulq %r21
+; EGPR-NDD-NEXT: mulq %r29
; EGPR-NDD-NEXT: addq %rdi, %rax
; EGPR-NDD-NEXT: movzbl %r8b, %edi
; EGPR-NDD-NEXT: adcq %rdi, %rdx
-; EGPR-NDD-NEXT: addq %rax, %r28, %rdi
+; EGPR-NDD-NEXT: addq %rax, %r30, %rdi
; EGPR-NDD-NEXT: adcq %rdx, %r27
; EGPR-NDD-NEXT: adcq $0, %rcx
; EGPR-NDD-NEXT: adcq $0, %rsi
; EGPR-NDD-NEXT: movq 80(%r22), %r8
; EGPR-NDD-NEXT: movq %r24, %rax
; EGPR-NDD-NEXT: mulq %r8
-; EGPR-NDD-NEXT: movq %rdx, %r28
-; EGPR-NDD-NEXT: movq %rax, %r29
+; EGPR-NDD-NEXT: movq %rdx, %r30
+; EGPR-NDD-NEXT: movq %rax, %r31
; EGPR-NDD-NEXT: movq %r23, %rax
; EGPR-NDD-NEXT: mulq %r8
-; EGPR-NDD-NEXT: addq %rax, %r28
+; EGPR-NDD-NEXT: addq %rax, %r30
; EGPR-NDD-NEXT: adcq $0, %rdx, %r9
; EGPR-NDD-NEXT: movq 88(%r22), %rbx
; EGPR-NDD-NEXT: movq %r24, %rax
; EGPR-NDD-NEXT: mulq %rbx
-; EGPR-NDD-NEXT: addq %rax, %r28
+; EGPR-NDD-NEXT: addq %rax, %r30
; EGPR-NDD-NEXT: adcq %rdx, %r9
; EGPR-NDD-NEXT: setb %r10b
; EGPR-NDD-NEXT: movq %r23, %rax
@@ -1540,9 +1540,9 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: addq %r9, %rax
; EGPR-NDD-NEXT: movzbl %r10b, %r9d
; EGPR-NDD-NEXT: adcq %r9, %rdx
-; EGPR-NDD-NEXT: addq %r29, %rdi
+; EGPR-NDD-NEXT: addq %r31, %rdi
; EGPR-NDD-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; EGPR-NDD-NEXT: adcq %r27, %r28, %rbp
+; EGPR-NDD-NEXT: adcq %r27, %r30, %rbp
; EGPR-NDD-NEXT: adcq $0, %rax
; EGPR-NDD-NEXT: adcq $0, %rdx
; EGPR-NDD-NEXT: addq %rax, %rcx
@@ -1550,15 +1550,15 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: setb %dil
; EGPR-NDD-NEXT: movq %r26, %rax
; EGPR-NDD-NEXT: mulq %r8
-; EGPR-NDD-NEXT: movq %rdx, %r28
-; EGPR-NDD-NEXT: movq %rax, %r29
+; EGPR-NDD-NEXT: movq %rdx, %r30
+; EGPR-NDD-NEXT: movq %rax, %r31
; EGPR-NDD-NEXT: movq %r14, %rax
; EGPR-NDD-NEXT: mulq %r8
-; EGPR-NDD-NEXT: addq %rax, %r28
+; EGPR-NDD-NEXT: addq %rax, %r30
; EGPR-NDD-NEXT: adcq $0, %rdx, %r9
; EGPR-NDD-NEXT: movq %r26, %rax
; EGPR-NDD-NEXT: mulq %rbx
-; EGPR-NDD-NEXT: addq %rax, %r28
+; EGPR-NDD-NEXT: addq %rax, %r30
; EGPR-NDD-NEXT: adcq %rdx, %r9
; EGPR-NDD-NEXT: setb %r10b
; EGPR-NDD-NEXT: movq %r14, %rax
@@ -1566,40 +1566,40 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: addq %r9, %rax
; EGPR-NDD-NEXT: movzbl %r10b, %r9d
; EGPR-NDD-NEXT: adcq %r9, %rdx
-; EGPR-NDD-NEXT: addq %rcx, %r29, %r27
-; EGPR-NDD-NEXT: adcq %rsi, %r28, %r12
+; EGPR-NDD-NEXT: addq %rcx, %r31, %r27
+; EGPR-NDD-NEXT: adcq %rsi, %r30, %r12
; EGPR-NDD-NEXT: movzbl %dil, %r19d
; EGPR-NDD-NEXT: adcq %rax, %r19
-; EGPR-NDD-NEXT: adcq $0, %rdx, %r29
-; EGPR-NDD-NEXT: imulq %r30, %rbx
-; EGPR-NDD-NEXT: movq %r30, %rax
+; EGPR-NDD-NEXT: adcq $0, %rdx, %r31
+; EGPR-NDD-NEXT: imulq %r20, %rbx
+; EGPR-NDD-NEXT: movq %r20, %rax
; EGPR-NDD-NEXT: mulq %r8
-; EGPR-NDD-NEXT: movq %rax, %r28
+; EGPR-NDD-NEXT: movq %rax, %r30
; EGPR-NDD-NEXT: addq %rbx, %rdx
; EGPR-NDD-NEXT: imulq %r18, %r8
; EGPR-NDD-NEXT: addq %rdx, %r8
-; EGPR-NDD-NEXT: imulq %r21, %r11, %rcx
+; EGPR-NDD-NEXT: imulq %r29, %r11, %rcx
; EGPR-NDD-NEXT: movq %r11, %rax
-; EGPR-NDD-NEXT: mulq %r20
+; EGPR-NDD-NEXT: mulq %r28
; EGPR-NDD-NEXT: addq %rdx, %rcx
-; EGPR-NDD-NEXT: imulq %r20, %r17, %r16
+; EGPR-NDD-NEXT: imulq %r28, %r17, %r16
; EGPR-NDD-NEXT: addq %r16, %rcx
-; EGPR-NDD-NEXT: addq %r28, %rax, %rsi
+; EGPR-NDD-NEXT: addq %r30, %rax, %rsi
; EGPR-NDD-NEXT: adcq %rcx, %r8
-; EGPR-NDD-NEXT: movq %r20, %rax
-; EGPR-NDD-NEXT: mulq %r30
-; EGPR-NDD-NEXT: movq %rdx, %r28
-; EGPR-NDD-NEXT: movq %rax, %r31
-; EGPR-NDD-NEXT: movq %r21, %rax
-; EGPR-NDD-NEXT: mulq %r30
-; EGPR-NDD-NEXT: addq %r28, %rax, %rcx
+; EGPR-NDD-NEXT: movq %r28, %rax
+; EGPR-NDD-NEXT: mulq %r20
+; EGPR-NDD-NEXT: movq %rdx, %r30
+; EGPR-NDD-NEXT: movq %rax, %r21
+; EGPR-NDD-NEXT: movq %r29, %rax
+; EGPR-NDD-NEXT: mulq %r20
+; EGPR-NDD-NEXT: addq %r30, %rax, %rcx
; EGPR-NDD-NEXT: adcq $0, %rdx, %rdi
-; EGPR-NDD-NEXT: movq %r20, %rax
+; EGPR-NDD-NEXT: movq %r28, %rax
; EGPR-NDD-NEXT: mulq %r18
; EGPR-NDD-NEXT: addq %rax, %rcx
; EGPR-NDD-NEXT: adcq %rdx, %rdi
; EGPR-NDD-NEXT: setb %r9b
-; EGPR-NDD-NEXT: movq %r21, %rax
+; EGPR-NDD-NEXT: movq %r29, %rax
; EGPR-NDD-NEXT: mulq %r18
; EGPR-NDD-NEXT: addq %rdi, %rax
; EGPR-NDD-NEXT: movzbl %r9b, %edi
@@ -1609,22 +1609,22 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: movq 112(%r22), %rdi
; EGPR-NDD-NEXT: movq %r24, %rax
; EGPR-NDD-NEXT: mulq %rdi
-; EGPR-NDD-NEXT: movq %rax, %r30
+; EGPR-NDD-NEXT: movq %rax, %r20
; EGPR-NDD-NEXT: imulq %r23, %rdi
; EGPR-NDD-NEXT: addq %rdi, %rdx
; EGPR-NDD-NEXT: imulq 120(%r22), %r24, %rax
; EGPR-NDD-NEXT: leaq (%rdx,%rax), %r9
-; EGPR-NDD-NEXT: movq 96(%r22), %r20
+; EGPR-NDD-NEXT: movq 96(%r22), %r28
; EGPR-NDD-NEXT: movq 104(%r22), %rdi
; EGPR-NDD-NEXT: imulq %rdi, %r26, %r10
; EGPR-NDD-NEXT: movq %r26, %rax
-; EGPR-NDD-NEXT: mulq %r20
+; EGPR-NDD-NEXT: mulq %r28
; EGPR-NDD-NEXT: addq %r10, %rdx
-; EGPR-NDD-NEXT: imulq %r20, %r14, %r25
+; EGPR-NDD-NEXT: imulq %r28, %r14, %r25
; EGPR-NDD-NEXT: addq %r25, %rdx
-; EGPR-NDD-NEXT: addq %rax, %r30
+; EGPR-NDD-NEXT: addq %rax, %r20
; EGPR-NDD-NEXT: adcq %rdx, %r9
-; EGPR-NDD-NEXT: movq %r20, %rax
+; EGPR-NDD-NEXT: movq %r28, %rax
; EGPR-NDD-NEXT: mulq %r24
; EGPR-NDD-NEXT: movq %rdx, %r25
; EGPR-NDD-NEXT: movq %rax, %r26
@@ -1632,7 +1632,7 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: mulq %r24
; EGPR-NDD-NEXT: addq %rax, %r25
; EGPR-NDD-NEXT: adcq $0, %rdx, %r10
-; EGPR-NDD-NEXT: movq %r20, %rax
+; EGPR-NDD-NEXT: movq %r28, %rax
; EGPR-NDD-NEXT: mulq %r23
; EGPR-NDD-NEXT: addq %rax, %r25
; EGPR-NDD-NEXT: adcq %rdx, %r10
@@ -1642,21 +1642,21 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: addq %r10, %rax
; EGPR-NDD-NEXT: movzbl %r11b, %edi
; EGPR-NDD-NEXT: adcq %rdi, %rdx
-; EGPR-NDD-NEXT: addq %r30, %rax
+; EGPR-NDD-NEXT: addq %r20, %rax
; EGPR-NDD-NEXT: adcq %r9, %rdx
-; EGPR-NDD-NEXT: addq %r31, %r26
+; EGPR-NDD-NEXT: addq %r21, %r26
; EGPR-NDD-NEXT: adcq %r25, %rcx
; EGPR-NDD-NEXT: adcq %rsi, %rax
; EGPR-NDD-NEXT: adcq %r8, %rdx
; EGPR-NDD-NEXT: addq %r26, %r27, %rbx
; EGPR-NDD-NEXT: adcq %rcx, %r12
; EGPR-NDD-NEXT: adcq %rax, %r19, %r13
-; EGPR-NDD-NEXT: adcq %rdx, %r29, %r28
+; EGPR-NDD-NEXT: adcq %rdx, %r31, %r30
; EGPR-NDD-NEXT: movq 80(%r15), %r24
; EGPR-NDD-NEXT: movq %r24, %rax
; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r16 # 8-byte Reload
; EGPR-NDD-NEXT: mulq %r16
-; EGPR-NDD-NEXT: movq %rax, %r30
+; EGPR-NDD-NEXT: movq %rax, %r20
; EGPR-NDD-NEXT: movq %rdx, %rdi
; EGPR-NDD-NEXT: movq 88(%r15), %r22
; EGPR-NDD-NEXT: movq %r22, %rax
@@ -1677,16 +1677,16 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: movq 64(%r15), %r26
; EGPR-NDD-NEXT: movq %r26, %rax
; EGPR-NDD-NEXT: mulq %r16
-; EGPR-NDD-NEXT: movq %rax, %r21
-; EGPR-NDD-NEXT: movq %rdx, %r31
+; EGPR-NDD-NEXT: movq %rax, %r29
+; EGPR-NDD-NEXT: movq %rdx, %r21
; EGPR-NDD-NEXT: movq 72(%r15), %r25
; EGPR-NDD-NEXT: movq %r25, %rax
; EGPR-NDD-NEXT: mulq %r16
-; EGPR-NDD-NEXT: addq %rax, %r31
+; EGPR-NDD-NEXT: addq %rax, %r21
; EGPR-NDD-NEXT: adcq $0, %rdx, %r8
; EGPR-NDD-NEXT: movq %r26, %rax
; EGPR-NDD-NEXT: mulq %r23
-; EGPR-NDD-NEXT: addq %r31, %rax, %r29
+; EGPR-NDD-NEXT: addq %r21, %rax, %r31
; EGPR-NDD-NEXT: adcq %rdx, %r8
; EGPR-NDD-NEXT: setb %r9b
; EGPR-NDD-NEXT: movq %r25, %rax
@@ -1694,23 +1694,23 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: addq %r8, %rax
; EGPR-NDD-NEXT: movzbl %r9b, %r8d
; EGPR-NDD-NEXT: adcq %r8, %rdx
-; EGPR-NDD-NEXT: addq %rax, %r30, %r20
+; EGPR-NDD-NEXT: addq %rax, %r20, %r28
; EGPR-NDD-NEXT: adcq %rdx, %rdi
; EGPR-NDD-NEXT: adcq $0, %rcx
; EGPR-NDD-NEXT: adcq $0, %rsi
; EGPR-NDD-NEXT: movq %r26, %rax
; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
; EGPR-NDD-NEXT: mulq %r10
-; EGPR-NDD-NEXT: movq %rdx, %r30
-; EGPR-NDD-NEXT: movq %rax, %r31
+; EGPR-NDD-NEXT: movq %rdx, %r20
+; EGPR-NDD-NEXT: movq %rax, %r21
; EGPR-NDD-NEXT: movq %r25, %rax
; EGPR-NDD-NEXT: mulq %r10
-; EGPR-NDD-NEXT: addq %rax, %r30
+; EGPR-NDD-NEXT: addq %rax, %r20
; EGPR-NDD-NEXT: adcq $0, %rdx, %r8
; EGPR-NDD-NEXT: movq %r26, %rax
; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
; EGPR-NDD-NEXT: mulq %r11
-; EGPR-NDD-NEXT: addq %r30, %rax, %r27
+; EGPR-NDD-NEXT: addq %r20, %rax, %r27
; EGPR-NDD-NEXT: adcq %rdx, %r8
; EGPR-NDD-NEXT: setb %r9b
; EGPR-NDD-NEXT: movq %r25, %rax
@@ -1718,7 +1718,7 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: addq %r8, %rax
; EGPR-NDD-NEXT: movzbl %r9b, %r8d
; EGPR-NDD-NEXT: adcq %r8, %rdx
-; EGPR-NDD-NEXT: addq %r31, %r20
+; EGPR-NDD-NEXT: addq %r21, %r28
; EGPR-NDD-NEXT: adcq %rdi, %r27
; EGPR-NDD-NEXT: adcq $0, %rax
; EGPR-NDD-NEXT: adcq $0, %rdx
@@ -1727,15 +1727,15 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: setb %dil
; EGPR-NDD-NEXT: movq %r24, %rax
; EGPR-NDD-NEXT: mulq %r10
-; EGPR-NDD-NEXT: movq %rdx, %r30
-; EGPR-NDD-NEXT: movq %rax, %r31
+; EGPR-NDD-NEXT: movq %rdx, %r20
+; EGPR-NDD-NEXT: movq %rax, %r21
; EGPR-NDD-NEXT: movq %r22, %rax
; EGPR-NDD-NEXT: mulq %r10
-; EGPR-NDD-NEXT: addq %rax, %r30
+; EGPR-NDD-NEXT: addq %rax, %r20
; EGPR-NDD-NEXT: adcq $0, %rdx, %r8
; EGPR-NDD-NEXT: movq %r24, %rax
; EGPR-NDD-NEXT: mulq %r11
-; EGPR-NDD-NEXT: addq %r30, %rax, %r19
+; EGPR-NDD-NEXT: addq %r20, %rax, %r19
; EGPR-NDD-NEXT: adcq %rdx, %r8
; EGPR-NDD-NEXT: setb %r9b
; EGPR-NDD-NEXT: movq %r22, %rax
@@ -1743,14 +1743,14 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: addq %r8, %rax
; EGPR-NDD-NEXT: movzbl %r9b, %r8d
; EGPR-NDD-NEXT: adcq %r8, %rdx
-; EGPR-NDD-NEXT: addq %rcx, %r31
+; EGPR-NDD-NEXT: addq %rcx, %r21
; EGPR-NDD-NEXT: adcq %rsi, %r19
; EGPR-NDD-NEXT: movzbl %dil, %ecx
; EGPR-NDD-NEXT: adcq %rax, %rcx
; EGPR-NDD-NEXT: adcq $0, %rdx, %rdi
-; EGPR-NDD-NEXT: movq 96(%r15), %r30
-; EGPR-NDD-NEXT: imulq %r11, %r30, %rsi
-; EGPR-NDD-NEXT: movq %r30, %rax
+; EGPR-NDD-NEXT: movq 96(%r15), %r20
+; EGPR-NDD-NEXT: imulq %r11, %r20, %rsi
+; EGPR-NDD-NEXT: movq %r20, %rax
; EGPR-NDD-NEXT: mulq %r10
; EGPR-NDD-NEXT: movq %rax, %r18
; EGPR-NDD-NEXT: addq %rsi, %rdx
@@ -1767,11 +1767,11 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: adcq %rsi, %rdx, %r9
; EGPR-NDD-NEXT: movq %r16, %rax
; EGPR-NDD-NEXT: movq %r16, %r18
-; EGPR-NDD-NEXT: mulq %r30
+; EGPR-NDD-NEXT: mulq %r20
; EGPR-NDD-NEXT: movq %rdx, %r17
; EGPR-NDD-NEXT: movq %rax, %rsi
; EGPR-NDD-NEXT: movq %r23, %rax
-; EGPR-NDD-NEXT: mulq %r30
+; EGPR-NDD-NEXT: mulq %r20
; EGPR-NDD-NEXT: addq %r17, %rax, %r11
; EGPR-NDD-NEXT: adcq $0, %rdx, %r16
; EGPR-NDD-NEXT: movq %r18, %rax
@@ -1797,14 +1797,14 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r23 # 8-byte Reload
; EGPR-NDD-NEXT: imulq %r23, %r24, %r16
; EGPR-NDD-NEXT: movq %r24, %rax
-; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r30 # 8-byte Reload
-; EGPR-NDD-NEXT: mulq %r30
+; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r20 # 8-byte Reload
+; EGPR-NDD-NEXT: mulq %r20
; EGPR-NDD-NEXT: addq %r16, %rdx
-; EGPR-NDD-NEXT: imulq %r30, %r22
+; EGPR-NDD-NEXT: imulq %r20, %r22
; EGPR-NDD-NEXT: addq %r22, %rdx
; EGPR-NDD-NEXT: addq %r9, %rax, %r16
; EGPR-NDD-NEXT: adcq %r8, %rdx, %r18
-; EGPR-NDD-NEXT: movq %r30, %rax
+; EGPR-NDD-NEXT: movq %r20, %rax
; EGPR-NDD-NEXT: mulq %r26
; EGPR-NDD-NEXT: movq %rdx, %r8
; EGPR-NDD-NEXT: movq %rax, %r9
@@ -1813,7 +1813,7 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: mulq %r26
; EGPR-NDD-NEXT: addq %rax, %r8
; EGPR-NDD-NEXT: adcq $0, %rdx, %r22
-; EGPR-NDD-NEXT: movq %r30, %rax
+; EGPR-NDD-NEXT: movq %r20, %rax
; EGPR-NDD-NEXT: mulq %r25
; EGPR-NDD-NEXT: addq %rax, %r8
; EGPR-NDD-NEXT: adcq %rdx, %r22
@@ -1829,21 +1829,21 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: adcq %r11, %r8
; EGPR-NDD-NEXT: adcq %r10, %rax
; EGPR-NDD-NEXT: adcq %r17, %rdx
-; EGPR-NDD-NEXT: addq %r31, %rsi
+; EGPR-NDD-NEXT: addq %r21, %rsi
; EGPR-NDD-NEXT: adcq %r19, %r8
; EGPR-NDD-NEXT: adcq %rcx, %rax
; EGPR-NDD-NEXT: adcq %rdi, %rdx
-; EGPR-NDD-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r21 # 8-byte Folded Reload
-; EGPR-NDD-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r29 # 8-byte Folded Reload
-; EGPR-NDD-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r20 # 8-byte Folded Reload
+; EGPR-NDD-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r29 # 8-byte Folded Reload
+; EGPR-NDD-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r31 # 8-byte Folded Reload
+; EGPR-NDD-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r28 # 8-byte Folded Reload
; EGPR-NDD-NEXT: adcq %rbp, %r27
; EGPR-NDD-NEXT: adcq %rbx, %rsi
; EGPR-NDD-NEXT: adcq %r12, %r8
; EGPR-NDD-NEXT: adcq %r13, %rax
-; EGPR-NDD-NEXT: adcq %r28, %rdx
-; EGPR-NDD-NEXT: addq %r21, {{[-0-9]+}}(%r{{[sb]}}p), %r21 # 8-byte Folded Reload
-; EGPR-NDD-NEXT: adcq %r29, {{[-0-9]+}}(%r{{[sb]}}p), %r29 # 8-byte Folded Reload
-; EGPR-NDD-NEXT: adcq %r20, {{[-0-9]+}}(%r{{[sb]}}p), %r20 # 8-byte Folded Reload
+; EGPR-NDD-NEXT: adcq %r30, %rdx
+; EGPR-NDD-NEXT: addq %r29, {{[-0-9]+}}(%r{{[sb]}}p), %r29 # 8-byte Folded Reload
+; EGPR-NDD-NEXT: adcq %r31, {{[-0-9]+}}(%r{{[sb]}}p), %r31 # 8-byte Folded Reload
+; EGPR-NDD-NEXT: adcq %r28, {{[-0-9]+}}(%r{{[sb]}}p), %r28 # 8-byte Folded Reload
; EGPR-NDD-NEXT: adcq %r27, {{[-0-9]+}}(%r{{[sb]}}p), %r27 # 8-byte Folded Reload
; EGPR-NDD-NEXT: adcq %rsi, (%rsp), %rsi # 8-byte Folded Reload
; EGPR-NDD-NEXT: adcq %r8, {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
@@ -1866,9 +1866,9 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind {
; EGPR-NDD-NEXT: movq %rdi, 48(%rcx)
; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; EGPR-NDD-NEXT: movq %rdi, 56(%rcx)
-; EGPR-NDD-NEXT: movq %r21, 64(%rcx)
-; EGPR-NDD-NEXT: movq %r29, 72(%rcx)
-; EGPR-NDD-NEXT: movq %r20, 80(%rcx)
+; EGPR-NDD-NEXT: movq %r29, 64(%rcx)
+; EGPR-NDD-NEXT: movq %r31, 72(%rcx)
+; EGPR-NDD-NEXT: movq %r28, 80(%rcx)
; EGPR-NDD-NEXT: movq %r27, 88(%rcx)
; EGPR-NDD-NEXT: movq %rsi, 96(%rcx)
; EGPR-NDD-NEXT: movq %r8, 104(%rcx)
diff --git a/llvm/test/CodeGen/X86/apx/setjmp-win64-abi.ll b/llvm/test/CodeGen/X86/apx/setjmp-win64-abi.ll
new file mode 100644
index 0000000000000..f1c8bddc26af7
--- /dev/null
+++ b/llvm/test/CodeGen/X86/apx/setjmp-win64-abi.ll
@@ -0,0 +1,237 @@
+; RUN: llc < %s -mtriple=x86_64-windows-msvc -mattr=+egpr -regalloc=greedy | FileCheck %s --check-prefixes=CHECK,ALLOC
+; RUN: llc < %s -mtriple=x86_64-windows-msvc -mattr=+egpr -regalloc=basic | FileCheck %s --check-prefixes=CHECK,ALLOC
+; RUN: llc < %s -mtriple=x86_64-windows-msvc -mattr=+egpr -regalloc=pbqp | FileCheck %s --check-prefixes=CHECK,ALLOC
+; RUN: llc < %s -mtriple=x86_64-uefi -mattr=+egpr -regalloc=greedy | FileCheck %s --check-prefixes=CHECK,ALLOC
+; RUN: llc < %s -mtriple=x86_64-uefi -mattr=+egpr -regalloc=basic | FileCheck %s --check-prefixes=CHECK,ALLOC
+; RUN: llc < %s -mtriple=x86_64-uefi -mattr=+egpr -regalloc=pbqp | FileCheck %s --check-prefixes=CHECK,ALLOC
+
+; Test that R30 and R31 and all their sub-registers (r30d, r31d) are not
+; allocated in functions that call setjmp (returns_twice) with Win64 APX
+; ABI support.
+;
+; Generated from the following C++ code:
+;
+; #include <cstdint>
+; #include <csetjmp>
+;
+; extern "C" jmp_buf jmpbuf;
+; extern "C" void external_call();
+;
+; template<typename T> T get_val();
+; template<typename T> void use_val(T val);
+;
+; template<typename T>
+; T test_setjmp() {
+; T v1 = get_val<T>();
+; T v2 = get_val<T>();
+; T v3 = get_val<T>();
+; T v4 = get_val<T>();
+; T v5 = get_val<T>();
+; T v6 = get_val<T>();
+; T v7 = get_val<T>();
+; T v8 = get_val<T>();
+; T v9 = get_val<T>();
+; T v10 = get_val<T>();
+; T v11 = get_val<T>();
+; T v12 = get_val<T>();
+;
+; if (_setjmp(jmpbuf) == 0) {
+; external_call();
+; use_val(v1);
+; use_val(v2);
+; use_val(v3);
+; use_val(v4);
+; use_val(v5);
+; use_val(v6);
+; use_val(v7);
+; use_val(v8);
+; use_val(v9);
+; use_val(v10);
+; use_val(v11);
+; use_val(v12);
+; }
+; return v1;
+; }
+;
+; template uint32_t test_setjmp<uint32_t>();
+; template uint64_t test_setjmp<uint64_t>();
+
+declare i32 @_setjmp(ptr) returns_twice
+declare i32 @_fake_setjmp(ptr)
+declare void @external_call()
+declare i64 @get_val_i64()
+declare void @use_val_i64(i64)
+declare i32 @get_val_i32()
+declare void @use_val_i32(i32)
+
+ at buf = external global [256 x i8], align 16
+
+; Without returns_twice, r30/r31 SHOULD be used.
+define i64 @test_no_setjmp_i64() nounwind {
+; CHECK-LABEL: test_no_setjmp_i64:
+; ALLOC: %r30
+; ALLOC: %r31
+; CHECK: retq
+entry:
+ %v1 = call i64 @get_val_i64()
+ %v2 = call i64 @get_val_i64()
+ %v3 = call i64 @get_val_i64()
+ %v4 = call i64 @get_val_i64()
+ %v5 = call i64 @get_val_i64()
+ %v6 = call i64 @get_val_i64()
+ %v7 = call i64 @get_val_i64()
+ %v8 = call i64 @get_val_i64()
+ %v9 = call i64 @get_val_i64()
+ %v10 = call i64 @get_val_i64()
+ %v11 = call i64 @get_val_i64()
+ %v12 = call i64 @get_val_i64()
+ %r = call i32 @_fake_setjmp(ptr @buf)
+ %cmp = icmp eq i32 %r, 0
+ br i1 %cmp, label %normal, label %longjmp_return
+
+normal:
+ call void @external_call()
+ call void @use_val_i64(i64 %v1)
+ call void @use_val_i64(i64 %v2)
+ call void @use_val_i64(i64 %v3)
+ call void @use_val_i64(i64 %v4)
+ call void @use_val_i64(i64 %v5)
+ call void @use_val_i64(i64 %v6)
+ call void @use_val_i64(i64 %v7)
+ call void @use_val_i64(i64 %v8)
+ call void @use_val_i64(i64 %v9)
+ call void @use_val_i64(i64 %v10)
+ call void @use_val_i64(i64 %v11)
+ call void @use_val_i64(i64 %v12)
+ br label %longjmp_return
+
+longjmp_return:
+ ret i64 %v1
+}
+
+define i32 @test_no_setjmp_i32() nounwind {
+; CHECK-LABEL: test_no_setjmp_i32:
+; ALLOC: %r30d
+; ALLOC: %r31d
+; CHECK: retq
+entry:
+ %v1 = call i32 @get_val_i32()
+ %v2 = call i32 @get_val_i32()
+ %v3 = call i32 @get_val_i32()
+ %v4 = call i32 @get_val_i32()
+ %v5 = call i32 @get_val_i32()
+ %v6 = call i32 @get_val_i32()
+ %v7 = call i32 @get_val_i32()
+ %v8 = call i32 @get_val_i32()
+ %v9 = call i32 @get_val_i32()
+ %v10 = call i32 @get_val_i32()
+ %v11 = call i32 @get_val_i32()
+ %v12 = call i32 @get_val_i32()
+ %r = call i32 @_fake_setjmp(ptr @buf)
+ %cmp = icmp eq i32 %r, 0
+ br i1 %cmp, label %normal, label %longjmp_return
+
+normal:
+ call void @external_call()
+ call void @use_val_i32(i32 %v1)
+ call void @use_val_i32(i32 %v2)
+ call void @use_val_i32(i32 %v3)
+ call void @use_val_i32(i32 %v4)
+ call void @use_val_i32(i32 %v5)
+ call void @use_val_i32(i32 %v6)
+ call void @use_val_i32(i32 %v7)
+ call void @use_val_i32(i32 %v8)
+ call void @use_val_i32(i32 %v9)
+ call void @use_val_i32(i32 %v10)
+ call void @use_val_i32(i32 %v11)
+ call void @use_val_i32(i32 %v12)
+ br label %longjmp_return
+
+longjmp_return:
+ ret i32 %v1
+}
+
+; Without returns_twice, r30/r31 must NOT be used.
+define i64 @test_setjmp_i64() nounwind {
+; CHECK-LABEL: test_setjmp_i64:
+; CHECK-NOT: r30
+; CHECK-NOT: r31
+; CHECK: retq
+entry:
+ %v1 = call i64 @get_val_i64()
+ %v2 = call i64 @get_val_i64()
+ %v3 = call i64 @get_val_i64()
+ %v4 = call i64 @get_val_i64()
+ %v5 = call i64 @get_val_i64()
+ %v6 = call i64 @get_val_i64()
+ %v7 = call i64 @get_val_i64()
+ %v8 = call i64 @get_val_i64()
+ %v9 = call i64 @get_val_i64()
+ %v10 = call i64 @get_val_i64()
+ %v11 = call i64 @get_val_i64()
+ %v12 = call i64 @get_val_i64()
+ %r = call i32 @_setjmp(ptr @buf) returns_twice
+ %cmp = icmp eq i32 %r, 0
+ br i1 %cmp, label %normal, label %longjmp_return
+
+normal:
+ call void @external_call()
+ call void @use_val_i64(i64 %v1)
+ call void @use_val_i64(i64 %v2)
+ call void @use_val_i64(i64 %v3)
+ call void @use_val_i64(i64 %v4)
+ call void @use_val_i64(i64 %v5)
+ call void @use_val_i64(i64 %v6)
+ call void @use_val_i64(i64 %v7)
+ call void @use_val_i64(i64 %v8)
+ call void @use_val_i64(i64 %v9)
+ call void @use_val_i64(i64 %v10)
+ call void @use_val_i64(i64 %v11)
+ call void @use_val_i64(i64 %v12)
+ br label %longjmp_return
+
+longjmp_return:
+ ret i64 %v1
+}
+
+define i32 @test_setjmp_i32() nounwind {
+; CHECK-LABEL: test_setjmp_i32:
+; CHECK-NOT: r30d
+; CHECK-NOT: r31d
+; CHECK: retq
+entry:
+ %v1 = call i32 @get_val_i32()
+ %v2 = call i32 @get_val_i32()
+ %v3 = call i32 @get_val_i32()
+ %v4 = call i32 @get_val_i32()
+ %v5 = call i32 @get_val_i32()
+ %v6 = call i32 @get_val_i32()
+ %v7 = call i32 @get_val_i32()
+ %v8 = call i32 @get_val_i32()
+ %v9 = call i32 @get_val_i32()
+ %v10 = call i32 @get_val_i32()
+ %v11 = call i32 @get_val_i32()
+ %v12 = call i32 @get_val_i32()
+ %r = call i32 @_setjmp(ptr @buf) returns_twice
+ %cmp = icmp eq i32 %r, 0
+ br i1 %cmp, label %normal, label %longjmp_return
+
+normal:
+ call void @external_call()
+ call void @use_val_i32(i32 %v1)
+ call void @use_val_i32(i32 %v2)
+ call void @use_val_i32(i32 %v3)
+ call void @use_val_i32(i32 %v4)
+ call void @use_val_i32(i32 %v5)
+ call void @use_val_i32(i32 %v6)
+ call void @use_val_i32(i32 %v7)
+ call void @use_val_i32(i32 %v8)
+ call void @use_val_i32(i32 %v9)
+ call void @use_val_i32(i32 %v10)
+ call void @use_val_i32(i32 %v11)
+ call void @use_val_i32(i32 %v12)
+ br label %longjmp_return
+
+longjmp_return:
+ ret i32 %v1
+}
diff --git a/llvm/test/CodeGen/X86/apx/win64-abi.ll b/llvm/test/CodeGen/X86/apx/win64-abi.ll
new file mode 100644
index 0000000000000..92bc1cdf154d1
--- /dev/null
+++ b/llvm/test/CodeGen/X86/apx/win64-abi.ll
@@ -0,0 +1,168 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-windows-msvc | FileCheck %s --check-prefixes=NO_APX_SSE
+; RUN: llc < %s -mtriple=x86_64-windows-msvc -mattr=+egpr | FileCheck %s --check-prefixes=APX_SSE
+; RUN: llc < %s -mtriple=x86_64-windows-msvc -mattr=-sse | FileCheck %s --check-prefixes=NO_APX_NOSSE
+; RUN: llc < %s -mtriple=x86_64-windows-msvc -mattr=+egpr,-sse | FileCheck %s --check-prefixes=APX_NOSSE
+; RUN: llc < %s -mtriple=x86_64-uefi | FileCheck %s --check-prefixes=NO_APX_SSE
+; RUN: llc < %s -mtriple=x86_64-uefi -mattr=+egpr | FileCheck %s --check-prefixes=APX_SSE
+; RUN: llc < %s -mtriple=x86_64-uefi -mattr=-sse | FileCheck %s --check-prefixes=NO_APX_NOSSE
+; RUN: llc < %s -mtriple=x86_64-uefi -mattr=+egpr,-sse | FileCheck %s --check-prefixes=APX_NOSSE
+
+; Test in SSE/no-SSE configurations that the Win64 APX ABI used on Windows
+; and UEFI targets makes r30 and r31 non-volatile (callee-saved) per
+; Microsoft spec, while the ordinary Win64 ABI treats them as caller-saved.
+
+declare void @external_function()
+
+define void @test_win64_apx_abi() nounwind {
+; NO_APX_SSE-LABEL: test_win64_apx_abi:
+; NO_APX_SSE: # %bb.0:
+; NO_APX_SSE-NEXT: pushq %r15
+; NO_APX_SSE-NEXT: pushq %r14
+; NO_APX_SSE-NEXT: pushq %r13
+; NO_APX_SSE-NEXT: pushq %r12
+; NO_APX_SSE-NEXT: pushq %rsi
+; NO_APX_SSE-NEXT: pushq %rdi
+; NO_APX_SSE-NEXT: pushq %rbp
+; NO_APX_SSE-NEXT: pushq %rbx
+; NO_APX_SSE-NEXT: subq $200, %rsp
+; NO_APX_SSE-NEXT: movaps %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: callq external_function
+; NO_APX_SSE-NEXT: #APP
+; NO_APX_SSE-NEXT: #NO_APP
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; NO_APX_SSE-NEXT: addq $200, %rsp
+; NO_APX_SSE-NEXT: popq %rbx
+; NO_APX_SSE-NEXT: popq %rbp
+; NO_APX_SSE-NEXT: popq %rdi
+; NO_APX_SSE-NEXT: popq %rsi
+; NO_APX_SSE-NEXT: popq %r12
+; NO_APX_SSE-NEXT: popq %r13
+; NO_APX_SSE-NEXT: popq %r14
+; NO_APX_SSE-NEXT: popq %r15
+; NO_APX_SSE-NEXT: retq
+;
+; APX_SSE-LABEL: test_win64_apx_abi:
+; APX_SSE: # %bb.0:
+; APX_SSE-NEXT: pushq %r31
+; APX_SSE-NEXT: pushq %r30
+; APX_SSE-NEXT: pushq %r15
+; APX_SSE-NEXT: pushq %r14
+; APX_SSE-NEXT: pushq %r13
+; APX_SSE-NEXT: pushq %r12
+; APX_SSE-NEXT: pushq %rsi
+; APX_SSE-NEXT: pushq %rdi
+; APX_SSE-NEXT: pushq %rbp
+; APX_SSE-NEXT: pushq %rbx
+; APX_SSE-NEXT: subq $200, %rsp
+; APX_SSE-NEXT: movaps %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: callq external_function
+; APX_SSE-NEXT: #APP
+; APX_SSE-NEXT: #NO_APP
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; APX_SSE-NEXT: addq $200, %rsp
+; APX_SSE-NEXT: popq %rbx
+; APX_SSE-NEXT: popq %rbp
+; APX_SSE-NEXT: popq %rdi
+; APX_SSE-NEXT: popq %rsi
+; APX_SSE-NEXT: popq %r12
+; APX_SSE-NEXT: popq %r13
+; APX_SSE-NEXT: popq %r14
+; APX_SSE-NEXT: popq %r15
+; APX_SSE-NEXT: popq %r30
+; APX_SSE-NEXT: popq %r31
+; APX_SSE-NEXT: retq
+;
+; NO_APX_NOSSE-LABEL: test_win64_apx_abi:
+; NO_APX_NOSSE: # %bb.0:
+; NO_APX_NOSSE-NEXT: pushq %r15
+; NO_APX_NOSSE-NEXT: pushq %r14
+; NO_APX_NOSSE-NEXT: pushq %r13
+; NO_APX_NOSSE-NEXT: pushq %r12
+; NO_APX_NOSSE-NEXT: pushq %rsi
+; NO_APX_NOSSE-NEXT: pushq %rdi
+; NO_APX_NOSSE-NEXT: pushq %rbp
+; NO_APX_NOSSE-NEXT: pushq %rbx
+; NO_APX_NOSSE-NEXT: subq $40, %rsp
+; NO_APX_NOSSE-NEXT: callq external_function
+; NO_APX_NOSSE-NEXT: #APP
+; NO_APX_NOSSE-NEXT: #NO_APP
+; NO_APX_NOSSE-NEXT: addq $40, %rsp
+; NO_APX_NOSSE-NEXT: popq %rbx
+; NO_APX_NOSSE-NEXT: popq %rbp
+; NO_APX_NOSSE-NEXT: popq %rdi
+; NO_APX_NOSSE-NEXT: popq %rsi
+; NO_APX_NOSSE-NEXT: popq %r12
+; NO_APX_NOSSE-NEXT: popq %r13
+; NO_APX_NOSSE-NEXT: popq %r14
+; NO_APX_NOSSE-NEXT: popq %r15
+; NO_APX_NOSSE-NEXT: retq
+;
+; APX_NOSSE-LABEL: test_win64_apx_abi:
+; APX_NOSSE: # %bb.0:
+; APX_NOSSE-NEXT: pushq %r31
+; APX_NOSSE-NEXT: pushq %r30
+; APX_NOSSE-NEXT: pushq %r15
+; APX_NOSSE-NEXT: pushq %r14
+; APX_NOSSE-NEXT: pushq %r13
+; APX_NOSSE-NEXT: pushq %r12
+; APX_NOSSE-NEXT: pushq %rsi
+; APX_NOSSE-NEXT: pushq %rdi
+; APX_NOSSE-NEXT: pushq %rbp
+; APX_NOSSE-NEXT: pushq %rbx
+; APX_NOSSE-NEXT: subq $40, %rsp
+; APX_NOSSE-NEXT: callq external_function
+; APX_NOSSE-NEXT: #APP
+; APX_NOSSE-NEXT: #NO_APP
+; APX_NOSSE-NEXT: addq $40, %rsp
+; APX_NOSSE-NEXT: popq %rbx
+; APX_NOSSE-NEXT: popq %rbp
+; APX_NOSSE-NEXT: popq %rdi
+; APX_NOSSE-NEXT: popq %rsi
+; APX_NOSSE-NEXT: popq %r12
+; APX_NOSSE-NEXT: popq %r13
+; APX_NOSSE-NEXT: popq %r14
+; APX_NOSSE-NEXT: popq %r15
+; APX_NOSSE-NEXT: popq %r30
+; APX_NOSSE-NEXT: popq %r31
+; APX_NOSSE-NEXT: retq
+ call void @external_function()
+ call void asm sideeffect "", "~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15},~{r16},~{r17},~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31},~{rbp},~{xmm0},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15}"()
+ ret void
+}
diff --git a/llvm/test/CodeGen/X86/apx/win64cc-abi.ll b/llvm/test/CodeGen/X86/apx/win64cc-abi.ll
new file mode 100644
index 0000000000000..b42e7424832ab
--- /dev/null
+++ b/llvm/test/CodeGen/X86/apx/win64cc-abi.ll
@@ -0,0 +1,164 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefixes=NO_APX_SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+egpr | FileCheck %s --check-prefixes=APX_SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=-sse | FileCheck %s --check-prefixes=NO_APX_NOSSE
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+egpr,-sse | FileCheck %s --check-prefixes=APX_NOSSE
+
+; Test in SSE/no-SSE configurations that Win64 APX ABI used with win64cc
+; functions makes r30 and r31 non-volatile (callee-saved) per Microsoft spec,
+; while the ordinary Win64 ABI treats them as caller-saved.
+
+declare win64cc void @external_function()
+
+define win64cc void @test_win64cc_apx_abi() nounwind {
+; NO_APX_SSE-LABEL: test_win64cc_apx_abi:
+; NO_APX_SSE: # %bb.0:
+; NO_APX_SSE-NEXT: pushq %r15
+; NO_APX_SSE-NEXT: pushq %r14
+; NO_APX_SSE-NEXT: pushq %r13
+; NO_APX_SSE-NEXT: pushq %r12
+; NO_APX_SSE-NEXT: pushq %rsi
+; NO_APX_SSE-NEXT: pushq %rdi
+; NO_APX_SSE-NEXT: pushq %rbp
+; NO_APX_SSE-NEXT: pushq %rbx
+; NO_APX_SSE-NEXT: subq $168, %rsp
+; NO_APX_SSE-NEXT: movaps %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; NO_APX_SSE-NEXT: movaps %xmm6, (%rsp) # 16-byte Spill
+; NO_APX_SSE-NEXT: callq external_function at PLT
+; NO_APX_SSE-NEXT: #APP
+; NO_APX_SSE-NEXT: #NO_APP
+; NO_APX_SSE-NEXT: movaps (%rsp), %xmm6 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; NO_APX_SSE-NEXT: addq $168, %rsp
+; NO_APX_SSE-NEXT: popq %rbx
+; NO_APX_SSE-NEXT: popq %rbp
+; NO_APX_SSE-NEXT: popq %rdi
+; NO_APX_SSE-NEXT: popq %rsi
+; NO_APX_SSE-NEXT: popq %r12
+; NO_APX_SSE-NEXT: popq %r13
+; NO_APX_SSE-NEXT: popq %r14
+; NO_APX_SSE-NEXT: popq %r15
+; NO_APX_SSE-NEXT: retq
+;
+; APX_SSE-LABEL: test_win64cc_apx_abi:
+; APX_SSE: # %bb.0:
+; APX_SSE-NEXT: pushq %r31
+; APX_SSE-NEXT: pushq %r30
+; APX_SSE-NEXT: pushq %r15
+; APX_SSE-NEXT: pushq %r14
+; APX_SSE-NEXT: pushq %r13
+; APX_SSE-NEXT: pushq %r12
+; APX_SSE-NEXT: pushq %rsi
+; APX_SSE-NEXT: pushq %rdi
+; APX_SSE-NEXT: pushq %rbp
+; APX_SSE-NEXT: pushq %rbx
+; APX_SSE-NEXT: subq $168, %rsp
+; APX_SSE-NEXT: movaps %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; APX_SSE-NEXT: movaps %xmm6, (%rsp) # 16-byte Spill
+; APX_SSE-NEXT: callq external_function at PLT
+; APX_SSE-NEXT: #APP
+; APX_SSE-NEXT: #NO_APP
+; APX_SSE-NEXT: movaps (%rsp), %xmm6 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; APX_SSE-NEXT: addq $168, %rsp
+; APX_SSE-NEXT: popq %rbx
+; APX_SSE-NEXT: popq %rbp
+; APX_SSE-NEXT: popq %rdi
+; APX_SSE-NEXT: popq %rsi
+; APX_SSE-NEXT: popq %r12
+; APX_SSE-NEXT: popq %r13
+; APX_SSE-NEXT: popq %r14
+; APX_SSE-NEXT: popq %r15
+; APX_SSE-NEXT: popq %r30
+; APX_SSE-NEXT: popq %r31
+; APX_SSE-NEXT: retq
+;
+; NO_APX_NOSSE-LABEL: test_win64cc_apx_abi:
+; NO_APX_NOSSE: # %bb.0:
+; NO_APX_NOSSE-NEXT: pushq %r15
+; NO_APX_NOSSE-NEXT: pushq %r14
+; NO_APX_NOSSE-NEXT: pushq %r13
+; NO_APX_NOSSE-NEXT: pushq %r12
+; NO_APX_NOSSE-NEXT: pushq %rsi
+; NO_APX_NOSSE-NEXT: pushq %rdi
+; NO_APX_NOSSE-NEXT: pushq %rbp
+; NO_APX_NOSSE-NEXT: pushq %rbx
+; NO_APX_NOSSE-NEXT: pushq %rax
+; NO_APX_NOSSE-NEXT: callq external_function at PLT
+; NO_APX_NOSSE-NEXT: #APP
+; NO_APX_NOSSE-NEXT: #NO_APP
+; NO_APX_NOSSE-NEXT: addq $8, %rsp
+; NO_APX_NOSSE-NEXT: popq %rbx
+; NO_APX_NOSSE-NEXT: popq %rbp
+; NO_APX_NOSSE-NEXT: popq %rdi
+; NO_APX_NOSSE-NEXT: popq %rsi
+; NO_APX_NOSSE-NEXT: popq %r12
+; NO_APX_NOSSE-NEXT: popq %r13
+; NO_APX_NOSSE-NEXT: popq %r14
+; NO_APX_NOSSE-NEXT: popq %r15
+; NO_APX_NOSSE-NEXT: retq
+;
+; APX_NOSSE-LABEL: test_win64cc_apx_abi:
+; APX_NOSSE: # %bb.0:
+; APX_NOSSE-NEXT: pushq %r31
+; APX_NOSSE-NEXT: pushq %r30
+; APX_NOSSE-NEXT: pushq %r15
+; APX_NOSSE-NEXT: pushq %r14
+; APX_NOSSE-NEXT: pushq %r13
+; APX_NOSSE-NEXT: pushq %r12
+; APX_NOSSE-NEXT: pushq %rsi
+; APX_NOSSE-NEXT: pushq %rdi
+; APX_NOSSE-NEXT: pushq %rbp
+; APX_NOSSE-NEXT: pushq %rbx
+; APX_NOSSE-NEXT: pushq %rax
+; APX_NOSSE-NEXT: callq external_function at PLT
+; APX_NOSSE-NEXT: #APP
+; APX_NOSSE-NEXT: #NO_APP
+; APX_NOSSE-NEXT: addq $8, %rsp
+; APX_NOSSE-NEXT: popq %rbx
+; APX_NOSSE-NEXT: popq %rbp
+; APX_NOSSE-NEXT: popq %rdi
+; APX_NOSSE-NEXT: popq %rsi
+; APX_NOSSE-NEXT: popq %r12
+; APX_NOSSE-NEXT: popq %r13
+; APX_NOSSE-NEXT: popq %r14
+; APX_NOSSE-NEXT: popq %r15
+; APX_NOSSE-NEXT: popq %r30
+; APX_NOSSE-NEXT: popq %r31
+; APX_NOSSE-NEXT: retq
+ call void @external_function()
+ call void asm sideeffect "", "~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15},~{r16},~{r17},~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31},~{rbp},~{xmm0},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15}"()
+ ret void
+}
>From e410b6ff51142aed6fc6a9ea58480efb6a6e8858 Mon Sep 17 00:00:00 2001
From: Timur Golubovich <timur.golubovich at intel.com>
Date: Fri, 29 May 2026 15:09:12 +0200
Subject: [PATCH 2/6] fixed comments
---
llvm/lib/Target/X86/X86CallingConv.td | 2 --
llvm/lib/Target/X86/X86RegisterInfo.cpp | 6 +++---
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td
index 441b565e4bc03..ba1bbd4ab0961 100644
--- a/llvm/lib/Target/X86/X86CallingConv.td
+++ b/llvm/lib/Target/X86/X86CallingConv.td
@@ -1135,8 +1135,6 @@ def CSR_32EHRet : CalleeSavedRegs<(add EAX, EDX, CSR_32)>;
def CSR_64EHRet : CalleeSavedRegs<(add RAX, RDX, CSR_64)>;
def CSR_Win64_NoSSE : CalleeSavedRegs<(add RBX, RBP, RDI, RSI, R12, R13, R14, R15)>;
-// When APX support is present, in Win64 ABI R30 and R31 are callee-saved
-// https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170#callercallee-saved-registers
def CSR_Win64_EGPR : CalleeSavedRegs<(add R30, R31)>;
def CSR_Win64_NoSSE_APX : CalleeSavedRegs<(add CSR_Win64_NoSSE, CSR_Win64_EGPR)>;
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 7ec5d11667bf7..f1a58a5349f5c 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -619,9 +619,9 @@ BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
if (!Is64Bit || !MF.getSubtarget<X86Subtarget>().hasEGPR())
Reserved.set(X86::R16, X86::R31WH + 1);
- // There is a problem with allocation R30/R31 registers in Win64 APX ABI
- // functions that have setjmp as unwinder won't be able to restore them:
- // https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170#setjmplongjmp
+ // Due to specifics of setjmp unwinding in Win64 APX ABI, the unwinder
+ // cannot restore R30/R31. Reserve them to prevent register allocation.
+ // https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention#setjmplongjmp
if (MF.exposesReturnsTwice()) {
unsigned NumReservedCSRs = 0;
for (const MCPhysReg &Reg : CSR_Win64_EGPR_SaveList)
>From 18bed7a7d9315f61c741c8dfbe2f3bd9120ab3b9 Mon Sep 17 00:00:00 2001
From: Timur Golubovich <timur.golubovich at intel.com>
Date: Tue, 2 Jun 2026 14:22:38 +0200
Subject: [PATCH 3/6] added warning threshold, release notes, rest of cc's
Co-authored-by: Daniel Paoliello <danpao at microsoft.com>
---
llvm/docs/ReleaseNotes.md | 9 +
llvm/lib/Target/X86/X86CallingConv.td | 22 +-
llvm/lib/Target/X86/X86RegisterInfo.cpp | 127 ++++---
llvm/lib/Target/X86/X86Subtarget.cpp | 3 +
llvm/test/CodeGen/X86/apx/setjmp-win64-abi.ll | 10 +-
llvm/test/CodeGen/X86/apx/win64-abi.ll | 322 +++++++++---------
llvm/test/CodeGen/X86/apx/win64cc-abi.ll | 164 ---------
7 files changed, 282 insertions(+), 375 deletions(-)
delete mode 100644 llvm/test/CodeGen/X86/apx/win64cc-abi.ll
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 013b5d0a52067..0fcf01ad3b7a3 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -241,6 +241,15 @@ Makes programs 10x faster by doing Special New Thing.
in use. This matches the behaviour of Intel syntax and aids with
compatibility when changing the default Clang syntax to the Intel syntax.
+* Implemented Win64 APX ABI callee-saved registers: R30 and R31 are now
+ treated as non-volatile in the Win64 calling convention when APX is
+ available, per the Microsoft x64 calling convention specification.
+
+* Functions using setjmp with Win64 APX ABI now reserve R30/R31 from
+ register allocation, as the unwinder cannot restore APX extended
+ registers across longjmp. A warning is emitted for large functions
+ where this reservation may impact performance.
+
### Changes to the OCaml bindings
### Changes to the Python bindings
diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td
index ba1bbd4ab0961..625bed9c03f1a 100644
--- a/llvm/lib/Target/X86/X86CallingConv.td
+++ b/llvm/lib/Target/X86/X86CallingConv.td
@@ -1135,16 +1135,17 @@ def CSR_32EHRet : CalleeSavedRegs<(add EAX, EDX, CSR_32)>;
def CSR_64EHRet : CalleeSavedRegs<(add RAX, RDX, CSR_64)>;
def CSR_Win64_NoSSE : CalleeSavedRegs<(add RBX, RBP, RDI, RSI, R12, R13, R14, R15)>;
-def CSR_Win64_EGPR : CalleeSavedRegs<(add R30, R31)>;
-def CSR_Win64_NoSSE_APX : CalleeSavedRegs<(add CSR_Win64_NoSSE, CSR_Win64_EGPR)>;
def CSR_Win64 : CalleeSavedRegs<(add CSR_Win64_NoSSE,
(sequence "XMM%u", 6, 15))>;
-def CSR_Win64_APX : CalleeSavedRegs<(add CSR_Win64_NoSSE_APX,
+def CSR_Win64_APX : CalleeSavedRegs<(add CSR_Win64_NoSSE,
+ R30, R31,
(sequence "XMM%u", 6, 15))>;
def CSR_Win64_SwiftError : CalleeSavedRegs<(sub CSR_Win64, R12)>;
+def CSR_Win64_APX_SwiftError : CalleeSavedRegs<(sub CSR_Win64_APX, R12)>;
def CSR_Win64_SwiftTail : CalleeSavedRegs<(sub CSR_Win64, R13, R14)>;
+def CSR_Win64_APX_SwiftTail : CalleeSavedRegs<(sub CSR_Win64_APX, R13, R14)>;
// The function used by Darwin to obtain the address of a thread-local variable
// uses rdi to pass a single parameter and rax for the return value. All other
@@ -1164,6 +1165,8 @@ def CSR_64_RT_MostRegs : CalleeSavedRegs<(add CSR_64, RAX, RCX, RDX, RSI, RDI,
def CSR_Win64_RT_MostRegs : CalleeSavedRegs<(add CSR_64_RT_MostRegs,
(sequence "XMM%u", 6, 15))>;
+def CSR_Win64_APX_RT_MostRegs : CalleeSavedRegs<(add CSR_64_RT_MostRegs, R30, R31,
+ (sequence "XMM%u", 6, 15))>;
// All registers - except r11 and return registers.
def CSR_64_RT_AllRegs : CalleeSavedRegs<(add CSR_64_RT_MostRegs,
@@ -1201,11 +1204,18 @@ def CSR_64_NoneRegs : CalleeSavedRegs<(add RBP)>;
def CSR_Win64_Intel_OCL_BI_AVX : CalleeSavedRegs<(add RBX, RBP, RDI, RSI, R12,
R13, R14, R15,
(sequence "YMM%u", 6, 15))>;
+def CSR_Win64_APX_Intel_OCL_BI_AVX : CalleeSavedRegs<(add RBX, RBP, RDI, RSI, R12,
+ R13, R14, R15, R30, R31,
+ (sequence "YMM%u", 6, 15))>;
def CSR_Win64_Intel_OCL_BI_AVX512 : CalleeSavedRegs<(add RBX, RBP, RDI, RSI,
R12, R13, R14, R15,
(sequence "ZMM%u", 6, 21),
K4, K5, K6, K7)>;
+def CSR_Win64_APX_Intel_OCL_BI_AVX512 : CalleeSavedRegs<(add RBX, RBP, RDI, RSI,
+ R12, R13, R14, R15, R30, R31,
+ (sequence "ZMM%u", 6, 21),
+ K4, K5, K6, K7)>;
//Standard C + XMM 8-15
def CSR_64_Intel_OCL_BI : CalleeSavedRegs<(add CSR_64,
(sequence "XMM%u", 8, 15))>;
@@ -1228,8 +1238,14 @@ def CSR_Win64_RegCall_NoSSE : CalleeSavedRegs<(add RBX, RBP,
(sequence "R%u", 10, 15))>;
def CSR_Win64_RegCall : CalleeSavedRegs<(add CSR_Win64_RegCall_NoSSE,
(sequence "XMM%u", 8, 15))>;
+def CSR_Win64_APX_RegCall : CalleeSavedRegs<(add CSR_Win64_RegCall_NoSSE, R30, R31,
+ (sequence "XMM%u", 8, 15))>;
def CSR_Win64_CFGuard_Check_NoSSE : CalleeSavedRegs<(add CSR_Win64_RegCall_NoSSE, RCX)>;
def CSR_Win64_CFGuard_Check : CalleeSavedRegs<(add CSR_Win64_RegCall, RCX)>;
+def CSR_Win64_APX_CFGuard_Check : CalleeSavedRegs<(add CSR_Win64_RegCall_NoSSE,
+ (sequence "R%u", 16, 31),
+ (sequence "XMM%u", 8, 15),
+ RCX)>;
def CSR_SysV64_RegCall_NoSSE : CalleeSavedRegs<(add RBX, RBP,
(sequence "R%u", 12, 15))>;
def CSR_SysV64_RegCall : CalleeSavedRegs<(add CSR_SysV64_RegCall_NoSSE,
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index f1a58a5349f5c..3038e4fbd5efa 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -27,7 +27,6 @@
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TileShapeInfo.h"
#include "llvm/CodeGen/VirtRegMap.h"
-#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Type.h"
#include "llvm/MC/MCContext.h"
@@ -51,6 +50,11 @@ static cl::opt<bool>
cl::desc("Disable two address hints for register "
"allocation"));
+static cl::opt<unsigned> SetjmpCSRWarningThreshold(
+ "x86-setjmp-csr-warning-threshold", cl::Hidden, cl::init(50),
+ cl::desc("Basic block count threshold for emitting a warning about "
+ "callee-saved registers reserved due to setjmp"));
+
extern cl::opt<bool> X86EnableAPXForRelocation;
X86RegisterInfo::X86RegisterInfo(const Triple &TT)
@@ -269,8 +273,10 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
return CSR_64_AllRegs_AVX_SaveList;
return CSR_64_AllRegs_SaveList;
case CallingConv::PreserveMost:
- return IsWin64 ? CSR_Win64_RT_MostRegs_SaveList
- : CSR_64_RT_MostRegs_SaveList;
+ if (IsWin64)
+ return (HasEGPR ? CSR_Win64_APX_RT_MostRegs_SaveList
+ : CSR_Win64_RT_MostRegs_SaveList);
+ return CSR_64_RT_MostRegs_SaveList;
case CallingConv::PreserveAll:
if (HasAVX)
return CSR_64_RT_AllRegs_AVX_SaveList;
@@ -284,11 +290,13 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
break;
case CallingConv::Intel_OCL_BI: {
if (HasAVX512 && IsWin64)
- return CSR_Win64_Intel_OCL_BI_AVX512_SaveList;
+ return (HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX512_SaveList
+ : CSR_Win64_Intel_OCL_BI_AVX512_SaveList);
if (HasAVX512 && Is64Bit)
return CSR_64_Intel_OCL_BI_AVX512_SaveList;
if (HasAVX && IsWin64)
- return CSR_Win64_Intel_OCL_BI_AVX_SaveList;
+ return (HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX_SaveList
+ : CSR_Win64_Intel_OCL_BI_AVX_SaveList);
if (HasAVX && Is64Bit)
return CSR_64_Intel_OCL_BI_AVX_SaveList;
if (!HasAVX && !IsWin64 && Is64Bit)
@@ -298,15 +306,18 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
case CallingConv::X86_RegCall:
if (Is64Bit) {
if (IsWin64) {
- return (HasSSE ? CSR_Win64_RegCall_SaveList :
- CSR_Win64_RegCall_NoSSE_SaveList);
+ if (HasEGPR) {
+ assert(HasSSE && "EGPR on Windows X64 requires SSE");
+ return CSR_Win64_APX_RegCall_SaveList;
+ }
+ return (HasSSE ? CSR_Win64_RegCall_SaveList
+ : CSR_Win64_RegCall_NoSSE_SaveList);
} else {
- return (HasSSE ? CSR_SysV64_RegCall_SaveList :
- CSR_SysV64_RegCall_NoSSE_SaveList);
+ return (HasSSE ? CSR_SysV64_RegCall_SaveList
+ : CSR_SysV64_RegCall_NoSSE_SaveList);
}
} else {
- return (HasSSE ? CSR_32_RegCall_SaveList :
- CSR_32_RegCall_NoSSE_SaveList);
+ return (HasSSE ? CSR_32_RegCall_SaveList : CSR_32_RegCall_NoSSE_SaveList);
}
case CallingConv::CFGuard_Check:
assert(!Is64Bit && "CFGuard check mechanism only used on 32-bit X86");
@@ -317,13 +328,18 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
return CSR_64_MostRegs_SaveList;
break;
case CallingConv::Win64:
- if (HasEGPR)
- return HasSSE ? CSR_Win64_APX_SaveList : CSR_Win64_NoSSE_APX_SaveList;
+ if (HasEGPR) {
+ assert(HasSSE && "EGPR on Windows X64 requires SSE");
+ return CSR_Win64_APX_SaveList;
+ }
return HasSSE ? CSR_Win64_SaveList : CSR_Win64_NoSSE_SaveList;
case CallingConv::SwiftTail:
if (!Is64Bit)
return CSR_32_SaveList;
- return IsWin64 ? CSR_Win64_SwiftTail_SaveList : CSR_64_SwiftTail_SaveList;
+ if (IsWin64)
+ return (HasEGPR ? CSR_Win64_APX_SwiftTail_SaveList
+ : CSR_Win64_SwiftTail_SaveList);
+ return CSR_64_SwiftTail_SaveList;
case CallingConv::X86_64_SysV:
if (CallsEHReturn)
return CSR_64EHRet_SaveList;
@@ -353,13 +369,18 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
if (Is64Bit) {
bool IsSwiftCC = Subtarget.getTargetLowering()->supportSwiftError() &&
F.getAttributes().hasAttrSomewhere(Attribute::SwiftError);
- if (IsSwiftCC)
- return IsWin64 ? CSR_Win64_SwiftError_SaveList
- : CSR_64_SwiftError_SaveList;
+ if (IsSwiftCC) {
+ if (IsWin64)
+ return (HasEGPR ? CSR_Win64_APX_SwiftError_SaveList
+ : CSR_Win64_SwiftError_SaveList);
+ return CSR_64_SwiftError_SaveList;
+ }
if (IsWin64 || IsUEFI64) {
- if (HasEGPR)
- return HasSSE ? CSR_Win64_APX_SaveList : CSR_Win64_NoSSE_APX_SaveList;
+ if (HasEGPR) {
+ assert(HasSSE && "EGPR on Windows X64 requires SSE");
+ return CSR_Win64_APX_SaveList;
+ }
return HasSSE ? CSR_Win64_SaveList : CSR_Win64_NoSSE_SaveList;
}
if (CallsEHReturn)
@@ -391,6 +412,7 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
bool HasSSE = Subtarget.hasSSE1();
bool HasAVX = Subtarget.hasAVX();
bool HasAVX512 = Subtarget.hasAVX512();
+ bool HasEGPR = Subtarget.hasEGPR();
switch (CC) {
case CallingConv::GHC:
@@ -401,7 +423,10 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
return CSR_64_AllRegs_AVX_RegMask;
return CSR_64_AllRegs_RegMask;
case CallingConv::PreserveMost:
- return IsWin64 ? CSR_Win64_RT_MostRegs_RegMask : CSR_64_RT_MostRegs_RegMask;
+ if (IsWin64)
+ return (HasEGPR ? CSR_Win64_APX_RT_MostRegs_RegMask
+ : CSR_Win64_RT_MostRegs_RegMask);
+ return CSR_64_RT_MostRegs_RegMask;
case CallingConv::PreserveAll:
if (HasAVX)
return CSR_64_RT_AllRegs_AVX_RegMask;
@@ -414,11 +439,13 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
break;
case CallingConv::Intel_OCL_BI: {
if (HasAVX512 && IsWin64)
- return CSR_Win64_Intel_OCL_BI_AVX512_RegMask;
+ return (HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX512_RegMask
+ : CSR_Win64_Intel_OCL_BI_AVX512_RegMask);
if (HasAVX512 && Is64Bit)
return CSR_64_Intel_OCL_BI_AVX512_RegMask;
if (HasAVX && IsWin64)
- return CSR_Win64_Intel_OCL_BI_AVX_RegMask;
+ return (HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX_RegMask
+ : CSR_Win64_Intel_OCL_BI_AVX_RegMask);
if (HasAVX && Is64Bit)
return CSR_64_Intel_OCL_BI_AVX_RegMask;
if (!HasAVX && !IsWin64 && Is64Bit)
@@ -428,18 +455,25 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
case CallingConv::X86_RegCall:
if (Is64Bit) {
if (IsWin64) {
- return (HasSSE ? CSR_Win64_RegCall_RegMask :
- CSR_Win64_RegCall_NoSSE_RegMask);
+ if (HasEGPR) {
+ assert(HasSSE && "EGPR on Windows X64 requires SSE");
+ return CSR_Win64_APX_RegCall_RegMask;
+ }
+ return (HasSSE ? CSR_Win64_RegCall_RegMask
+ : CSR_Win64_RegCall_NoSSE_RegMask);
} else {
- return (HasSSE ? CSR_SysV64_RegCall_RegMask :
- CSR_SysV64_RegCall_NoSSE_RegMask);
+ return (HasSSE ? CSR_SysV64_RegCall_RegMask
+ : CSR_SysV64_RegCall_NoSSE_RegMask);
}
} else {
- return (HasSSE ? CSR_32_RegCall_RegMask :
- CSR_32_RegCall_NoSSE_RegMask);
+ return (HasSSE ? CSR_32_RegCall_RegMask : CSR_32_RegCall_NoSSE_RegMask);
}
case CallingConv::CFGuard_Check:
if (Is64Bit) {
+ if (HasEGPR) {
+ assert(HasSSE && "EGPR on Windows X64 requires SSE");
+ return CSR_Win64_APX_CFGuard_Check_RegMask;
+ }
return (HasSSE ? CSR_Win64_CFGuard_Check_RegMask
: CSR_Win64_CFGuard_Check_NoSSE_RegMask);
} else {
@@ -451,11 +485,14 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
return CSR_64_MostRegs_RegMask;
break;
case CallingConv::Win64:
- return CSR_Win64_APX_RegMask;
+ return (HasEGPR ? CSR_Win64_APX_RegMask : CSR_Win64_RegMask);
case CallingConv::SwiftTail:
if (!Is64Bit)
return CSR_32_RegMask;
- return IsWin64 ? CSR_Win64_SwiftTail_RegMask : CSR_64_SwiftTail_RegMask;
+ if (IsWin64)
+ return (HasEGPR ? CSR_Win64_APX_SwiftTail_RegMask
+ : CSR_Win64_SwiftTail_RegMask);
+ return CSR_64_SwiftTail_RegMask;
case CallingConv::X86_64_SysV:
return CSR_64_RegMask;
case CallingConv::X86_INTR:
@@ -486,10 +523,16 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
const Function &F = MF.getFunction();
bool IsSwiftCC = Subtarget.getTargetLowering()->supportSwiftError() &&
F.getAttributes().hasAttrSomewhere(Attribute::SwiftError);
- if (IsSwiftCC)
- return IsWin64 ? CSR_Win64_SwiftError_RegMask : CSR_64_SwiftError_RegMask;
+ if (IsSwiftCC) {
+ if (IsWin64)
+ return (HasEGPR ? CSR_Win64_APX_SwiftError_RegMask
+ : CSR_Win64_SwiftError_RegMask);
+ return CSR_64_SwiftError_RegMask;
+ }
- return (IsWin64 || IsUEFI64) ? CSR_Win64_APX_RegMask : CSR_64_RegMask;
+ if (IsWin64 || IsUEFI64)
+ return (HasEGPR ? CSR_Win64_APX_RegMask : CSR_Win64_RegMask);
+ return CSR_64_RegMask;
}
return CSR_32_RegMask;
@@ -622,23 +665,21 @@ BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
// Due to specifics of setjmp unwinding in Win64 APX ABI, the unwinder
// cannot restore R30/R31. Reserve them to prevent register allocation.
// https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention#setjmplongjmp
- if (MF.exposesReturnsTwice()) {
+ if (MF.exposesReturnsTwice() && ST.isTargetWin64()) {
unsigned NumReservedCSRs = 0;
- for (const MCPhysReg &Reg : CSR_Win64_EGPR_SaveList)
+ for (unsigned Reg = X86::R16; Reg <= X86::R31; ++Reg)
if (isCalleeSavedPhysReg(Reg, MF)) {
++NumReservedCSRs;
for (const MCPhysReg &SubReg : subregs_inclusive(Reg))
Reserved.set(SubReg);
}
- if (NumReservedCSRs && MF.size() > 50 &&
+ if (NumReservedCSRs && MF.size() > SetjmpCSRWarningThreshold &&
!MF.getRegInfo().reservedRegsFrozen()) {
- const Function &F = MF.getFunction();
- F.getContext().diagnose(DiagnosticInfoResourceLimit(
- F,
- "callee-saved registers reserved due to setjmp; "
- "this may impact performance in large functions (" +
- Twine(NumReservedCSRs) + " registers excluded from allocation)",
- MF.size(), 50, DS_Warning));
+ MF.getContext().reportWarning(
+ SMLoc(), Twine(NumReservedCSRs) +
+ " callee-saved register(s) reserved due to setjmp in '" +
+ MF.getName() +
+ "'; this may impact performance in large functions");
}
}
diff --git a/llvm/lib/Target/X86/X86Subtarget.cpp b/llvm/lib/Target/X86/X86Subtarget.cpp
index 44d71e0064963..c576ba9e45506 100644
--- a/llvm/lib/Target/X86/X86Subtarget.cpp
+++ b/llvm/lib/Target/X86/X86Subtarget.cpp
@@ -287,6 +287,9 @@ void X86Subtarget::initSubtargetFeatures(StringRef CPU, StringRef TuneCPU,
reportFatalUsageError("64-bit code requested on a subtarget that doesn't "
"support it!");
+ if (HasEGPR && !hasSSE1() && isTargetWin64())
+ reportFatalUsageError("EGPR on Windows x64 requires SSE");
+
// Stack alignment is 16 bytes on Darwin, Linux, kFreeBSD, Hurd and for all
// 64-bit targets. On Solaris (32-bit), stack alignment is 4 bytes
// following the i386 psABI, while on Illumos it is always 16 bytes.
diff --git a/llvm/test/CodeGen/X86/apx/setjmp-win64-abi.ll b/llvm/test/CodeGen/X86/apx/setjmp-win64-abi.ll
index f1c8bddc26af7..c9026265e2c35 100644
--- a/llvm/test/CodeGen/X86/apx/setjmp-win64-abi.ll
+++ b/llvm/test/CodeGen/X86/apx/setjmp-win64-abi.ll
@@ -1,9 +1,13 @@
; RUN: llc < %s -mtriple=x86_64-windows-msvc -mattr=+egpr -regalloc=greedy | FileCheck %s --check-prefixes=CHECK,ALLOC
; RUN: llc < %s -mtriple=x86_64-windows-msvc -mattr=+egpr -regalloc=basic | FileCheck %s --check-prefixes=CHECK,ALLOC
; RUN: llc < %s -mtriple=x86_64-windows-msvc -mattr=+egpr -regalloc=pbqp | FileCheck %s --check-prefixes=CHECK,ALLOC
-; RUN: llc < %s -mtriple=x86_64-uefi -mattr=+egpr -regalloc=greedy | FileCheck %s --check-prefixes=CHECK,ALLOC
-; RUN: llc < %s -mtriple=x86_64-uefi -mattr=+egpr -regalloc=basic | FileCheck %s --check-prefixes=CHECK,ALLOC
-; RUN: llc < %s -mtriple=x86_64-uefi -mattr=+egpr -regalloc=pbqp | FileCheck %s --check-prefixes=CHECK,ALLOC
+; RUN: llc < %s -mtriple=x86_64-windows-msvc -mattr=+egpr -x86-setjmp-csr-warning-threshold=1 2>&1 | FileCheck %s --check-prefix=WARN
+;
+; Test that warnings are emitted exactly once per setjmp function.
+; WARN: warning: {{.*}}callee-saved register(s) reserved due to setjmp in 'test_setjmp_i64'
+; WARN-NOT: warning: {{.*}}callee-saved register(s) reserved due to setjmp in 'test_setjmp_i64'
+; WARN: warning: {{.*}}callee-saved register(s) reserved due to setjmp in 'test_setjmp_i32'
+; WARN-NOT: warning: {{.*}}callee-saved register(s) reserved due to setjmp in 'test_setjmp_i32'
; Test that R30 and R31 and all their sub-registers (r30d, r31d) are not
; allocated in functions that call setjmp (returns_twice) with Win64 APX
diff --git a/llvm/test/CodeGen/X86/apx/win64-abi.ll b/llvm/test/CodeGen/X86/apx/win64-abi.ll
index 92bc1cdf154d1..8aba657f868e7 100644
--- a/llvm/test/CodeGen/X86/apx/win64-abi.ll
+++ b/llvm/test/CodeGen/X86/apx/win64-abi.ll
@@ -1,168 +1,166 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=x86_64-windows-msvc | FileCheck %s --check-prefixes=NO_APX_SSE
-; RUN: llc < %s -mtriple=x86_64-windows-msvc -mattr=+egpr | FileCheck %s --check-prefixes=APX_SSE
-; RUN: llc < %s -mtriple=x86_64-windows-msvc -mattr=-sse | FileCheck %s --check-prefixes=NO_APX_NOSSE
-; RUN: llc < %s -mtriple=x86_64-windows-msvc -mattr=+egpr,-sse | FileCheck %s --check-prefixes=APX_NOSSE
-; RUN: llc < %s -mtriple=x86_64-uefi | FileCheck %s --check-prefixes=NO_APX_SSE
-; RUN: llc < %s -mtriple=x86_64-uefi -mattr=+egpr | FileCheck %s --check-prefixes=APX_SSE
-; RUN: llc < %s -mtriple=x86_64-uefi -mattr=-sse | FileCheck %s --check-prefixes=NO_APX_NOSSE
-; RUN: llc < %s -mtriple=x86_64-uefi -mattr=+egpr,-sse | FileCheck %s --check-prefixes=APX_NOSSE
+; RUN: llc -mtriple=x86_64-windows-msvc -mattr=+egpr < %s | FileCheck %s
+; RUN: not llc -mtriple=x86_64-windows-msvc -mattr=+egpr,-sse < %s
-; Test in SSE/no-SSE configurations that the Win64 APX ABI used on Windows
-; and UEFI targets makes r30 and r31 non-volatile (callee-saved) per
-; Microsoft spec, while the ordinary Win64 ABI treats them as caller-saved.
+declare void @external()
-declare void @external_function()
+; A callee that clobbers all EGPR registers must preserve nonvolatile R30/R31
+; and mustn't preserve volatile R16-R29.
+define void @test_callee_clobbers_all_egprs() nounwind {
+; CHECK-LABEL: test_callee_clobbers_all_egprs:
+; CHECK-NOT: pushq %r16
+; CHECK-NOT: pushq %r17
+; CHECK-NOT: pushq %r18
+; CHECK-NOT: pushq %r19
+; CHECK-NOT: pushq %r20
+; CHECK-NOT: pushq %r21
+; CHECK-NOT: pushq %r22
+; CHECK-NOT: pushq %r23
+; CHECK-NOT: pushq %r24
+; CHECK-NOT: pushq %r25
+; CHECK-NOT: pushq %r26
+; CHECK-NOT: pushq %r27
+; CHECK-NOT: pushq %r28
+; CHECK-NOT: pushq %r29
+; CHECK-DAG: pushq %r30
+; CHECK-DAG: pushq %r31
+; CHECK: callq external
+; CHECK-DAG: popq %r31
+; CHECK-DAG: popq %r30
+; CHECK-NOT: popq %r29
+; CHECK-NOT: popq %r28
+; CHECK-NOT: popq %r27
+; CHECK-NOT: popq %r26
+; CHECK-NOT: popq %r25
+; CHECK-NOT: popq %r24
+; CHECK-NOT: popq %r23
+; CHECK-NOT: popq %r22
+; CHECK-NOT: popq %r21
+; CHECK-NOT: popq %r20
+; CHECK-NOT: popq %r19
+; CHECK-NOT: popq %r18
+; CHECK-NOT: popq %r17
+; CHECK-NOT: popq %r16
+; CHECK: retq
+ call void @external()
+ call void asm sideeffect "", "~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15},~{r16},~{r17},~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31},~{rbp},~{xmm0},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15}"()
+ ret void
+}
+
+; A caller should NOT save/restore R30/R31 around calls (they are preserved
+; by the callee).
+define void @test_caller_no_save() nounwind {
+; CHECK-LABEL: test_caller_no_save:
+; CHECK-NOT: pushq %r30
+; CHECK-NOT: pushq %r31
+; CHECK: callq external
+; CHECK-NOT: popq %r31
+; CHECK-NOT: popq %r30
+; CHECK: retq
+ call void @external()
+ ret void
+}
+
+; PreserveMost CC with EGPR must preserve R30 and R31.
+define preserve_mostcc void @test_preservemost_cc_apx_abi() nounwind {
+; CHECK-LABEL: test_preservemost_cc_apx_abi:
+; CHECK-DAG: pushq %r30
+; CHECK-DAG: pushq %r31
+; CHECK-DAG: popq %r31
+; CHECK-DAG: popq %r30
+ call void asm sideeffect "", "~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15},~{r16},~{r17},~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31},~{rbp},~{xmm0},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15}"()
+ ret void
+}
+
+; Intel OCL BI AVX CC with EGPR must preserve R30 and R31.
+define intel_ocl_bicc void @test_intel_ocl_bicc_avx_apx_abi() nounwind "target-features"="+avx" {
+; CHECK-LABEL: test_intel_ocl_bicc_avx_apx_abi:
+; CHECK-DAG: pushq %r30
+; CHECK-DAG: pushq %r31
+; CHECK-DAG: popq %r31
+; CHECK-DAG: popq %r30
+ call void asm sideeffect "", "~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15},~{r16},~{r17},~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31},~{rbp},~{xmm0},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15}"()
+ ret void
+}
+
+; Intel OCL BI AVX512 CC with EGPR must preserve R30 and R31.
+define intel_ocl_bicc void @test_intel_ocl_bicc_apx_avx512_abi() nounwind "target-features"="+avx512f" {
+; CHECK-LABEL: test_intel_ocl_bicc_apx_avx512_abi:
+; CHECK-DAG: pushq %r30
+; CHECK-DAG: pushq %r31
+; CHECK-DAG: popq %r31
+; CHECK-DAG: popq %r30
+ call void asm sideeffect "", "~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15},~{r16},~{r17},~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31},~{rbp},~{xmm0},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15}"()
+ ret void
+}
-define void @test_win64_apx_abi() nounwind {
-; NO_APX_SSE-LABEL: test_win64_apx_abi:
-; NO_APX_SSE: # %bb.0:
-; NO_APX_SSE-NEXT: pushq %r15
-; NO_APX_SSE-NEXT: pushq %r14
-; NO_APX_SSE-NEXT: pushq %r13
-; NO_APX_SSE-NEXT: pushq %r12
-; NO_APX_SSE-NEXT: pushq %rsi
-; NO_APX_SSE-NEXT: pushq %rdi
-; NO_APX_SSE-NEXT: pushq %rbp
-; NO_APX_SSE-NEXT: pushq %rbx
-; NO_APX_SSE-NEXT: subq $200, %rsp
-; NO_APX_SSE-NEXT: movaps %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: callq external_function
-; NO_APX_SSE-NEXT: #APP
-; NO_APX_SSE-NEXT: #NO_APP
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; NO_APX_SSE-NEXT: addq $200, %rsp
-; NO_APX_SSE-NEXT: popq %rbx
-; NO_APX_SSE-NEXT: popq %rbp
-; NO_APX_SSE-NEXT: popq %rdi
-; NO_APX_SSE-NEXT: popq %rsi
-; NO_APX_SSE-NEXT: popq %r12
-; NO_APX_SSE-NEXT: popq %r13
-; NO_APX_SSE-NEXT: popq %r14
-; NO_APX_SSE-NEXT: popq %r15
-; NO_APX_SSE-NEXT: retq
-;
-; APX_SSE-LABEL: test_win64_apx_abi:
-; APX_SSE: # %bb.0:
-; APX_SSE-NEXT: pushq %r31
-; APX_SSE-NEXT: pushq %r30
-; APX_SSE-NEXT: pushq %r15
-; APX_SSE-NEXT: pushq %r14
-; APX_SSE-NEXT: pushq %r13
-; APX_SSE-NEXT: pushq %r12
-; APX_SSE-NEXT: pushq %rsi
-; APX_SSE-NEXT: pushq %rdi
-; APX_SSE-NEXT: pushq %rbp
-; APX_SSE-NEXT: pushq %rbx
-; APX_SSE-NEXT: subq $200, %rsp
-; APX_SSE-NEXT: movaps %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: callq external_function
-; APX_SSE-NEXT: #APP
-; APX_SSE-NEXT: #NO_APP
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; APX_SSE-NEXT: addq $200, %rsp
-; APX_SSE-NEXT: popq %rbx
-; APX_SSE-NEXT: popq %rbp
-; APX_SSE-NEXT: popq %rdi
-; APX_SSE-NEXT: popq %rsi
-; APX_SSE-NEXT: popq %r12
-; APX_SSE-NEXT: popq %r13
-; APX_SSE-NEXT: popq %r14
-; APX_SSE-NEXT: popq %r15
-; APX_SSE-NEXT: popq %r30
-; APX_SSE-NEXT: popq %r31
-; APX_SSE-NEXT: retq
-;
-; NO_APX_NOSSE-LABEL: test_win64_apx_abi:
-; NO_APX_NOSSE: # %bb.0:
-; NO_APX_NOSSE-NEXT: pushq %r15
-; NO_APX_NOSSE-NEXT: pushq %r14
-; NO_APX_NOSSE-NEXT: pushq %r13
-; NO_APX_NOSSE-NEXT: pushq %r12
-; NO_APX_NOSSE-NEXT: pushq %rsi
-; NO_APX_NOSSE-NEXT: pushq %rdi
-; NO_APX_NOSSE-NEXT: pushq %rbp
-; NO_APX_NOSSE-NEXT: pushq %rbx
-; NO_APX_NOSSE-NEXT: subq $40, %rsp
-; NO_APX_NOSSE-NEXT: callq external_function
-; NO_APX_NOSSE-NEXT: #APP
-; NO_APX_NOSSE-NEXT: #NO_APP
-; NO_APX_NOSSE-NEXT: addq $40, %rsp
-; NO_APX_NOSSE-NEXT: popq %rbx
-; NO_APX_NOSSE-NEXT: popq %rbp
-; NO_APX_NOSSE-NEXT: popq %rdi
-; NO_APX_NOSSE-NEXT: popq %rsi
-; NO_APX_NOSSE-NEXT: popq %r12
-; NO_APX_NOSSE-NEXT: popq %r13
-; NO_APX_NOSSE-NEXT: popq %r14
-; NO_APX_NOSSE-NEXT: popq %r15
-; NO_APX_NOSSE-NEXT: retq
-;
-; APX_NOSSE-LABEL: test_win64_apx_abi:
-; APX_NOSSE: # %bb.0:
-; APX_NOSSE-NEXT: pushq %r31
-; APX_NOSSE-NEXT: pushq %r30
-; APX_NOSSE-NEXT: pushq %r15
-; APX_NOSSE-NEXT: pushq %r14
-; APX_NOSSE-NEXT: pushq %r13
-; APX_NOSSE-NEXT: pushq %r12
-; APX_NOSSE-NEXT: pushq %rsi
-; APX_NOSSE-NEXT: pushq %rdi
-; APX_NOSSE-NEXT: pushq %rbp
-; APX_NOSSE-NEXT: pushq %rbx
-; APX_NOSSE-NEXT: subq $40, %rsp
-; APX_NOSSE-NEXT: callq external_function
-; APX_NOSSE-NEXT: #APP
-; APX_NOSSE-NEXT: #NO_APP
-; APX_NOSSE-NEXT: addq $40, %rsp
-; APX_NOSSE-NEXT: popq %rbx
-; APX_NOSSE-NEXT: popq %rbp
-; APX_NOSSE-NEXT: popq %rdi
-; APX_NOSSE-NEXT: popq %rsi
-; APX_NOSSE-NEXT: popq %r12
-; APX_NOSSE-NEXT: popq %r13
-; APX_NOSSE-NEXT: popq %r14
-; APX_NOSSE-NEXT: popq %r15
-; APX_NOSSE-NEXT: popq %r30
-; APX_NOSSE-NEXT: popq %r31
-; APX_NOSSE-NEXT: retq
- call void @external_function()
+; RegCall CC with EGPR must preserve R30 and R31.
+define x86_regcallcc void @test_x86_regcallcc_apx_abi() nounwind {
+; CHECK-LABEL: test_x86_regcallcc_apx_abi:
+; CHECK-DAG: pushq %r30
+; CHECK-DAG: pushq %r31
+; CHECK-DAG: popq %r31
+; CHECK-DAG: popq %r30
call void asm sideeffect "", "~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15},~{r16},~{r17},~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31},~{rbp},~{xmm0},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15}"()
ret void
}
+
+; SwiftError CC with EGPR must preserve R30 and R31, but NOT R12.
+define void @test_swifterror_apx_abi(ptr swifterror %err) nounwind {
+; CHECK-LABEL: test_swifterror_apx_abi:
+; CHECK-DAG: pushq %r30
+; CHECK-DAG: pushq %r31
+; CHECK-NOT: pushq %r12
+; CHECK-DAG: popq %r31
+; CHECK-DAG: popq %r30
+ call void asm sideeffect "", "~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15},~{r16},~{r17},~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31},~{rbp},~{xmm0},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15}"()
+ ret void
+}
+
+; SwiftTail CC with EGPR must preserve R30 and R31, but NOT R13 or R14.
+define swifttailcc void @test_swifttailcc_apx_abi() nounwind {
+; CHECK-LABEL: test_swifttailcc_apx_abi:
+; CHECK-DAG: pushq %r30
+; CHECK-DAG: pushq %r31
+; CHECK-NOT: pushq %r13
+; CHECK-NOT: pushq %r14
+; CHECK-DAG: popq %r31
+; CHECK-DAG: popq %r30
+ call void asm sideeffect "", "~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15},~{r16},~{r17},~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31},~{rbp},~{xmm0},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15}"()
+ ret void
+}
+
+; CFGuard dispatch must preserve R30 (the check stub never touches EGPR).
+; The CFGuard check call should NOT cause an extra spill of R30.
+
+declare i32 @target_func()
+
+define i64 @test_cfguard_preserves_r30() nounwind {
+; CHECK-LABEL: test_cfguard_preserves_r30:
+; CHECK: pushq %r30
+; CHECK: __guard_dispatch_icall_fptr
+; CHECK-NOT: movq %r30, {{.*}}(%rsp)
+; CHECK: popq %r30
+; CHECK: retq
+entry:
+ %val = call i64 asm sideeffect "", "={r30}"()
+ %fptr = alloca ptr, align 8
+ store ptr @target_func, ptr %fptr, align 8
+ %0 = load ptr, ptr %fptr, align 8
+ %1 = call i32 %0()
+ ret i64 %val
+}
+
+; PUSH2/POP2 must be used for callee-saved registers including R30+R31 pair.
+define void @test_push2pop2_r30_r31() nounwind "target-features"="+egpr,+push2pop2" {
+; CHECK-LABEL: test_push2pop2_r30_r31:
+; CHECK: push2 %r30, %r31
+; CHECK: callq external
+; CHECK: pop2 %r31, %r30
+; CHECK: retq
+ call void @external()
+ call void asm sideeffect "", "~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15},~{r16},~{r17},~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31},~{rbp},~{xmm0},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15}"()
+ ret void
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 2, !"cfguard", i32 2}
diff --git a/llvm/test/CodeGen/X86/apx/win64cc-abi.ll b/llvm/test/CodeGen/X86/apx/win64cc-abi.ll
deleted file mode 100644
index b42e7424832ab..0000000000000
--- a/llvm/test/CodeGen/X86/apx/win64cc-abi.ll
+++ /dev/null
@@ -1,164 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefixes=NO_APX_SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+egpr | FileCheck %s --check-prefixes=APX_SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=-sse | FileCheck %s --check-prefixes=NO_APX_NOSSE
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+egpr,-sse | FileCheck %s --check-prefixes=APX_NOSSE
-
-; Test in SSE/no-SSE configurations that Win64 APX ABI used with win64cc
-; functions makes r30 and r31 non-volatile (callee-saved) per Microsoft spec,
-; while the ordinary Win64 ABI treats them as caller-saved.
-
-declare win64cc void @external_function()
-
-define win64cc void @test_win64cc_apx_abi() nounwind {
-; NO_APX_SSE-LABEL: test_win64cc_apx_abi:
-; NO_APX_SSE: # %bb.0:
-; NO_APX_SSE-NEXT: pushq %r15
-; NO_APX_SSE-NEXT: pushq %r14
-; NO_APX_SSE-NEXT: pushq %r13
-; NO_APX_SSE-NEXT: pushq %r12
-; NO_APX_SSE-NEXT: pushq %rsi
-; NO_APX_SSE-NEXT: pushq %rdi
-; NO_APX_SSE-NEXT: pushq %rbp
-; NO_APX_SSE-NEXT: pushq %rbx
-; NO_APX_SSE-NEXT: subq $168, %rsp
-; NO_APX_SSE-NEXT: movaps %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; NO_APX_SSE-NEXT: movaps %xmm6, (%rsp) # 16-byte Spill
-; NO_APX_SSE-NEXT: callq external_function at PLT
-; NO_APX_SSE-NEXT: #APP
-; NO_APX_SSE-NEXT: #NO_APP
-; NO_APX_SSE-NEXT: movaps (%rsp), %xmm6 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; NO_APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; NO_APX_SSE-NEXT: addq $168, %rsp
-; NO_APX_SSE-NEXT: popq %rbx
-; NO_APX_SSE-NEXT: popq %rbp
-; NO_APX_SSE-NEXT: popq %rdi
-; NO_APX_SSE-NEXT: popq %rsi
-; NO_APX_SSE-NEXT: popq %r12
-; NO_APX_SSE-NEXT: popq %r13
-; NO_APX_SSE-NEXT: popq %r14
-; NO_APX_SSE-NEXT: popq %r15
-; NO_APX_SSE-NEXT: retq
-;
-; APX_SSE-LABEL: test_win64cc_apx_abi:
-; APX_SSE: # %bb.0:
-; APX_SSE-NEXT: pushq %r31
-; APX_SSE-NEXT: pushq %r30
-; APX_SSE-NEXT: pushq %r15
-; APX_SSE-NEXT: pushq %r14
-; APX_SSE-NEXT: pushq %r13
-; APX_SSE-NEXT: pushq %r12
-; APX_SSE-NEXT: pushq %rsi
-; APX_SSE-NEXT: pushq %rdi
-; APX_SSE-NEXT: pushq %rbp
-; APX_SSE-NEXT: pushq %rbx
-; APX_SSE-NEXT: subq $168, %rsp
-; APX_SSE-NEXT: movaps %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; APX_SSE-NEXT: movaps %xmm6, (%rsp) # 16-byte Spill
-; APX_SSE-NEXT: callq external_function at PLT
-; APX_SSE-NEXT: #APP
-; APX_SSE-NEXT: #NO_APP
-; APX_SSE-NEXT: movaps (%rsp), %xmm6 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; APX_SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; APX_SSE-NEXT: addq $168, %rsp
-; APX_SSE-NEXT: popq %rbx
-; APX_SSE-NEXT: popq %rbp
-; APX_SSE-NEXT: popq %rdi
-; APX_SSE-NEXT: popq %rsi
-; APX_SSE-NEXT: popq %r12
-; APX_SSE-NEXT: popq %r13
-; APX_SSE-NEXT: popq %r14
-; APX_SSE-NEXT: popq %r15
-; APX_SSE-NEXT: popq %r30
-; APX_SSE-NEXT: popq %r31
-; APX_SSE-NEXT: retq
-;
-; NO_APX_NOSSE-LABEL: test_win64cc_apx_abi:
-; NO_APX_NOSSE: # %bb.0:
-; NO_APX_NOSSE-NEXT: pushq %r15
-; NO_APX_NOSSE-NEXT: pushq %r14
-; NO_APX_NOSSE-NEXT: pushq %r13
-; NO_APX_NOSSE-NEXT: pushq %r12
-; NO_APX_NOSSE-NEXT: pushq %rsi
-; NO_APX_NOSSE-NEXT: pushq %rdi
-; NO_APX_NOSSE-NEXT: pushq %rbp
-; NO_APX_NOSSE-NEXT: pushq %rbx
-; NO_APX_NOSSE-NEXT: pushq %rax
-; NO_APX_NOSSE-NEXT: callq external_function at PLT
-; NO_APX_NOSSE-NEXT: #APP
-; NO_APX_NOSSE-NEXT: #NO_APP
-; NO_APX_NOSSE-NEXT: addq $8, %rsp
-; NO_APX_NOSSE-NEXT: popq %rbx
-; NO_APX_NOSSE-NEXT: popq %rbp
-; NO_APX_NOSSE-NEXT: popq %rdi
-; NO_APX_NOSSE-NEXT: popq %rsi
-; NO_APX_NOSSE-NEXT: popq %r12
-; NO_APX_NOSSE-NEXT: popq %r13
-; NO_APX_NOSSE-NEXT: popq %r14
-; NO_APX_NOSSE-NEXT: popq %r15
-; NO_APX_NOSSE-NEXT: retq
-;
-; APX_NOSSE-LABEL: test_win64cc_apx_abi:
-; APX_NOSSE: # %bb.0:
-; APX_NOSSE-NEXT: pushq %r31
-; APX_NOSSE-NEXT: pushq %r30
-; APX_NOSSE-NEXT: pushq %r15
-; APX_NOSSE-NEXT: pushq %r14
-; APX_NOSSE-NEXT: pushq %r13
-; APX_NOSSE-NEXT: pushq %r12
-; APX_NOSSE-NEXT: pushq %rsi
-; APX_NOSSE-NEXT: pushq %rdi
-; APX_NOSSE-NEXT: pushq %rbp
-; APX_NOSSE-NEXT: pushq %rbx
-; APX_NOSSE-NEXT: pushq %rax
-; APX_NOSSE-NEXT: callq external_function at PLT
-; APX_NOSSE-NEXT: #APP
-; APX_NOSSE-NEXT: #NO_APP
-; APX_NOSSE-NEXT: addq $8, %rsp
-; APX_NOSSE-NEXT: popq %rbx
-; APX_NOSSE-NEXT: popq %rbp
-; APX_NOSSE-NEXT: popq %rdi
-; APX_NOSSE-NEXT: popq %rsi
-; APX_NOSSE-NEXT: popq %r12
-; APX_NOSSE-NEXT: popq %r13
-; APX_NOSSE-NEXT: popq %r14
-; APX_NOSSE-NEXT: popq %r15
-; APX_NOSSE-NEXT: popq %r30
-; APX_NOSSE-NEXT: popq %r31
-; APX_NOSSE-NEXT: retq
- call void @external_function()
- call void asm sideeffect "", "~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15},~{r16},~{r17},~{r18},~{r19},~{r20},~{r21},~{r22},~{r23},~{r24},~{r25},~{r26},~{r27},~{r28},~{r29},~{r30},~{r31},~{rbp},~{xmm0},~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15}"()
- ret void
-}
>From 5893577576bd789bbebda57381e41882176a4920 Mon Sep 17 00:00:00 2001
From: Timur Golubovich <timur.golubovich at intel.com>
Date: Tue, 2 Jun 2026 19:55:55 +0200
Subject: [PATCH 4/6] removed new line
---
llvm/lib/Target/X86/X86CallingConv.td | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td
index 625bed9c03f1a..2e03f8996969b 100644
--- a/llvm/lib/Target/X86/X86CallingConv.td
+++ b/llvm/lib/Target/X86/X86CallingConv.td
@@ -1138,8 +1138,7 @@ def CSR_Win64_NoSSE : CalleeSavedRegs<(add RBX, RBP, RDI, RSI, R12, R13, R14, R1
def CSR_Win64 : CalleeSavedRegs<(add CSR_Win64_NoSSE,
(sequence "XMM%u", 6, 15))>;
-def CSR_Win64_APX : CalleeSavedRegs<(add CSR_Win64_NoSSE,
- R30, R31,
+def CSR_Win64_APX : CalleeSavedRegs<(add CSR_Win64_NoSSE, R30, R31,
(sequence "XMM%u", 6, 15))>;
def CSR_Win64_SwiftError : CalleeSavedRegs<(sub CSR_Win64, R12)>;
>From 8784ecd76bbc28e3b84fc44dfec7ad7699e57447 Mon Sep 17 00:00:00 2001
From: Timur Golubovich <timur.golubovich at intel.com>
Date: Wed, 3 Jun 2026 18:16:31 +0200
Subject: [PATCH 5/6] removed redundant parentheses, asserts and else branches
---
llvm/lib/Target/X86/X86RegisterInfo.cpp | 137 +++++++++++-------------
1 file changed, 60 insertions(+), 77 deletions(-)
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 3038e4fbd5efa..006eced29f231 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -290,13 +290,13 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
break;
case CallingConv::Intel_OCL_BI: {
if (HasAVX512 && IsWin64)
- return (HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX512_SaveList
- : CSR_Win64_Intel_OCL_BI_AVX512_SaveList);
+ return HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX512_SaveList
+ : CSR_Win64_Intel_OCL_BI_AVX512_SaveList;
if (HasAVX512 && Is64Bit)
return CSR_64_Intel_OCL_BI_AVX512_SaveList;
if (HasAVX && IsWin64)
- return (HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX_SaveList
- : CSR_Win64_Intel_OCL_BI_AVX_SaveList);
+ return HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX_SaveList
+ : CSR_Win64_Intel_OCL_BI_AVX_SaveList;
if (HasAVX && Is64Bit)
return CSR_64_Intel_OCL_BI_AVX_SaveList;
if (!HasAVX && !IsWin64 && Is64Bit)
@@ -306,19 +306,15 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
case CallingConv::X86_RegCall:
if (Is64Bit) {
if (IsWin64) {
- if (HasEGPR) {
- assert(HasSSE && "EGPR on Windows X64 requires SSE");
- return CSR_Win64_APX_RegCall_SaveList;
- }
- return (HasSSE ? CSR_Win64_RegCall_SaveList
- : CSR_Win64_RegCall_NoSSE_SaveList);
- } else {
- return (HasSSE ? CSR_SysV64_RegCall_SaveList
- : CSR_SysV64_RegCall_NoSSE_SaveList);
+ if (HasSSE)
+ return HasEGPR ? CSR_Win64_APX_RegCall_SaveList
+ : CSR_Win64_RegCall_SaveList;
+ return CSR_Win64_RegCall_NoSSE_SaveList;
}
- } else {
- return (HasSSE ? CSR_32_RegCall_SaveList : CSR_32_RegCall_NoSSE_SaveList);
+ return HasSSE ? CSR_SysV64_RegCall_SaveList
+ : CSR_SysV64_RegCall_NoSSE_SaveList;
}
+ return (HasSSE ? CSR_32_RegCall_SaveList : CSR_32_RegCall_NoSSE_SaveList);
case CallingConv::CFGuard_Check:
assert(!Is64Bit && "CFGuard check mechanism only used on 32-bit X86");
return (HasSSE ? CSR_Win32_CFGuard_Check_SaveList
@@ -328,17 +324,15 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
return CSR_64_MostRegs_SaveList;
break;
case CallingConv::Win64:
- if (HasEGPR) {
- assert(HasSSE && "EGPR on Windows X64 requires SSE");
- return CSR_Win64_APX_SaveList;
- }
- return HasSSE ? CSR_Win64_SaveList : CSR_Win64_NoSSE_SaveList;
+ if (HasSSE)
+ return HasEGPR ? CSR_Win64_APX_SaveList : CSR_Win64_SaveList;
+ return CSR_Win64_NoSSE_SaveList;
case CallingConv::SwiftTail:
if (!Is64Bit)
return CSR_32_SaveList;
if (IsWin64)
- return (HasEGPR ? CSR_Win64_APX_SwiftTail_SaveList
- : CSR_Win64_SwiftTail_SaveList);
+ return HasEGPR ? CSR_Win64_APX_SwiftTail_SaveList
+ : CSR_Win64_SwiftTail_SaveList;
return CSR_64_SwiftTail_SaveList;
case CallingConv::X86_64_SysV:
if (CallsEHReturn)
@@ -353,15 +347,14 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
if (HasSSE)
return CSR_64_AllRegs_SaveList;
return CSR_64_AllRegs_NoSSE_SaveList;
- } else {
- if (HasAVX512)
- return CSR_32_AllRegs_AVX512_SaveList;
- if (HasAVX)
- return CSR_32_AllRegs_AVX_SaveList;
- if (HasSSE)
- return CSR_32_AllRegs_SSE_SaveList;
- return CSR_32_AllRegs_SaveList;
}
+ if (HasAVX512)
+ return CSR_32_AllRegs_AVX512_SaveList;
+ if (HasAVX)
+ return CSR_32_AllRegs_AVX_SaveList;
+ if (HasSSE)
+ return CSR_32_AllRegs_SSE_SaveList;
+ return CSR_32_AllRegs_SaveList;
default:
break;
}
@@ -371,17 +364,15 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
F.getAttributes().hasAttrSomewhere(Attribute::SwiftError);
if (IsSwiftCC) {
if (IsWin64)
- return (HasEGPR ? CSR_Win64_APX_SwiftError_SaveList
- : CSR_Win64_SwiftError_SaveList);
+ return HasEGPR ? CSR_Win64_APX_SwiftError_SaveList
+ : CSR_Win64_SwiftError_SaveList;
return CSR_64_SwiftError_SaveList;
}
if (IsWin64 || IsUEFI64) {
- if (HasEGPR) {
- assert(HasSSE && "EGPR on Windows X64 requires SSE");
- return CSR_Win64_APX_SaveList;
- }
- return HasSSE ? CSR_Win64_SaveList : CSR_Win64_NoSSE_SaveList;
+ if (HasSSE)
+ return HasEGPR ? CSR_Win64_APX_SaveList : CSR_Win64_SaveList;
+ return CSR_Win64_NoSSE_SaveList;
}
if (CallsEHReturn)
return CSR_64EHRet_SaveList;
@@ -424,8 +415,8 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
return CSR_64_AllRegs_RegMask;
case CallingConv::PreserveMost:
if (IsWin64)
- return (HasEGPR ? CSR_Win64_APX_RT_MostRegs_RegMask
- : CSR_Win64_RT_MostRegs_RegMask);
+ return HasEGPR ? CSR_Win64_APX_RT_MostRegs_RegMask
+ : CSR_Win64_RT_MostRegs_RegMask;
return CSR_64_RT_MostRegs_RegMask;
case CallingConv::PreserveAll:
if (HasAVX)
@@ -439,13 +430,13 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
break;
case CallingConv::Intel_OCL_BI: {
if (HasAVX512 && IsWin64)
- return (HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX512_RegMask
- : CSR_Win64_Intel_OCL_BI_AVX512_RegMask);
+ return HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX512_RegMask
+ : CSR_Win64_Intel_OCL_BI_AVX512_RegMask;
if (HasAVX512 && Is64Bit)
return CSR_64_Intel_OCL_BI_AVX512_RegMask;
if (HasAVX && IsWin64)
- return (HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX_RegMask
- : CSR_Win64_Intel_OCL_BI_AVX_RegMask);
+ return HasEGPR ? CSR_Win64_APX_Intel_OCL_BI_AVX_RegMask
+ : CSR_Win64_Intel_OCL_BI_AVX_RegMask;
if (HasAVX && Is64Bit)
return CSR_64_Intel_OCL_BI_AVX_RegMask;
if (!HasAVX && !IsWin64 && Is64Bit)
@@ -455,43 +446,36 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
case CallingConv::X86_RegCall:
if (Is64Bit) {
if (IsWin64) {
- if (HasEGPR) {
- assert(HasSSE && "EGPR on Windows X64 requires SSE");
- return CSR_Win64_APX_RegCall_RegMask;
- }
- return (HasSSE ? CSR_Win64_RegCall_RegMask
- : CSR_Win64_RegCall_NoSSE_RegMask);
- } else {
- return (HasSSE ? CSR_SysV64_RegCall_RegMask
- : CSR_SysV64_RegCall_NoSSE_RegMask);
+ if (HasSSE)
+ return HasEGPR ? CSR_Win64_APX_RegCall_RegMask
+ : CSR_Win64_RegCall_RegMask;
+ return CSR_Win64_RegCall_NoSSE_RegMask;
}
- } else {
- return (HasSSE ? CSR_32_RegCall_RegMask : CSR_32_RegCall_NoSSE_RegMask);
+ return HasSSE ? CSR_SysV64_RegCall_RegMask
+ : CSR_SysV64_RegCall_NoSSE_RegMask;
}
+ return HasSSE ? CSR_32_RegCall_RegMask : CSR_32_RegCall_NoSSE_RegMask;
case CallingConv::CFGuard_Check:
if (Is64Bit) {
- if (HasEGPR) {
- assert(HasSSE && "EGPR on Windows X64 requires SSE");
- return CSR_Win64_APX_CFGuard_Check_RegMask;
- }
- return (HasSSE ? CSR_Win64_CFGuard_Check_RegMask
- : CSR_Win64_CFGuard_Check_NoSSE_RegMask);
- } else {
- return (HasSSE ? CSR_Win32_CFGuard_Check_RegMask
- : CSR_Win32_CFGuard_Check_NoSSE_RegMask);
+ if (HasSSE)
+ return HasEGPR ? CSR_Win64_APX_CFGuard_Check_RegMask
+ : CSR_Win64_CFGuard_Check_RegMask;
+ return CSR_Win64_CFGuard_Check_NoSSE_RegMask;
}
+ return HasSSE ? CSR_Win32_CFGuard_Check_RegMask
+ : CSR_Win32_CFGuard_Check_NoSSE_RegMask;
case CallingConv::Cold:
if (Is64Bit)
return CSR_64_MostRegs_RegMask;
break;
case CallingConv::Win64:
- return (HasEGPR ? CSR_Win64_APX_RegMask : CSR_Win64_RegMask);
+ return HasEGPR ? CSR_Win64_APX_RegMask : CSR_Win64_RegMask;
case CallingConv::SwiftTail:
if (!Is64Bit)
return CSR_32_RegMask;
if (IsWin64)
- return (HasEGPR ? CSR_Win64_APX_SwiftTail_RegMask
- : CSR_Win64_SwiftTail_RegMask);
+ return HasEGPR ? CSR_Win64_APX_SwiftTail_RegMask
+ : CSR_Win64_SwiftTail_RegMask;
return CSR_64_SwiftTail_RegMask;
case CallingConv::X86_64_SysV:
return CSR_64_RegMask;
@@ -504,15 +488,14 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
if (HasSSE)
return CSR_64_AllRegs_RegMask;
return CSR_64_AllRegs_NoSSE_RegMask;
- } else {
- if (HasAVX512)
- return CSR_32_AllRegs_AVX512_RegMask;
- if (HasAVX)
- return CSR_32_AllRegs_AVX_RegMask;
- if (HasSSE)
- return CSR_32_AllRegs_SSE_RegMask;
- return CSR_32_AllRegs_RegMask;
}
+ if (HasAVX512)
+ return CSR_32_AllRegs_AVX512_RegMask;
+ if (HasAVX)
+ return CSR_32_AllRegs_AVX_RegMask;
+ if (HasSSE)
+ return CSR_32_AllRegs_SSE_RegMask;
+ return CSR_32_AllRegs_RegMask;
default:
break;
}
@@ -525,13 +508,13 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
F.getAttributes().hasAttrSomewhere(Attribute::SwiftError);
if (IsSwiftCC) {
if (IsWin64)
- return (HasEGPR ? CSR_Win64_APX_SwiftError_RegMask
- : CSR_Win64_SwiftError_RegMask);
+ return HasEGPR ? CSR_Win64_APX_SwiftError_RegMask
+ : CSR_Win64_SwiftError_RegMask;
return CSR_64_SwiftError_RegMask;
}
if (IsWin64 || IsUEFI64)
- return (HasEGPR ? CSR_Win64_APX_RegMask : CSR_Win64_RegMask);
+ return HasEGPR ? CSR_Win64_APX_RegMask : CSR_Win64_RegMask;
return CSR_64_RegMask;
}
>From d0bf408c6de5f85965ca1879fe894704107cd110 Mon Sep 17 00:00:00 2001
From: Timur Golubovich <timur.golubovich at intel.com>
Date: Thu, 4 Jun 2026 16:47:28 +0200
Subject: [PATCH 6/6] removed all parentheses in return statements
---
llvm/lib/Target/X86/X86RegisterInfo.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 006eced29f231..c84e0f441a459 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -274,8 +274,8 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
return CSR_64_AllRegs_SaveList;
case CallingConv::PreserveMost:
if (IsWin64)
- return (HasEGPR ? CSR_Win64_APX_RT_MostRegs_SaveList
- : CSR_Win64_RT_MostRegs_SaveList);
+ return HasEGPR ? CSR_Win64_APX_RT_MostRegs_SaveList
+ : CSR_Win64_RT_MostRegs_SaveList;
return CSR_64_RT_MostRegs_SaveList;
case CallingConv::PreserveAll:
if (HasAVX)
@@ -314,11 +314,11 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
return HasSSE ? CSR_SysV64_RegCall_SaveList
: CSR_SysV64_RegCall_NoSSE_SaveList;
}
- return (HasSSE ? CSR_32_RegCall_SaveList : CSR_32_RegCall_NoSSE_SaveList);
+ return HasSSE ? CSR_32_RegCall_SaveList : CSR_32_RegCall_NoSSE_SaveList;
case CallingConv::CFGuard_Check:
assert(!Is64Bit && "CFGuard check mechanism only used on 32-bit X86");
- return (HasSSE ? CSR_Win32_CFGuard_Check_SaveList
- : CSR_Win32_CFGuard_Check_NoSSE_SaveList);
+ return HasSSE ? CSR_Win32_CFGuard_Check_SaveList
+ : CSR_Win32_CFGuard_Check_NoSSE_SaveList;
case CallingConv::Cold:
if (Is64Bit)
return CSR_64_MostRegs_SaveList;
More information about the llvm-commits
mailing list