[llvm] [X86] Add i256 shift / funnel shift coverage to match i512 tests (PR #184346)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 3 06:21:31 PST 2026
https://github.com/RKSimon created https://github.com/llvm/llvm-project/pull/184346
shift-i256.ll - added x86-64/x86-64-v2/x86-64-v3/x86-64-v4 coverage and retained the x86 test coverage
>From 3ef46b42b6eab59ae2c564fed700e20973850676 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Tue, 3 Mar 2026 14:19:20 +0000
Subject: [PATCH] [X86] Add i256 shift / funnel shift coverage to match i512
tests
shift-i256.ll - added x86-64/x86-64-v2/x86-64-v3/x86-64-v4 coverage and retained the x86 test coverage
---
llvm/test/CodeGen/X86/funnel-shift-i256.ll | 2056 ++++++++++++
llvm/test/CodeGen/X86/shift-i256.ll | 3488 ++++++++++++++++++--
2 files changed, 5228 insertions(+), 316 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/funnel-shift-i256.ll
diff --git a/llvm/test/CodeGen/X86/funnel-shift-i256.ll b/llvm/test/CodeGen/X86/funnel-shift-i256.ll
new file mode 100644
index 0000000000000..7b787ab6a7a59
--- /dev/null
+++ b/llvm/test/CodeGen/X86/funnel-shift-i256.ll
@@ -0,0 +1,2056 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s -check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s -check-prefixes=SSE,SSE42
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s -check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=knl | FileCheck %s -check-prefixes=AVX,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s -check-prefixes=AVX,AVX512,AVX512VL
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 -mattr=+avx512vbmi2 | FileCheck %s -check-prefixes=AVX,AVX512,AVX512VBMI
+
+define i256 @fshl_i256(i256 %a0, i256 %a1, i256 %a2) nounwind {
+; SSE-LABEL: fshl_i256:
+; SSE: # %bb.0:
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq %rcx, %r10
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE-NEXT: testb $-128, %cl
+; SSE-NEXT: je .LBB0_1
+; SSE-NEXT: # %bb.2:
+; SSE-NEXT: movq %rbx, %r11
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; SSE-NEXT: jmp .LBB0_3
+; SSE-NEXT: .LBB0_1:
+; SSE-NEXT: movq %rdi, %r9
+; SSE-NEXT: movq %rdx, %r11
+; SSE-NEXT: movq %rsi, %rdi
+; SSE-NEXT: movq %r10, %rsi
+; SSE-NEXT: movq %r8, %rdx
+; SSE-NEXT: .LBB0_3:
+; SSE-NEXT: testb $64, %cl
+; SSE-NEXT: cmovneq %rsi, %rdx
+; SSE-NEXT: cmovneq %r11, %rsi
+; SSE-NEXT: cmovneq %rdi, %r11
+; SSE-NEXT: cmoveq %rbx, %r9
+; SSE-NEXT: cmovneq %rbx, %rdi
+; SSE-NEXT: movq %rsi, %r8
+; SSE-NEXT: shldq %cl, %r11, %r8
+; SSE-NEXT: shldq %cl, %rsi, %rdx
+; SSE-NEXT: movq %rdi, %rsi
+; SSE-NEXT: shldq %cl, %r9, %rsi
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shldq %cl, %rdi, %r11
+; SSE-NEXT: movq %rdx, 24(%rax)
+; SSE-NEXT: movq %r8, 16(%rax)
+; SSE-NEXT: movq %r11, 8(%rax)
+; SSE-NEXT: movq %rsi, (%rax)
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: retq
+;
+; AVX-LABEL: fshl_i256:
+; AVX: # %bb.0:
+; AVX-NEXT: pushq %rbx
+; AVX-NEXT: movq %rcx, %r10
+; AVX-NEXT: movq %rdi, %rax
+; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX-NEXT: testb $-128, %cl
+; AVX-NEXT: je .LBB0_1
+; AVX-NEXT: # %bb.2:
+; AVX-NEXT: movq %rbx, %r11
+; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX-NEXT: jmp .LBB0_3
+; AVX-NEXT: .LBB0_1:
+; AVX-NEXT: movq %rdi, %r9
+; AVX-NEXT: movq %rdx, %r11
+; AVX-NEXT: movq %rsi, %rdi
+; AVX-NEXT: movq %r10, %rsi
+; AVX-NEXT: movq %r8, %rdx
+; AVX-NEXT: .LBB0_3:
+; AVX-NEXT: testb $64, %cl
+; AVX-NEXT: cmovneq %rsi, %rdx
+; AVX-NEXT: cmovneq %r11, %rsi
+; AVX-NEXT: cmovneq %rdi, %r11
+; AVX-NEXT: cmoveq %rbx, %r9
+; AVX-NEXT: cmovneq %rbx, %rdi
+; AVX-NEXT: movq %rsi, %r8
+; AVX-NEXT: shldq %cl, %r11, %r8
+; AVX-NEXT: shldq %cl, %rsi, %rdx
+; AVX-NEXT: movq %rdi, %rsi
+; AVX-NEXT: shldq %cl, %r9, %rsi
+; AVX-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX-NEXT: shldq %cl, %rdi, %r11
+; AVX-NEXT: movq %rdx, 24(%rax)
+; AVX-NEXT: movq %r8, 16(%rax)
+; AVX-NEXT: movq %r11, 8(%rax)
+; AVX-NEXT: movq %rsi, (%rax)
+; AVX-NEXT: popq %rbx
+; AVX-NEXT: retq
+ %r = call i256 @llvm.fshl.i256(i256 %a0, i256 %a1, i256 %a2)
+ ret i256 %r
+}
+
+define i256 @fshr_i256(i256 %a0, i256 %a1, i256 %a2) nounwind {
+; SSE-LABEL: fshr_i256:
+; SSE: # %bb.0:
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq %rcx, %r10
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; SSE-NEXT: movzbl {{[0-9]+}}(%rsp), %ecx
+; SSE-NEXT: testb %cl, %cl
+; SSE-NEXT: js .LBB1_1
+; SSE-NEXT: # %bb.2:
+; SSE-NEXT: movq %rbx, %r11
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; SSE-NEXT: jmp .LBB1_3
+; SSE-NEXT: .LBB1_1:
+; SSE-NEXT: movq %rdi, %r9
+; SSE-NEXT: movq %rsi, %rdi
+; SSE-NEXT: movq %rdx, %r11
+; SSE-NEXT: movq %r10, %rsi
+; SSE-NEXT: movq %r8, %rdx
+; SSE-NEXT: .LBB1_3:
+; SSE-NEXT: testb $64, %cl
+; SSE-NEXT: cmoveq %rsi, %rdx
+; SSE-NEXT: cmoveq %r11, %rsi
+; SSE-NEXT: cmoveq %rdi, %r11
+; SSE-NEXT: cmoveq %rbx, %rdi
+; SSE-NEXT: cmovneq %rbx, %r9
+; SSE-NEXT: movq %r11, %r8
+; SSE-NEXT: shrdq %cl, %rsi, %r8
+; SSE-NEXT: shrdq %cl, %rdx, %rsi
+; SSE-NEXT: shrdq %cl, %rdi, %r9
+; SSE-NEXT: shrdq %cl, %r11, %rdi
+; SSE-NEXT: movq %rsi, 24(%rax)
+; SSE-NEXT: movq %r8, 16(%rax)
+; SSE-NEXT: movq %rdi, 8(%rax)
+; SSE-NEXT: movq %r9, (%rax)
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: retq
+;
+; AVX-LABEL: fshr_i256:
+; AVX: # %bb.0:
+; AVX-NEXT: pushq %rbx
+; AVX-NEXT: movq %rcx, %r10
+; AVX-NEXT: movq %rdi, %rax
+; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX-NEXT: movzbl {{[0-9]+}}(%rsp), %ecx
+; AVX-NEXT: testb %cl, %cl
+; AVX-NEXT: js .LBB1_1
+; AVX-NEXT: # %bb.2:
+; AVX-NEXT: movq %rbx, %r11
+; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX-NEXT: jmp .LBB1_3
+; AVX-NEXT: .LBB1_1:
+; AVX-NEXT: movq %rdi, %r9
+; AVX-NEXT: movq %rsi, %rdi
+; AVX-NEXT: movq %rdx, %r11
+; AVX-NEXT: movq %r10, %rsi
+; AVX-NEXT: movq %r8, %rdx
+; AVX-NEXT: .LBB1_3:
+; AVX-NEXT: testb $64, %cl
+; AVX-NEXT: cmoveq %rsi, %rdx
+; AVX-NEXT: cmoveq %r11, %rsi
+; AVX-NEXT: cmoveq %rdi, %r11
+; AVX-NEXT: cmoveq %rbx, %rdi
+; AVX-NEXT: cmovneq %rbx, %r9
+; AVX-NEXT: movq %r11, %r8
+; AVX-NEXT: shrdq %cl, %rsi, %r8
+; AVX-NEXT: shrdq %cl, %rdx, %rsi
+; AVX-NEXT: shrdq %cl, %rdi, %r9
+; AVX-NEXT: shrdq %cl, %r11, %rdi
+; AVX-NEXT: movq %rsi, 24(%rax)
+; AVX-NEXT: movq %r8, 16(%rax)
+; AVX-NEXT: movq %rdi, 8(%rax)
+; AVX-NEXT: movq %r9, (%rax)
+; AVX-NEXT: popq %rbx
+; AVX-NEXT: retq
+ %r = call i256 @llvm.fshr.i256(i256 %a0, i256 %a1, i256 %a2)
+ ret i256 %r
+}
+
+define i256 @fshl_rot_i256(i256 %a0, i256 %a1) nounwind {
+; SSE-LABEL: fshl_rot_i256:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: testb $-128, %r9b
+; SSE-NEXT: movq %r8, %rcx
+; SSE-NEXT: cmovneq %rdx, %rcx
+; SSE-NEXT: movq %rsi, %r10
+; SSE-NEXT: cmovneq %rax, %r10
+; SSE-NEXT: cmovneq %r8, %rdx
+; SSE-NEXT: cmovneq %rsi, %rax
+; SSE-NEXT: testb $64, %r9b
+; SSE-NEXT: movq %rax, %rsi
+; SSE-NEXT: cmovneq %rdx, %rsi
+; SSE-NEXT: cmovneq %r10, %rdx
+; SSE-NEXT: cmoveq %rcx, %rax
+; SSE-NEXT: cmovneq %rcx, %r10
+; SSE-NEXT: movq %r10, %r8
+; SSE-NEXT: movl %r9d, %ecx
+; SSE-NEXT: shldq %cl, %rax, %r8
+; SSE-NEXT: movq %rdx, %r11
+; SSE-NEXT: shldq %cl, %r10, %r11
+; SSE-NEXT: movq %rsi, %r10
+; SSE-NEXT: shldq %cl, %rdx, %r10
+; SSE-NEXT: shldq %cl, %rsi, %rax
+; SSE-NEXT: movq %rax, 24(%rdi)
+; SSE-NEXT: movq %r10, 16(%rdi)
+; SSE-NEXT: movq %r11, 8(%rdi)
+; SSE-NEXT: movq %r8, (%rdi)
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: retq
+;
+; AVX-LABEL: fshl_rot_i256:
+; AVX: # %bb.0:
+; AVX-NEXT: movq %rcx, %rax
+; AVX-NEXT: testb $-128, %r9b
+; AVX-NEXT: movq %r8, %rcx
+; AVX-NEXT: cmovneq %rdx, %rcx
+; AVX-NEXT: movq %rsi, %r10
+; AVX-NEXT: cmovneq %rax, %r10
+; AVX-NEXT: cmovneq %r8, %rdx
+; AVX-NEXT: cmovneq %rsi, %rax
+; AVX-NEXT: testb $64, %r9b
+; AVX-NEXT: movq %rax, %rsi
+; AVX-NEXT: cmovneq %rdx, %rsi
+; AVX-NEXT: cmovneq %r10, %rdx
+; AVX-NEXT: cmoveq %rcx, %rax
+; AVX-NEXT: cmovneq %rcx, %r10
+; AVX-NEXT: movq %r10, %r8
+; AVX-NEXT: movl %r9d, %ecx
+; AVX-NEXT: shldq %cl, %rax, %r8
+; AVX-NEXT: movq %rdx, %r11
+; AVX-NEXT: shldq %cl, %r10, %r11
+; AVX-NEXT: movq %rsi, %r10
+; AVX-NEXT: shldq %cl, %rdx, %r10
+; AVX-NEXT: shldq %cl, %rsi, %rax
+; AVX-NEXT: movq %rax, 24(%rdi)
+; AVX-NEXT: movq %r10, 16(%rdi)
+; AVX-NEXT: movq %r11, 8(%rdi)
+; AVX-NEXT: movq %r8, (%rdi)
+; AVX-NEXT: movq %rdi, %rax
+; AVX-NEXT: retq
+ %r = call i256 @llvm.fshl.i256(i256 %a0, i256 %a0, i256 %a1)
+ ret i256 %r
+}
+
+define i256 @fshr_rot_i256(i256 %a0, i256 %a1) nounwind {
+; SSE-LABEL: fshr_rot_i256:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: testb %r9b, %r9b
+; SSE-NEXT: movq %r8, %rcx
+; SSE-NEXT: cmovnsq %rdx, %rcx
+; SSE-NEXT: movq %rsi, %r10
+; SSE-NEXT: cmovnsq %rax, %r10
+; SSE-NEXT: cmovnsq %r8, %rdx
+; SSE-NEXT: cmovnsq %rsi, %rax
+; SSE-NEXT: testb $64, %r9b
+; SSE-NEXT: movq %rax, %rsi
+; SSE-NEXT: cmoveq %rdx, %rsi
+; SSE-NEXT: cmoveq %r10, %rdx
+; SSE-NEXT: cmoveq %rcx, %r10
+; SSE-NEXT: cmovneq %rcx, %rax
+; SSE-NEXT: movq %rax, %r8
+; SSE-NEXT: movl %r9d, %ecx
+; SSE-NEXT: shrdq %cl, %r10, %r8
+; SSE-NEXT: shrdq %cl, %rdx, %r10
+; SSE-NEXT: shrdq %cl, %rsi, %rdx
+; SSE-NEXT: shrdq %cl, %rax, %rsi
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq %rsi, 24(%rdi)
+; SSE-NEXT: movq %rdx, 16(%rdi)
+; SSE-NEXT: movq %r10, 8(%rdi)
+; SSE-NEXT: movq %r8, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: fshr_rot_i256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rcx, %rax
+; AVX2-NEXT: testb %r9b, %r9b
+; AVX2-NEXT: movq %r8, %rcx
+; AVX2-NEXT: cmovnsq %rdx, %rcx
+; AVX2-NEXT: movq %rsi, %r10
+; AVX2-NEXT: cmovnsq %rax, %r10
+; AVX2-NEXT: cmovnsq %r8, %rdx
+; AVX2-NEXT: cmovnsq %rsi, %rax
+; AVX2-NEXT: testb $64, %r9b
+; AVX2-NEXT: movq %rax, %rsi
+; AVX2-NEXT: cmoveq %rdx, %rsi
+; AVX2-NEXT: cmoveq %r10, %rdx
+; AVX2-NEXT: cmoveq %rcx, %r10
+; AVX2-NEXT: cmovneq %rcx, %rax
+; AVX2-NEXT: movq %rax, %r8
+; AVX2-NEXT: movl %r9d, %ecx
+; AVX2-NEXT: shrdq %cl, %r10, %r8
+; AVX2-NEXT: shrdq %cl, %rdx, %r10
+; AVX2-NEXT: shrdq %cl, %rsi, %rdx
+; AVX2-NEXT: shrdq %cl, %rax, %rsi
+; AVX2-NEXT: movq %rsi, 24(%rdi)
+; AVX2-NEXT: movq %rdx, 16(%rdi)
+; AVX2-NEXT: movq %r10, 8(%rdi)
+; AVX2-NEXT: movq %r8, (%rdi)
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: fshr_rot_i256:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rcx, %rax
+; AVX512F-NEXT: testb %r9b, %r9b
+; AVX512F-NEXT: movq %r8, %rcx
+; AVX512F-NEXT: cmovnsq %rdx, %rcx
+; AVX512F-NEXT: movq %rsi, %r10
+; AVX512F-NEXT: cmovnsq %rax, %r10
+; AVX512F-NEXT: cmovnsq %r8, %rdx
+; AVX512F-NEXT: cmovnsq %rsi, %rax
+; AVX512F-NEXT: testb $64, %r9b
+; AVX512F-NEXT: movq %rax, %rsi
+; AVX512F-NEXT: cmoveq %rdx, %rsi
+; AVX512F-NEXT: cmoveq %r10, %rdx
+; AVX512F-NEXT: cmoveq %rcx, %r10
+; AVX512F-NEXT: cmovneq %rcx, %rax
+; AVX512F-NEXT: movq %rax, %r8
+; AVX512F-NEXT: movl %r9d, %ecx
+; AVX512F-NEXT: shrdq %cl, %r10, %r8
+; AVX512F-NEXT: shrdq %cl, %rdx, %r10
+; AVX512F-NEXT: shrdq %cl, %rsi, %rdx
+; AVX512F-NEXT: shrdq %cl, %rax, %rsi
+; AVX512F-NEXT: movq %rsi, 24(%rdi)
+; AVX512F-NEXT: movq %rdx, 16(%rdi)
+; AVX512F-NEXT: movq %r10, 8(%rdi)
+; AVX512F-NEXT: movq %r8, (%rdi)
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: fshr_rot_i256:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rcx, %rax
+; AVX512VL-NEXT: testb %r9b, %r9b
+; AVX512VL-NEXT: movq %r8, %rcx
+; AVX512VL-NEXT: cmovnsq %rdx, %rcx
+; AVX512VL-NEXT: movq %rsi, %r10
+; AVX512VL-NEXT: cmovnsq %rax, %r10
+; AVX512VL-NEXT: cmovnsq %r8, %rdx
+; AVX512VL-NEXT: cmovnsq %rsi, %rax
+; AVX512VL-NEXT: testb $64, %r9b
+; AVX512VL-NEXT: movq %rax, %rsi
+; AVX512VL-NEXT: cmoveq %rdx, %rsi
+; AVX512VL-NEXT: cmoveq %r10, %rdx
+; AVX512VL-NEXT: cmoveq %rcx, %r10
+; AVX512VL-NEXT: cmovneq %rcx, %rax
+; AVX512VL-NEXT: movq %rax, %r8
+; AVX512VL-NEXT: movl %r9d, %ecx
+; AVX512VL-NEXT: shrdq %cl, %r10, %r8
+; AVX512VL-NEXT: shrdq %cl, %rdx, %r10
+; AVX512VL-NEXT: shrdq %cl, %rsi, %rdx
+; AVX512VL-NEXT: shrdq %cl, %rax, %rsi
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movq %rsi, 24(%rdi)
+; AVX512VL-NEXT: movq %rdx, 16(%rdi)
+; AVX512VL-NEXT: movq %r10, 8(%rdi)
+; AVX512VL-NEXT: movq %r8, (%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: fshr_rot_i256:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rcx, %rax
+; AVX512VBMI-NEXT: testb %r9b, %r9b
+; AVX512VBMI-NEXT: movq %r8, %rcx
+; AVX512VBMI-NEXT: cmovnsq %rdx, %rcx
+; AVX512VBMI-NEXT: movq %rsi, %r10
+; AVX512VBMI-NEXT: cmovnsq %rax, %r10
+; AVX512VBMI-NEXT: cmovnsq %r8, %rdx
+; AVX512VBMI-NEXT: cmovnsq %rsi, %rax
+; AVX512VBMI-NEXT: testb $64, %r9b
+; AVX512VBMI-NEXT: movq %rax, %rsi
+; AVX512VBMI-NEXT: cmoveq %rdx, %rsi
+; AVX512VBMI-NEXT: cmoveq %r10, %rdx
+; AVX512VBMI-NEXT: cmoveq %rcx, %r10
+; AVX512VBMI-NEXT: cmovneq %rcx, %rax
+; AVX512VBMI-NEXT: movq %rax, %r8
+; AVX512VBMI-NEXT: movl %r9d, %ecx
+; AVX512VBMI-NEXT: shrdq %cl, %r10, %r8
+; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r10
+; AVX512VBMI-NEXT: shrdq %cl, %rsi, %rdx
+; AVX512VBMI-NEXT: shrdq %cl, %rax, %rsi
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movq %rsi, 24(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
+; AVX512VBMI-NEXT: movq %r10, 8(%rdi)
+; AVX512VBMI-NEXT: movq %r8, (%rdi)
+; AVX512VBMI-NEXT: retq
+ %r = call i256 @llvm.fshr.i256(i256 %a0, i256 %a0, i256 %a1)
+ ret i256 %r
+}
+
+define i256 @fshl_i256_load(ptr %p0, ptr %p1, i256 %a2) nounwind {
+; SSE-LABEL: fshl_i256_load:
+; SSE: # %bb.0:
+; SSE-NEXT: movq (%rsi), %rax
+; SSE-NEXT: movq 8(%rsi), %r8
+; SSE-NEXT: movq 24(%rdx), %r9
+; SSE-NEXT: testb $-128, %cl
+; SSE-NEXT: movq 8(%rdx), %r11
+; SSE-NEXT: cmoveq %r9, %r11
+; SSE-NEXT: movq 16(%rdx), %r10
+; SSE-NEXT: movq (%rdx), %rdx
+; SSE-NEXT: cmoveq %r10, %rdx
+; SSE-NEXT: cmoveq %rax, %r10
+; SSE-NEXT: cmoveq %r8, %r9
+; SSE-NEXT: cmoveq 16(%rsi), %rax
+; SSE-NEXT: cmoveq 24(%rsi), %r8
+; SSE-NEXT: testb $64, %cl
+; SSE-NEXT: cmovneq %rax, %r8
+; SSE-NEXT: cmovneq %r9, %rax
+; SSE-NEXT: cmovneq %r10, %r9
+; SSE-NEXT: cmoveq %r11, %rdx
+; SSE-NEXT: cmovneq %r11, %r10
+; SSE-NEXT: movq %r10, %rsi
+; SSE-NEXT: shldq %cl, %rdx, %rsi
+; SSE-NEXT: movq %r9, %rdx
+; SSE-NEXT: shldq %cl, %r10, %rdx
+; SSE-NEXT: movq %rax, %r10
+; SSE-NEXT: shldq %cl, %r9, %r10
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shldq %cl, %rax, %r8
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq %r8, 24(%rdi)
+; SSE-NEXT: movq %r10, 16(%rdi)
+; SSE-NEXT: movq %rdx, 8(%rdi)
+; SSE-NEXT: movq %rsi, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: fshl_i256_load:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq (%rsi), %r9
+; AVX2-NEXT: movq 8(%rsi), %r8
+; AVX2-NEXT: movq 24(%rdx), %r10
+; AVX2-NEXT: testb $-128, %cl
+; AVX2-NEXT: movq 8(%rdx), %rbx
+; AVX2-NEXT: cmoveq %r10, %rbx
+; AVX2-NEXT: movq 16(%rdx), %r11
+; AVX2-NEXT: movq (%rdx), %rdx
+; AVX2-NEXT: cmoveq %r11, %rdx
+; AVX2-NEXT: cmoveq %r9, %r11
+; AVX2-NEXT: cmoveq %r8, %r10
+; AVX2-NEXT: cmoveq 16(%rsi), %r9
+; AVX2-NEXT: cmoveq 24(%rsi), %r8
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: testb $64, %cl
+; AVX2-NEXT: cmovneq %r9, %r8
+; AVX2-NEXT: cmovneq %r10, %r9
+; AVX2-NEXT: cmovneq %r11, %r10
+; AVX2-NEXT: cmoveq %rbx, %rdx
+; AVX2-NEXT: cmovneq %rbx, %r11
+; AVX2-NEXT: movq %r11, %rsi
+; AVX2-NEXT: shldq %cl, %rdx, %rsi
+; AVX2-NEXT: movq %r10, %rdx
+; AVX2-NEXT: shldq %cl, %r11, %rdx
+; AVX2-NEXT: movq %r9, %rdi
+; AVX2-NEXT: shldq %cl, %r10, %rdi
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shldq %cl, %r9, %r8
+; AVX2-NEXT: movq %r8, 24(%rax)
+; AVX2-NEXT: movq %rdi, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %rsi, (%rax)
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: fshl_i256_load:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: pushq %rbx
+; AVX512F-NEXT: movq (%rsi), %r9
+; AVX512F-NEXT: movq 8(%rsi), %r8
+; AVX512F-NEXT: movq 24(%rdx), %r10
+; AVX512F-NEXT: testb $-128, %cl
+; AVX512F-NEXT: movq 8(%rdx), %rbx
+; AVX512F-NEXT: cmoveq %r10, %rbx
+; AVX512F-NEXT: movq 16(%rdx), %r11
+; AVX512F-NEXT: movq (%rdx), %rdx
+; AVX512F-NEXT: cmoveq %r11, %rdx
+; AVX512F-NEXT: cmoveq %r9, %r11
+; AVX512F-NEXT: cmoveq %r8, %r10
+; AVX512F-NEXT: cmoveq 16(%rsi), %r9
+; AVX512F-NEXT: cmoveq 24(%rsi), %r8
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: testb $64, %cl
+; AVX512F-NEXT: cmovneq %r9, %r8
+; AVX512F-NEXT: cmovneq %r10, %r9
+; AVX512F-NEXT: cmovneq %r11, %r10
+; AVX512F-NEXT: cmoveq %rbx, %rdx
+; AVX512F-NEXT: cmovneq %rbx, %r11
+; AVX512F-NEXT: movq %r11, %rsi
+; AVX512F-NEXT: shldq %cl, %rdx, %rsi
+; AVX512F-NEXT: movq %r10, %rdx
+; AVX512F-NEXT: shldq %cl, %r11, %rdx
+; AVX512F-NEXT: movq %r9, %rdi
+; AVX512F-NEXT: shldq %cl, %r10, %rdi
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shldq %cl, %r9, %r8
+; AVX512F-NEXT: movq %r8, 24(%rax)
+; AVX512F-NEXT: movq %rdi, 16(%rax)
+; AVX512F-NEXT: movq %rdx, 8(%rax)
+; AVX512F-NEXT: movq %rsi, (%rax)
+; AVX512F-NEXT: popq %rbx
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: fshl_i256_load:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq (%rsi), %rax
+; AVX512VL-NEXT: movq 8(%rsi), %r8
+; AVX512VL-NEXT: movq 16(%rdx), %r10
+; AVX512VL-NEXT: movq 24(%rdx), %r9
+; AVX512VL-NEXT: testb $-128, %cl
+; AVX512VL-NEXT: movq 8(%rdx), %r11
+; AVX512VL-NEXT: cmoveq %r9, %r11
+; AVX512VL-NEXT: movq (%rdx), %rdx
+; AVX512VL-NEXT: cmoveq %r10, %rdx
+; AVX512VL-NEXT: cmoveq %rax, %r10
+; AVX512VL-NEXT: cmoveq %r8, %r9
+; AVX512VL-NEXT: cmoveq 16(%rsi), %rax
+; AVX512VL-NEXT: cmoveq 24(%rsi), %r8
+; AVX512VL-NEXT: testb $64, %cl
+; AVX512VL-NEXT: cmovneq %rax, %r8
+; AVX512VL-NEXT: cmovneq %r9, %rax
+; AVX512VL-NEXT: cmovneq %r10, %r9
+; AVX512VL-NEXT: cmoveq %r11, %rdx
+; AVX512VL-NEXT: cmovneq %r11, %r10
+; AVX512VL-NEXT: movq %r10, %rsi
+; AVX512VL-NEXT: shldq %cl, %rdx, %rsi
+; AVX512VL-NEXT: movq %r9, %rdx
+; AVX512VL-NEXT: shldq %cl, %r10, %rdx
+; AVX512VL-NEXT: movq %rax, %r10
+; AVX512VL-NEXT: shldq %cl, %r9, %r10
+; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VL-NEXT: shldq %cl, %rax, %r8
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movq %r8, 24(%rdi)
+; AVX512VL-NEXT: movq %r10, 16(%rdi)
+; AVX512VL-NEXT: movq %rdx, 8(%rdi)
+; AVX512VL-NEXT: movq %rsi, (%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: fshl_i256_load:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq (%rsi), %rax
+; AVX512VBMI-NEXT: movq 8(%rsi), %r8
+; AVX512VBMI-NEXT: movq 16(%rdx), %r10
+; AVX512VBMI-NEXT: movq 24(%rdx), %r9
+; AVX512VBMI-NEXT: testb $-128, %cl
+; AVX512VBMI-NEXT: movq 8(%rdx), %r11
+; AVX512VBMI-NEXT: cmoveq %r9, %r11
+; AVX512VBMI-NEXT: movq (%rdx), %rdx
+; AVX512VBMI-NEXT: cmoveq %r10, %rdx
+; AVX512VBMI-NEXT: cmoveq %rax, %r10
+; AVX512VBMI-NEXT: cmoveq %r8, %r9
+; AVX512VBMI-NEXT: cmoveq 16(%rsi), %rax
+; AVX512VBMI-NEXT: cmoveq 24(%rsi), %r8
+; AVX512VBMI-NEXT: testb $64, %cl
+; AVX512VBMI-NEXT: cmovneq %rax, %r8
+; AVX512VBMI-NEXT: cmovneq %r9, %rax
+; AVX512VBMI-NEXT: cmovneq %r10, %r9
+; AVX512VBMI-NEXT: cmoveq %r11, %rdx
+; AVX512VBMI-NEXT: cmovneq %r11, %r10
+; AVX512VBMI-NEXT: movq %r10, %rsi
+; AVX512VBMI-NEXT: shldq %cl, %rdx, %rsi
+; AVX512VBMI-NEXT: movq %r9, %rdx
+; AVX512VBMI-NEXT: shldq %cl, %r10, %rdx
+; AVX512VBMI-NEXT: movq %rax, %r10
+; AVX512VBMI-NEXT: shldq %cl, %r9, %r10
+; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VBMI-NEXT: shldq %cl, %rax, %r8
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movq %r8, 24(%rdi)
+; AVX512VBMI-NEXT: movq %r10, 16(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, 8(%rdi)
+; AVX512VBMI-NEXT: movq %rsi, (%rdi)
+; AVX512VBMI-NEXT: retq
+ %a0 = load i256, ptr %p0
+ %a1 = load i256, ptr %p1
+ %r = call i256 @llvm.fshl.i256(i256 %a0, i256 %a1, i256 %a2)
+ ret i256 %r
+}
+
+define i256 @fshr_i256_load(ptr %p0, ptr %p1, i256 %a2) nounwind {
+; SSE-LABEL: fshr_i256_load:
+; SSE: # %bb.0:
+; SSE-NEXT: movq (%rsi), %r8
+; SSE-NEXT: movq 8(%rsi), %rax
+; SSE-NEXT: movq 16(%rdx), %r9
+; SSE-NEXT: testb %cl, %cl
+; SSE-NEXT: movq (%rdx), %r11
+; SSE-NEXT: cmovsq %r9, %r11
+; SSE-NEXT: movq 24(%rdx), %r10
+; SSE-NEXT: movq 8(%rdx), %rdx
+; SSE-NEXT: cmovsq %r10, %rdx
+; SSE-NEXT: cmovsq %r8, %r9
+; SSE-NEXT: cmovsq %rax, %r10
+; SSE-NEXT: cmovsq 16(%rsi), %r8
+; SSE-NEXT: cmovsq 24(%rsi), %rax
+; SSE-NEXT: testb $64, %cl
+; SSE-NEXT: cmoveq %r8, %rax
+; SSE-NEXT: cmoveq %r10, %r8
+; SSE-NEXT: cmoveq %r9, %r10
+; SSE-NEXT: cmoveq %rdx, %r9
+; SSE-NEXT: cmoveq %r11, %rdx
+; SSE-NEXT: shrdq %cl, %r9, %rdx
+; SSE-NEXT: shrdq %cl, %r10, %r9
+; SSE-NEXT: shrdq %cl, %r8, %r10
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shrdq %cl, %rax, %r8
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq %r8, 24(%rdi)
+; SSE-NEXT: movq %r10, 16(%rdi)
+; SSE-NEXT: movq %r9, 8(%rdi)
+; SSE-NEXT: movq %rdx, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: fshr_i256_load:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq (%rsi), %r8
+; AVX2-NEXT: movq 8(%rsi), %r10
+; AVX2-NEXT: movq 16(%rdx), %r9
+; AVX2-NEXT: testb %cl, %cl
+; AVX2-NEXT: movq (%rdx), %rbx
+; AVX2-NEXT: cmovsq %r9, %rbx
+; AVX2-NEXT: movq 24(%rdx), %r11
+; AVX2-NEXT: movq 8(%rdx), %rdx
+; AVX2-NEXT: cmovsq %r11, %rdx
+; AVX2-NEXT: cmovsq %r8, %r9
+; AVX2-NEXT: cmovsq %r10, %r11
+; AVX2-NEXT: cmovsq 16(%rsi), %r8
+; AVX2-NEXT: cmovsq 24(%rsi), %r10
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: testb $64, %cl
+; AVX2-NEXT: cmoveq %r8, %r10
+; AVX2-NEXT: cmoveq %r11, %r8
+; AVX2-NEXT: cmoveq %r9, %r11
+; AVX2-NEXT: cmoveq %rdx, %r9
+; AVX2-NEXT: cmoveq %rbx, %rdx
+; AVX2-NEXT: shrdq %cl, %r9, %rdx
+; AVX2-NEXT: shrdq %cl, %r11, %r9
+; AVX2-NEXT: shrdq %cl, %r8, %r11
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shrdq %cl, %r10, %r8
+; AVX2-NEXT: movq %r8, 24(%rdi)
+; AVX2-NEXT: movq %r11, 16(%rdi)
+; AVX2-NEXT: movq %r9, 8(%rdi)
+; AVX2-NEXT: movq %rdx, (%rdi)
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: fshr_i256_load:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: pushq %rbx
+; AVX512F-NEXT: movq (%rsi), %r8
+; AVX512F-NEXT: movq 8(%rsi), %r10
+; AVX512F-NEXT: movq 16(%rdx), %r9
+; AVX512F-NEXT: testb %cl, %cl
+; AVX512F-NEXT: movq (%rdx), %rbx
+; AVX512F-NEXT: cmovsq %r9, %rbx
+; AVX512F-NEXT: movq 24(%rdx), %r11
+; AVX512F-NEXT: movq 8(%rdx), %rdx
+; AVX512F-NEXT: cmovsq %r11, %rdx
+; AVX512F-NEXT: cmovsq %r8, %r9
+; AVX512F-NEXT: cmovsq %r10, %r11
+; AVX512F-NEXT: cmovsq 16(%rsi), %r8
+; AVX512F-NEXT: cmovsq 24(%rsi), %r10
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: testb $64, %cl
+; AVX512F-NEXT: cmoveq %r8, %r10
+; AVX512F-NEXT: cmoveq %r11, %r8
+; AVX512F-NEXT: cmoveq %r9, %r11
+; AVX512F-NEXT: cmoveq %rdx, %r9
+; AVX512F-NEXT: cmoveq %rbx, %rdx
+; AVX512F-NEXT: shrdq %cl, %r9, %rdx
+; AVX512F-NEXT: shrdq %cl, %r11, %r9
+; AVX512F-NEXT: shrdq %cl, %r8, %r11
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shrdq %cl, %r10, %r8
+; AVX512F-NEXT: movq %r8, 24(%rdi)
+; AVX512F-NEXT: movq %r11, 16(%rdi)
+; AVX512F-NEXT: movq %r9, 8(%rdi)
+; AVX512F-NEXT: movq %rdx, (%rdi)
+; AVX512F-NEXT: popq %rbx
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: fshr_i256_load:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq (%rsi), %r8
+; AVX512VL-NEXT: movq 8(%rsi), %rax
+; AVX512VL-NEXT: movq 16(%rdx), %r9
+; AVX512VL-NEXT: movq 24(%rdx), %r10
+; AVX512VL-NEXT: testb %cl, %cl
+; AVX512VL-NEXT: movq (%rdx), %r11
+; AVX512VL-NEXT: cmovsq %r9, %r11
+; AVX512VL-NEXT: movq 8(%rdx), %rdx
+; AVX512VL-NEXT: cmovsq %r10, %rdx
+; AVX512VL-NEXT: cmovsq %r8, %r9
+; AVX512VL-NEXT: cmovsq %rax, %r10
+; AVX512VL-NEXT: cmovsq 16(%rsi), %r8
+; AVX512VL-NEXT: cmovsq 24(%rsi), %rax
+; AVX512VL-NEXT: testb $64, %cl
+; AVX512VL-NEXT: cmoveq %r8, %rax
+; AVX512VL-NEXT: cmoveq %r10, %r8
+; AVX512VL-NEXT: cmoveq %r9, %r10
+; AVX512VL-NEXT: cmoveq %rdx, %r9
+; AVX512VL-NEXT: cmoveq %r11, %rdx
+; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
+; AVX512VL-NEXT: shrdq %cl, %r10, %r9
+; AVX512VL-NEXT: shrdq %cl, %r8, %r10
+; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VL-NEXT: shrdq %cl, %rax, %r8
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movq %r8, 24(%rdi)
+; AVX512VL-NEXT: movq %r10, 16(%rdi)
+; AVX512VL-NEXT: movq %r9, 8(%rdi)
+; AVX512VL-NEXT: movq %rdx, (%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: fshr_i256_load:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq (%rsi), %r8
+; AVX512VBMI-NEXT: movq 8(%rsi), %rax
+; AVX512VBMI-NEXT: movq 16(%rdx), %r9
+; AVX512VBMI-NEXT: movq 24(%rdx), %r10
+; AVX512VBMI-NEXT: testb %cl, %cl
+; AVX512VBMI-NEXT: movq (%rdx), %r11
+; AVX512VBMI-NEXT: cmovsq %r9, %r11
+; AVX512VBMI-NEXT: movq 8(%rdx), %rdx
+; AVX512VBMI-NEXT: cmovsq %r10, %rdx
+; AVX512VBMI-NEXT: cmovsq %r8, %r9
+; AVX512VBMI-NEXT: cmovsq %rax, %r10
+; AVX512VBMI-NEXT: cmovsq 16(%rsi), %r8
+; AVX512VBMI-NEXT: cmovsq 24(%rsi), %rax
+; AVX512VBMI-NEXT: testb $64, %cl
+; AVX512VBMI-NEXT: cmoveq %r8, %rax
+; AVX512VBMI-NEXT: cmoveq %r10, %r8
+; AVX512VBMI-NEXT: cmoveq %r9, %r10
+; AVX512VBMI-NEXT: cmoveq %rdx, %r9
+; AVX512VBMI-NEXT: cmoveq %r11, %rdx
+; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
+; AVX512VBMI-NEXT: shrdq %cl, %r10, %r9
+; AVX512VBMI-NEXT: shrdq %cl, %r8, %r10
+; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VBMI-NEXT: shrdq %cl, %rax, %r8
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movq %r8, 24(%rdi)
+; AVX512VBMI-NEXT: movq %r10, 16(%rdi)
+; AVX512VBMI-NEXT: movq %r9, 8(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, (%rdi)
+; AVX512VBMI-NEXT: retq
+ %a0 = load i256, ptr %p0
+ %a1 = load i256, ptr %p1
+ %r = call i256 @llvm.fshr.i256(i256 %a0, i256 %a1, i256 %a2)
+ ret i256 %r
+}
+
+define i256 @fshl_rot_i256_load(ptr %p0, i256 %a2) nounwind {
+; SSE-LABEL: fshl_rot_i256_load:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdx, %rcx
+; SSE-NEXT: movq 16(%rsi), %rdx
+; SSE-NEXT: movq 8(%rsi), %rax
+; SSE-NEXT: movq 24(%rsi), %r8
+; SSE-NEXT: testb $-128, %cl
+; SSE-NEXT: movq %r8, %r9
+; SSE-NEXT: cmovneq %rax, %r9
+; SSE-NEXT: movq (%rsi), %rsi
+; SSE-NEXT: movq %rsi, %r10
+; SSE-NEXT: cmovneq %rdx, %r10
+; SSE-NEXT: cmovneq %r8, %rax
+; SSE-NEXT: cmovneq %rsi, %rdx
+; SSE-NEXT: testb $64, %cl
+; SSE-NEXT: movq %rdx, %rsi
+; SSE-NEXT: cmovneq %rax, %rsi
+; SSE-NEXT: cmovneq %r10, %rax
+; SSE-NEXT: cmoveq %r9, %rdx
+; SSE-NEXT: cmovneq %r9, %r10
+; SSE-NEXT: movq %r10, %r8
+; SSE-NEXT: shldq %cl, %rdx, %r8
+; SSE-NEXT: movq %rax, %r9
+; SSE-NEXT: shldq %cl, %r10, %r9
+; SSE-NEXT: movq %rsi, %r10
+; SSE-NEXT: shldq %cl, %rax, %r10
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shldq %cl, %rsi, %rdx
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq %rdx, 24(%rdi)
+; SSE-NEXT: movq %r10, 16(%rdi)
+; SSE-NEXT: movq %r9, 8(%rdi)
+; SSE-NEXT: movq %r8, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: fshl_rot_i256_load:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: movq 16(%rsi), %rdx
+; AVX2-NEXT: movq (%rsi), %r9
+; AVX2-NEXT: movq 8(%rsi), %r8
+; AVX2-NEXT: movq 24(%rsi), %rax
+; AVX2-NEXT: testb $-128, %cl
+; AVX2-NEXT: movq %rax, %rsi
+; AVX2-NEXT: cmovneq %r8, %rsi
+; AVX2-NEXT: movq %r9, %r10
+; AVX2-NEXT: cmovneq %rdx, %r10
+; AVX2-NEXT: cmovneq %rax, %r8
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: cmovneq %r9, %rdx
+; AVX2-NEXT: testb $64, %cl
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: cmovneq %r8, %rdi
+; AVX2-NEXT: cmovneq %r10, %r8
+; AVX2-NEXT: cmoveq %rsi, %rdx
+; AVX2-NEXT: cmovneq %rsi, %r10
+; AVX2-NEXT: movq %r10, %rsi
+; AVX2-NEXT: shldq %cl, %rdx, %rsi
+; AVX2-NEXT: movq %r8, %r9
+; AVX2-NEXT: shldq %cl, %r10, %r9
+; AVX2-NEXT: movq %rdi, %r10
+; AVX2-NEXT: shldq %cl, %r8, %r10
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shldq %cl, %rdi, %rdx
+; AVX2-NEXT: movq %rdx, 24(%rax)
+; AVX2-NEXT: movq %r10, 16(%rax)
+; AVX2-NEXT: movq %r9, 8(%rax)
+; AVX2-NEXT: movq %rsi, (%rax)
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: fshl_rot_i256_load:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rdx, %rcx
+; AVX512F-NEXT: movq 16(%rsi), %rdx
+; AVX512F-NEXT: movq (%rsi), %r9
+; AVX512F-NEXT: movq 8(%rsi), %r8
+; AVX512F-NEXT: movq 24(%rsi), %rax
+; AVX512F-NEXT: testb $-128, %cl
+; AVX512F-NEXT: movq %rax, %rsi
+; AVX512F-NEXT: cmovneq %r8, %rsi
+; AVX512F-NEXT: movq %r9, %r10
+; AVX512F-NEXT: cmovneq %rdx, %r10
+; AVX512F-NEXT: cmovneq %rax, %r8
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: cmovneq %r9, %rdx
+; AVX512F-NEXT: testb $64, %cl
+; AVX512F-NEXT: movq %rdx, %rdi
+; AVX512F-NEXT: cmovneq %r8, %rdi
+; AVX512F-NEXT: cmovneq %r10, %r8
+; AVX512F-NEXT: cmoveq %rsi, %rdx
+; AVX512F-NEXT: cmovneq %rsi, %r10
+; AVX512F-NEXT: movq %r10, %rsi
+; AVX512F-NEXT: shldq %cl, %rdx, %rsi
+; AVX512F-NEXT: movq %r8, %r9
+; AVX512F-NEXT: shldq %cl, %r10, %r9
+; AVX512F-NEXT: movq %rdi, %r10
+; AVX512F-NEXT: shldq %cl, %r8, %r10
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shldq %cl, %rdi, %rdx
+; AVX512F-NEXT: movq %rdx, 24(%rax)
+; AVX512F-NEXT: movq %r10, 16(%rax)
+; AVX512F-NEXT: movq %r9, 8(%rax)
+; AVX512F-NEXT: movq %rsi, (%rax)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: fshl_rot_i256_load:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rdx, %rcx
+; AVX512VL-NEXT: movq 16(%rsi), %rdx
+; AVX512VL-NEXT: movq (%rsi), %r8
+; AVX512VL-NEXT: movq 8(%rsi), %rax
+; AVX512VL-NEXT: movq 24(%rsi), %rsi
+; AVX512VL-NEXT: testb $-128, %cl
+; AVX512VL-NEXT: movq %rsi, %r9
+; AVX512VL-NEXT: cmovneq %rax, %r9
+; AVX512VL-NEXT: movq %r8, %r10
+; AVX512VL-NEXT: cmovneq %rdx, %r10
+; AVX512VL-NEXT: cmovneq %rsi, %rax
+; AVX512VL-NEXT: cmovneq %r8, %rdx
+; AVX512VL-NEXT: testb $64, %cl
+; AVX512VL-NEXT: movq %rdx, %rsi
+; AVX512VL-NEXT: cmovneq %rax, %rsi
+; AVX512VL-NEXT: cmovneq %r10, %rax
+; AVX512VL-NEXT: cmoveq %r9, %rdx
+; AVX512VL-NEXT: cmovneq %r9, %r10
+; AVX512VL-NEXT: movq %r10, %r8
+; AVX512VL-NEXT: shldq %cl, %rdx, %r8
+; AVX512VL-NEXT: movq %rax, %r9
+; AVX512VL-NEXT: shldq %cl, %r10, %r9
+; AVX512VL-NEXT: movq %rsi, %r10
+; AVX512VL-NEXT: shldq %cl, %rax, %r10
+; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VL-NEXT: shldq %cl, %rsi, %rdx
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movq %rdx, 24(%rdi)
+; AVX512VL-NEXT: movq %r10, 16(%rdi)
+; AVX512VL-NEXT: movq %r9, 8(%rdi)
+; AVX512VL-NEXT: movq %r8, (%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: fshl_rot_i256_load:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rdx, %rcx
+; AVX512VBMI-NEXT: movq 16(%rsi), %rdx
+; AVX512VBMI-NEXT: movq (%rsi), %r8
+; AVX512VBMI-NEXT: movq 8(%rsi), %rax
+; AVX512VBMI-NEXT: movq 24(%rsi), %rsi
+; AVX512VBMI-NEXT: testb $-128, %cl
+; AVX512VBMI-NEXT: movq %rsi, %r9
+; AVX512VBMI-NEXT: cmovneq %rax, %r9
+; AVX512VBMI-NEXT: movq %r8, %r10
+; AVX512VBMI-NEXT: cmovneq %rdx, %r10
+; AVX512VBMI-NEXT: cmovneq %rsi, %rax
+; AVX512VBMI-NEXT: cmovneq %r8, %rdx
+; AVX512VBMI-NEXT: testb $64, %cl
+; AVX512VBMI-NEXT: movq %rdx, %rsi
+; AVX512VBMI-NEXT: cmovneq %rax, %rsi
+; AVX512VBMI-NEXT: cmovneq %r10, %rax
+; AVX512VBMI-NEXT: cmoveq %r9, %rdx
+; AVX512VBMI-NEXT: cmovneq %r9, %r10
+; AVX512VBMI-NEXT: movq %r10, %r8
+; AVX512VBMI-NEXT: shldq %cl, %rdx, %r8
+; AVX512VBMI-NEXT: movq %rax, %r9
+; AVX512VBMI-NEXT: shldq %cl, %r10, %r9
+; AVX512VBMI-NEXT: movq %rsi, %r10
+; AVX512VBMI-NEXT: shldq %cl, %rax, %r10
+; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VBMI-NEXT: shldq %cl, %rsi, %rdx
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movq %rdx, 24(%rdi)
+; AVX512VBMI-NEXT: movq %r10, 16(%rdi)
+; AVX512VBMI-NEXT: movq %r9, 8(%rdi)
+; AVX512VBMI-NEXT: movq %r8, (%rdi)
+; AVX512VBMI-NEXT: retq
+ %a0 = load i256, ptr %p0
+ %r = call i256 @llvm.fshl.i256(i256 %a0, i256 %a0, i256 %a2)
+ ret i256 %r
+}
+
+define i256 @fshr_rot_i256_load(ptr %p0, i256 %a2) nounwind {
+; SSE-LABEL: fshr_rot_i256_load:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdx, %rcx
+; SSE-NEXT: movq 16(%rsi), %rax
+; SSE-NEXT: movq 8(%rsi), %rdx
+; SSE-NEXT: movq 24(%rsi), %r8
+; SSE-NEXT: testb %cl, %cl
+; SSE-NEXT: movq %r8, %r9
+; SSE-NEXT: cmovnsq %rdx, %r9
+; SSE-NEXT: movq (%rsi), %r10
+; SSE-NEXT: movq %r10, %rsi
+; SSE-NEXT: cmovnsq %rax, %rsi
+; SSE-NEXT: cmovnsq %r8, %rdx
+; SSE-NEXT: cmovnsq %r10, %rax
+; SSE-NEXT: testb $64, %cl
+; SSE-NEXT: movq %rax, %r8
+; SSE-NEXT: cmoveq %rdx, %r8
+; SSE-NEXT: cmoveq %rsi, %rdx
+; SSE-NEXT: cmoveq %r9, %rsi
+; SSE-NEXT: cmovneq %r9, %rax
+; SSE-NEXT: movq %rax, %r9
+; SSE-NEXT: shrdq %cl, %rsi, %r9
+; SSE-NEXT: shrdq %cl, %rdx, %rsi
+; SSE-NEXT: shrdq %cl, %r8, %rdx
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shrdq %cl, %rax, %r8
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq %r8, 24(%rdi)
+; SSE-NEXT: movq %rdx, 16(%rdi)
+; SSE-NEXT: movq %rsi, 8(%rdi)
+; SSE-NEXT: movq %r9, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: fshr_rot_i256_load:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: movq 16(%rsi), %r8
+; AVX2-NEXT: movq (%rsi), %r9
+; AVX2-NEXT: movq 8(%rsi), %rdx
+; AVX2-NEXT: movq 24(%rsi), %rax
+; AVX2-NEXT: testb %cl, %cl
+; AVX2-NEXT: movq %rax, %r10
+; AVX2-NEXT: cmovnsq %rdx, %r10
+; AVX2-NEXT: movq %r9, %rsi
+; AVX2-NEXT: cmovnsq %r8, %rsi
+; AVX2-NEXT: cmovnsq %rax, %rdx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: cmovnsq %r9, %r8
+; AVX2-NEXT: testb $64, %cl
+; AVX2-NEXT: movq %r8, %rdi
+; AVX2-NEXT: cmoveq %rdx, %rdi
+; AVX2-NEXT: cmoveq %rsi, %rdx
+; AVX2-NEXT: cmoveq %r10, %rsi
+; AVX2-NEXT: cmovneq %r10, %r8
+; AVX2-NEXT: movq %r8, %r9
+; AVX2-NEXT: shrdq %cl, %rsi, %r9
+; AVX2-NEXT: shrdq %cl, %rdx, %rsi
+; AVX2-NEXT: shrdq %cl, %rdi, %rdx
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shrdq %cl, %r8, %rdi
+; AVX2-NEXT: movq %rdi, 24(%rax)
+; AVX2-NEXT: movq %rdx, 16(%rax)
+; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: movq %r9, (%rax)
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: fshr_rot_i256_load:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rdx, %rcx
+; AVX512F-NEXT: movq 16(%rsi), %r8
+; AVX512F-NEXT: movq (%rsi), %r9
+; AVX512F-NEXT: movq 8(%rsi), %rdx
+; AVX512F-NEXT: movq 24(%rsi), %rax
+; AVX512F-NEXT: testb %cl, %cl
+; AVX512F-NEXT: movq %rax, %r10
+; AVX512F-NEXT: cmovnsq %rdx, %r10
+; AVX512F-NEXT: movq %r9, %rsi
+; AVX512F-NEXT: cmovnsq %r8, %rsi
+; AVX512F-NEXT: cmovnsq %rax, %rdx
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: cmovnsq %r9, %r8
+; AVX512F-NEXT: testb $64, %cl
+; AVX512F-NEXT: movq %r8, %rdi
+; AVX512F-NEXT: cmoveq %rdx, %rdi
+; AVX512F-NEXT: cmoveq %rsi, %rdx
+; AVX512F-NEXT: cmoveq %r10, %rsi
+; AVX512F-NEXT: cmovneq %r10, %r8
+; AVX512F-NEXT: movq %r8, %r9
+; AVX512F-NEXT: shrdq %cl, %rsi, %r9
+; AVX512F-NEXT: shrdq %cl, %rdx, %rsi
+; AVX512F-NEXT: shrdq %cl, %rdi, %rdx
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shrdq %cl, %r8, %rdi
+; AVX512F-NEXT: movq %rdi, 24(%rax)
+; AVX512F-NEXT: movq %rdx, 16(%rax)
+; AVX512F-NEXT: movq %rsi, 8(%rax)
+; AVX512F-NEXT: movq %r9, (%rax)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: fshr_rot_i256_load:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rdx, %rcx
+; AVX512VL-NEXT: movq 16(%rsi), %rax
+; AVX512VL-NEXT: movq (%rsi), %r8
+; AVX512VL-NEXT: movq 8(%rsi), %rdx
+; AVX512VL-NEXT: movq 24(%rsi), %r9
+; AVX512VL-NEXT: testb %cl, %cl
+; AVX512VL-NEXT: movq %r9, %r10
+; AVX512VL-NEXT: cmovnsq %rdx, %r10
+; AVX512VL-NEXT: movq %r8, %rsi
+; AVX512VL-NEXT: cmovnsq %rax, %rsi
+; AVX512VL-NEXT: cmovnsq %r9, %rdx
+; AVX512VL-NEXT: cmovnsq %r8, %rax
+; AVX512VL-NEXT: testb $64, %cl
+; AVX512VL-NEXT: movq %rax, %r8
+; AVX512VL-NEXT: cmoveq %rdx, %r8
+; AVX512VL-NEXT: cmoveq %rsi, %rdx
+; AVX512VL-NEXT: cmoveq %r10, %rsi
+; AVX512VL-NEXT: cmovneq %r10, %rax
+; AVX512VL-NEXT: movq %rax, %r9
+; AVX512VL-NEXT: shrdq %cl, %rsi, %r9
+; AVX512VL-NEXT: shrdq %cl, %rdx, %rsi
+; AVX512VL-NEXT: shrdq %cl, %r8, %rdx
+; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VL-NEXT: shrdq %cl, %rax, %r8
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movq %r8, 24(%rdi)
+; AVX512VL-NEXT: movq %rdx, 16(%rdi)
+; AVX512VL-NEXT: movq %rsi, 8(%rdi)
+; AVX512VL-NEXT: movq %r9, (%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: fshr_rot_i256_load:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rdx, %rcx
+; AVX512VBMI-NEXT: movq 16(%rsi), %rax
+; AVX512VBMI-NEXT: movq (%rsi), %r8
+; AVX512VBMI-NEXT: movq 8(%rsi), %rdx
+; AVX512VBMI-NEXT: movq 24(%rsi), %r9
+; AVX512VBMI-NEXT: testb %cl, %cl
+; AVX512VBMI-NEXT: movq %r9, %r10
+; AVX512VBMI-NEXT: cmovnsq %rdx, %r10
+; AVX512VBMI-NEXT: movq %r8, %rsi
+; AVX512VBMI-NEXT: cmovnsq %rax, %rsi
+; AVX512VBMI-NEXT: cmovnsq %r9, %rdx
+; AVX512VBMI-NEXT: cmovnsq %r8, %rax
+; AVX512VBMI-NEXT: testb $64, %cl
+; AVX512VBMI-NEXT: movq %rax, %r8
+; AVX512VBMI-NEXT: cmoveq %rdx, %r8
+; AVX512VBMI-NEXT: cmoveq %rsi, %rdx
+; AVX512VBMI-NEXT: cmoveq %r10, %rsi
+; AVX512VBMI-NEXT: cmovneq %r10, %rax
+; AVX512VBMI-NEXT: movq %rax, %r9
+; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r9
+; AVX512VBMI-NEXT: shrdq %cl, %rdx, %rsi
+; AVX512VBMI-NEXT: shrdq %cl, %r8, %rdx
+; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VBMI-NEXT: shrdq %cl, %rax, %r8
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movq %r8, 24(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
+; AVX512VBMI-NEXT: movq %rsi, 8(%rdi)
+; AVX512VBMI-NEXT: movq %r9, (%rdi)
+; AVX512VBMI-NEXT: retq
+ %a0 = load i256, ptr %p0
+ %r = call i256 @llvm.fshr.i256(i256 %a0, i256 %a0, i256 %a2)
+ ret i256 %r
+}
+
+define i256 @fshl_i256_vector(<4 x i64> %v0, <4 x i64> %v1, i256 %a2) nounwind {
+; SSE2-LABEL: fshl_i256_vector:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movq %rsi, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %r10
+; SSE2-NEXT: movq %xmm0, %r9
+; SSE2-NEXT: movq %xmm2, %r8
+; SSE2-NEXT: movq %xmm3, %rsi
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %r11
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rdx
+; SSE2-NEXT: testb $-128, %cl
+; SSE2-NEXT: cmoveq %rdx, %r11
+; SSE2-NEXT: cmoveq %rsi, %r8
+; SSE2-NEXT: cmoveq %r9, %rsi
+; SSE2-NEXT: cmoveq %r10, %rdx
+; SSE2-NEXT: cmovneq %r9, %rax
+; SSE2-NEXT: movq %xmm4, %r9
+; SSE2-NEXT: cmovneq %r10, %r9
+; SSE2-NEXT: testb $64, %cl
+; SSE2-NEXT: cmovneq %rax, %r9
+; SSE2-NEXT: cmovneq %rdx, %rax
+; SSE2-NEXT: cmovneq %rsi, %rdx
+; SSE2-NEXT: cmoveq %r11, %r8
+; SSE2-NEXT: cmovneq %r11, %rsi
+; SSE2-NEXT: movq %rsi, %r10
+; SSE2-NEXT: shldq %cl, %r8, %r10
+; SSE2-NEXT: movq %rdx, %r8
+; SSE2-NEXT: shldq %cl, %rsi, %r8
+; SSE2-NEXT: movq %rax, %rsi
+; SSE2-NEXT: shldq %cl, %rdx, %rsi
+; SSE2-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE2-NEXT: shldq %cl, %rax, %r9
+; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: movq %r9, 24(%rdi)
+; SSE2-NEXT: movq %rsi, 16(%rdi)
+; SSE2-NEXT: movq %r8, 8(%rdi)
+; SSE2-NEXT: movq %r10, (%rdi)
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: fshl_i256_vector:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pushq %rbx
+; SSE42-NEXT: movq %rsi, %rcx
+; SSE42-NEXT: pextrq $1, %xmm1, %rdx
+; SSE42-NEXT: movq %xmm1, %rax
+; SSE42-NEXT: pextrq $1, %xmm0, %r10
+; SSE42-NEXT: movq %xmm0, %r11
+; SSE42-NEXT: movq %xmm2, %r8
+; SSE42-NEXT: pextrq $1, %xmm2, %rbx
+; SSE42-NEXT: pextrq $1, %xmm3, %rsi
+; SSE42-NEXT: movq %xmm3, %r9
+; SSE42-NEXT: testb $-128, %cl
+; SSE42-NEXT: cmoveq %rsi, %rbx
+; SSE42-NEXT: cmoveq %r9, %r8
+; SSE42-NEXT: cmoveq %r11, %r9
+; SSE42-NEXT: cmoveq %r10, %rsi
+; SSE42-NEXT: cmovneq %r11, %rax
+; SSE42-NEXT: cmovneq %r10, %rdx
+; SSE42-NEXT: testb $64, %cl
+; SSE42-NEXT: cmovneq %rax, %rdx
+; SSE42-NEXT: cmovneq %rsi, %rax
+; SSE42-NEXT: cmovneq %r9, %rsi
+; SSE42-NEXT: cmoveq %rbx, %r8
+; SSE42-NEXT: cmovneq %rbx, %r9
+; SSE42-NEXT: movq %r9, %r10
+; SSE42-NEXT: shldq %cl, %r8, %r10
+; SSE42-NEXT: movq %rsi, %r8
+; SSE42-NEXT: shldq %cl, %r9, %r8
+; SSE42-NEXT: movq %rax, %r9
+; SSE42-NEXT: shldq %cl, %rsi, %r9
+; SSE42-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE42-NEXT: shldq %cl, %rax, %rdx
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: movq %rdx, 24(%rdi)
+; SSE42-NEXT: movq %r9, 16(%rdi)
+; SSE42-NEXT: movq %r8, 8(%rdi)
+; SSE42-NEXT: movq %r10, (%rdi)
+; SSE42-NEXT: popq %rbx
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: fshl_i256_vector:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpextrq $1, %xmm2, %rdx
+; AVX2-NEXT: vmovq %xmm2, %rsi
+; AVX2-NEXT: vpextrq $1, %xmm0, %rax
+; AVX2-NEXT: vmovq %xmm0, %r11
+; AVX2-NEXT: vmovq %xmm1, %r9
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT: vpextrq $1, %xmm1, %rbx
+; AVX2-NEXT: vmovq %xmm0, %r10
+; AVX2-NEXT: vpextrq $1, %xmm0, %r8
+; AVX2-NEXT: testb $-128, %cl
+; AVX2-NEXT: cmoveq %r8, %rbx
+; AVX2-NEXT: cmoveq %r10, %r9
+; AVX2-NEXT: cmoveq %r11, %r10
+; AVX2-NEXT: cmoveq %rax, %r8
+; AVX2-NEXT: cmovneq %r11, %rsi
+; AVX2-NEXT: cmovneq %rax, %rdx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: testb $64, %cl
+; AVX2-NEXT: cmovneq %rsi, %rdx
+; AVX2-NEXT: cmovneq %r8, %rsi
+; AVX2-NEXT: cmovneq %r10, %r8
+; AVX2-NEXT: cmoveq %rbx, %r9
+; AVX2-NEXT: cmovneq %rbx, %r10
+; AVX2-NEXT: movq %r10, %rdi
+; AVX2-NEXT: shldq %cl, %r9, %rdi
+; AVX2-NEXT: movq %r8, %r9
+; AVX2-NEXT: shldq %cl, %r10, %r9
+; AVX2-NEXT: movq %rsi, %r10
+; AVX2-NEXT: shldq %cl, %r8, %r10
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shldq %cl, %rsi, %rdx
+; AVX2-NEXT: movq %rdx, 24(%rax)
+; AVX2-NEXT: movq %r10, 16(%rax)
+; AVX2-NEXT: movq %r9, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: fshl_i256_vector:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: pushq %rbx
+; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512F-NEXT: vpextrq $1, %xmm2, %rdx
+; AVX512F-NEXT: vmovq %xmm2, %rsi
+; AVX512F-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512F-NEXT: vmovq %xmm0, %r11
+; AVX512F-NEXT: vmovq %xmm1, %r9
+; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rbx
+; AVX512F-NEXT: vmovq %xmm0, %r10
+; AVX512F-NEXT: vpextrq $1, %xmm0, %r8
+; AVX512F-NEXT: testb $-128, %cl
+; AVX512F-NEXT: cmoveq %r8, %rbx
+; AVX512F-NEXT: cmoveq %r10, %r9
+; AVX512F-NEXT: cmoveq %r11, %r10
+; AVX512F-NEXT: cmoveq %rax, %r8
+; AVX512F-NEXT: cmovneq %r11, %rsi
+; AVX512F-NEXT: cmovneq %rax, %rdx
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: testb $64, %cl
+; AVX512F-NEXT: cmovneq %rsi, %rdx
+; AVX512F-NEXT: cmovneq %r8, %rsi
+; AVX512F-NEXT: cmovneq %r10, %r8
+; AVX512F-NEXT: cmoveq %rbx, %r9
+; AVX512F-NEXT: cmovneq %rbx, %r10
+; AVX512F-NEXT: movq %r10, %rdi
+; AVX512F-NEXT: shldq %cl, %r9, %rdi
+; AVX512F-NEXT: movq %r8, %r9
+; AVX512F-NEXT: shldq %cl, %r10, %r9
+; AVX512F-NEXT: movq %rsi, %r10
+; AVX512F-NEXT: shldq %cl, %r8, %r10
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shldq %cl, %rsi, %rdx
+; AVX512F-NEXT: movq %rdx, 24(%rax)
+; AVX512F-NEXT: movq %r10, 16(%rax)
+; AVX512F-NEXT: movq %r9, 8(%rax)
+; AVX512F-NEXT: movq %rdi, (%rax)
+; AVX512F-NEXT: popq %rbx
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: fshl_i256_vector:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: pushq %rbx
+; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512VL-NEXT: vpextrq $1, %xmm2, %rdx
+; AVX512VL-NEXT: vmovq %xmm2, %rax
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512VL-NEXT: vmovq %xmm0, %r11
+; AVX512VL-NEXT: vmovq %xmm1, %r8
+; AVX512VL-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %rbx
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %rsi
+; AVX512VL-NEXT: vmovq %xmm0, %r9
+; AVX512VL-NEXT: testb $-128, %cl
+; AVX512VL-NEXT: cmoveq %rsi, %rbx
+; AVX512VL-NEXT: cmoveq %r9, %r8
+; AVX512VL-NEXT: cmoveq %r11, %r9
+; AVX512VL-NEXT: cmoveq %r10, %rsi
+; AVX512VL-NEXT: cmovneq %r11, %rax
+; AVX512VL-NEXT: cmovneq %r10, %rdx
+; AVX512VL-NEXT: testb $64, %cl
+; AVX512VL-NEXT: cmovneq %rax, %rdx
+; AVX512VL-NEXT: cmovneq %rsi, %rax
+; AVX512VL-NEXT: cmovneq %r9, %rsi
+; AVX512VL-NEXT: cmoveq %rbx, %r8
+; AVX512VL-NEXT: cmovneq %rbx, %r9
+; AVX512VL-NEXT: movq %r9, %r10
+; AVX512VL-NEXT: shldq %cl, %r8, %r10
+; AVX512VL-NEXT: movq %rsi, %r8
+; AVX512VL-NEXT: shldq %cl, %r9, %r8
+; AVX512VL-NEXT: movq %rax, %r9
+; AVX512VL-NEXT: shldq %cl, %rsi, %r9
+; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VL-NEXT: shldq %cl, %rax, %rdx
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movq %rdx, 24(%rdi)
+; AVX512VL-NEXT: movq %r9, 16(%rdi)
+; AVX512VL-NEXT: movq %r8, 8(%rdi)
+; AVX512VL-NEXT: movq %r10, (%rdi)
+; AVX512VL-NEXT: popq %rbx
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: fshl_i256_vector:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: pushq %rbx
+; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512VBMI-NEXT: vpextrq $1, %xmm2, %rdx
+; AVX512VBMI-NEXT: vmovq %xmm2, %rax
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512VBMI-NEXT: vmovq %xmm0, %r11
+; AVX512VBMI-NEXT: vmovq %xmm1, %r8
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rbx
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rsi
+; AVX512VBMI-NEXT: vmovq %xmm0, %r9
+; AVX512VBMI-NEXT: testb $-128, %cl
+; AVX512VBMI-NEXT: cmoveq %rsi, %rbx
+; AVX512VBMI-NEXT: cmoveq %r9, %r8
+; AVX512VBMI-NEXT: cmoveq %r11, %r9
+; AVX512VBMI-NEXT: cmoveq %r10, %rsi
+; AVX512VBMI-NEXT: cmovneq %r11, %rax
+; AVX512VBMI-NEXT: cmovneq %r10, %rdx
+; AVX512VBMI-NEXT: testb $64, %cl
+; AVX512VBMI-NEXT: cmovneq %rax, %rdx
+; AVX512VBMI-NEXT: cmovneq %rsi, %rax
+; AVX512VBMI-NEXT: cmovneq %r9, %rsi
+; AVX512VBMI-NEXT: cmoveq %rbx, %r8
+; AVX512VBMI-NEXT: cmovneq %rbx, %r9
+; AVX512VBMI-NEXT: movq %r9, %r10
+; AVX512VBMI-NEXT: shldq %cl, %r8, %r10
+; AVX512VBMI-NEXT: movq %rsi, %r8
+; AVX512VBMI-NEXT: shldq %cl, %r9, %r8
+; AVX512VBMI-NEXT: movq %rax, %r9
+; AVX512VBMI-NEXT: shldq %cl, %rsi, %r9
+; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VBMI-NEXT: shldq %cl, %rax, %rdx
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movq %rdx, 24(%rdi)
+; AVX512VBMI-NEXT: movq %r9, 16(%rdi)
+; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
+; AVX512VBMI-NEXT: movq %r10, (%rdi)
+; AVX512VBMI-NEXT: popq %rbx
+; AVX512VBMI-NEXT: vzeroupper
+; AVX512VBMI-NEXT: retq
+ %a0 = bitcast <4 x i64> %v0 to i256
+ %a1 = bitcast <4 x i64> %v1 to i256
+ %r = call i256 @llvm.fshl.i256(i256 %a0, i256 %a1, i256 %a2)
+ ret i256 %r
+}
+
+define i256 @fshr_i256_vector(<4 x i64> %v0, <4 x i64> %v1, i256 %a2) nounwind {
+; SSE2-LABEL: fshr_i256_vector:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movq %rsi, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rdx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: movq %xmm0, %r10
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rsi
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %r8
+; SSE2-NEXT: movq %xmm2, %r11
+; SSE2-NEXT: movq %xmm3, %r9
+; SSE2-NEXT: testb %cl, %cl
+; SSE2-NEXT: cmovsq %r9, %r11
+; SSE2-NEXT: cmovsq %r8, %rsi
+; SSE2-NEXT: cmovsq %r10, %r9
+; SSE2-NEXT: cmovsq %rax, %r8
+; SSE2-NEXT: cmovnsq %r10, %rdx
+; SSE2-NEXT: movq %xmm4, %r10
+; SSE2-NEXT: cmovnsq %rax, %r10
+; SSE2-NEXT: testb $64, %cl
+; SSE2-NEXT: cmoveq %rdx, %r10
+; SSE2-NEXT: cmoveq %r8, %rdx
+; SSE2-NEXT: cmoveq %r9, %r8
+; SSE2-NEXT: cmoveq %rsi, %r9
+; SSE2-NEXT: cmoveq %r11, %rsi
+; SSE2-NEXT: shrdq %cl, %r9, %rsi
+; SSE2-NEXT: shrdq %cl, %r8, %r9
+; SSE2-NEXT: shrdq %cl, %rdx, %r8
+; SSE2-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE2-NEXT: shrdq %cl, %r10, %rdx
+; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: movq %rdx, 24(%rdi)
+; SSE2-NEXT: movq %r8, 16(%rdi)
+; SSE2-NEXT: movq %r9, 8(%rdi)
+; SSE2-NEXT: movq %rsi, (%rdi)
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: fshr_i256_vector:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pushq %rbx
+; SSE42-NEXT: movq %rsi, %rcx
+; SSE42-NEXT: pextrq $1, %xmm1, %rax
+; SSE42-NEXT: movq %xmm1, %rdx
+; SSE42-NEXT: pextrq $1, %xmm0, %r10
+; SSE42-NEXT: pextrq $1, %xmm2, %rsi
+; SSE42-NEXT: movq %xmm0, %r11
+; SSE42-NEXT: pextrq $1, %xmm3, %r8
+; SSE42-NEXT: movq %xmm2, %rbx
+; SSE42-NEXT: movq %xmm3, %r9
+; SSE42-NEXT: testb %cl, %cl
+; SSE42-NEXT: cmovsq %r9, %rbx
+; SSE42-NEXT: cmovsq %r8, %rsi
+; SSE42-NEXT: cmovsq %r11, %r9
+; SSE42-NEXT: cmovsq %r10, %r8
+; SSE42-NEXT: cmovnsq %r11, %rdx
+; SSE42-NEXT: cmovnsq %r10, %rax
+; SSE42-NEXT: testb $64, %cl
+; SSE42-NEXT: cmoveq %rdx, %rax
+; SSE42-NEXT: cmoveq %r8, %rdx
+; SSE42-NEXT: cmoveq %r9, %r8
+; SSE42-NEXT: cmoveq %rsi, %r9
+; SSE42-NEXT: cmoveq %rbx, %rsi
+; SSE42-NEXT: shrdq %cl, %r9, %rsi
+; SSE42-NEXT: shrdq %cl, %r8, %r9
+; SSE42-NEXT: shrdq %cl, %rdx, %r8
+; SSE42-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE42-NEXT: shrdq %cl, %rax, %rdx
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: movq %rdx, 24(%rdi)
+; SSE42-NEXT: movq %r8, 16(%rdi)
+; SSE42-NEXT: movq %r9, 8(%rdi)
+; SSE42-NEXT: movq %rsi, (%rdi)
+; SSE42-NEXT: popq %rbx
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: fshr_i256_vector:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpextrq $1, %xmm2, %rdx
+; AVX2-NEXT: vpextrq $1, %xmm0, %rax
+; AVX2-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX2-NEXT: vmovq %xmm2, %r8
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT: vpextrq $1, %xmm2, %r9
+; AVX2-NEXT: vmovq %xmm0, %r11
+; AVX2-NEXT: vmovq %xmm1, %rbx
+; AVX2-NEXT: vmovq %xmm2, %r10
+; AVX2-NEXT: testb %cl, %cl
+; AVX2-NEXT: cmovsq %r10, %rbx
+; AVX2-NEXT: cmovsq %r9, %rsi
+; AVX2-NEXT: cmovsq %r11, %r10
+; AVX2-NEXT: cmovsq %rax, %r9
+; AVX2-NEXT: cmovnsq %r11, %r8
+; AVX2-NEXT: cmovnsq %rax, %rdx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: testb $64, %cl
+; AVX2-NEXT: cmoveq %r8, %rdx
+; AVX2-NEXT: cmoveq %r9, %r8
+; AVX2-NEXT: cmoveq %r10, %r9
+; AVX2-NEXT: cmoveq %rsi, %r10
+; AVX2-NEXT: cmoveq %rbx, %rsi
+; AVX2-NEXT: shrdq %cl, %r10, %rsi
+; AVX2-NEXT: shrdq %cl, %r9, %r10
+; AVX2-NEXT: shrdq %cl, %r8, %r9
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shrdq %cl, %rdx, %r8
+; AVX2-NEXT: movq %r8, 24(%rdi)
+; AVX2-NEXT: movq %r9, 16(%rdi)
+; AVX2-NEXT: movq %r10, 8(%rdi)
+; AVX2-NEXT: movq %rsi, (%rdi)
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: fshr_i256_vector:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: pushq %rbx
+; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512F-NEXT: vpextrq $1, %xmm2, %rdx
+; AVX512F-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512F-NEXT: vmovq %xmm2, %r8
+; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX512F-NEXT: vpextrq $1, %xmm2, %r9
+; AVX512F-NEXT: vmovq %xmm0, %r11
+; AVX512F-NEXT: vmovq %xmm1, %rbx
+; AVX512F-NEXT: vmovq %xmm2, %r10
+; AVX512F-NEXT: testb %cl, %cl
+; AVX512F-NEXT: cmovsq %r10, %rbx
+; AVX512F-NEXT: cmovsq %r9, %rsi
+; AVX512F-NEXT: cmovsq %r11, %r10
+; AVX512F-NEXT: cmovsq %rax, %r9
+; AVX512F-NEXT: cmovnsq %r11, %r8
+; AVX512F-NEXT: cmovnsq %rax, %rdx
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: testb $64, %cl
+; AVX512F-NEXT: cmoveq %r8, %rdx
+; AVX512F-NEXT: cmoveq %r9, %r8
+; AVX512F-NEXT: cmoveq %r10, %r9
+; AVX512F-NEXT: cmoveq %rsi, %r10
+; AVX512F-NEXT: cmoveq %rbx, %rsi
+; AVX512F-NEXT: shrdq %cl, %r10, %rsi
+; AVX512F-NEXT: shrdq %cl, %r9, %r10
+; AVX512F-NEXT: shrdq %cl, %r8, %r9
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shrdq %cl, %rdx, %r8
+; AVX512F-NEXT: movq %r8, 24(%rdi)
+; AVX512F-NEXT: movq %r9, 16(%rdi)
+; AVX512F-NEXT: movq %r10, 8(%rdi)
+; AVX512F-NEXT: movq %rsi, (%rdi)
+; AVX512F-NEXT: popq %rbx
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: fshr_i256_vector:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: pushq %rbx
+; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512VL-NEXT: vpextrq $1, %xmm2, %rax
+; AVX512VL-NEXT: vmovq %xmm2, %rdx
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512VL-NEXT: vmovq %xmm0, %r11
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512VL-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %r8
+; AVX512VL-NEXT: vmovq %xmm1, %rbx
+; AVX512VL-NEXT: vmovq %xmm0, %r9
+; AVX512VL-NEXT: testb %cl, %cl
+; AVX512VL-NEXT: cmovsq %r9, %rbx
+; AVX512VL-NEXT: cmovsq %r8, %rsi
+; AVX512VL-NEXT: cmovsq %r11, %r9
+; AVX512VL-NEXT: cmovsq %r10, %r8
+; AVX512VL-NEXT: cmovnsq %r11, %rdx
+; AVX512VL-NEXT: cmovnsq %r10, %rax
+; AVX512VL-NEXT: testb $64, %cl
+; AVX512VL-NEXT: cmoveq %rdx, %rax
+; AVX512VL-NEXT: cmoveq %r8, %rdx
+; AVX512VL-NEXT: cmoveq %r9, %r8
+; AVX512VL-NEXT: cmoveq %rsi, %r9
+; AVX512VL-NEXT: cmoveq %rbx, %rsi
+; AVX512VL-NEXT: shrdq %cl, %r9, %rsi
+; AVX512VL-NEXT: shrdq %cl, %r8, %r9
+; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VL-NEXT: shrdq %cl, %rax, %rdx
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movq %rdx, 24(%rdi)
+; AVX512VL-NEXT: movq %r8, 16(%rdi)
+; AVX512VL-NEXT: movq %r9, 8(%rdi)
+; AVX512VL-NEXT: movq %rsi, (%rdi)
+; AVX512VL-NEXT: popq %rbx
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: fshr_i256_vector:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: pushq %rbx
+; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512VBMI-NEXT: vpextrq $1, %xmm2, %rax
+; AVX512VBMI-NEXT: vmovq %xmm2, %rdx
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512VBMI-NEXT: vmovq %xmm0, %r11
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r8
+; AVX512VBMI-NEXT: vmovq %xmm1, %rbx
+; AVX512VBMI-NEXT: vmovq %xmm0, %r9
+; AVX512VBMI-NEXT: testb %cl, %cl
+; AVX512VBMI-NEXT: cmovsq %r9, %rbx
+; AVX512VBMI-NEXT: cmovsq %r8, %rsi
+; AVX512VBMI-NEXT: cmovsq %r11, %r9
+; AVX512VBMI-NEXT: cmovsq %r10, %r8
+; AVX512VBMI-NEXT: cmovnsq %r11, %rdx
+; AVX512VBMI-NEXT: cmovnsq %r10, %rax
+; AVX512VBMI-NEXT: testb $64, %cl
+; AVX512VBMI-NEXT: cmoveq %rdx, %rax
+; AVX512VBMI-NEXT: cmoveq %r8, %rdx
+; AVX512VBMI-NEXT: cmoveq %r9, %r8
+; AVX512VBMI-NEXT: cmoveq %rsi, %r9
+; AVX512VBMI-NEXT: cmoveq %rbx, %rsi
+; AVX512VBMI-NEXT: shrdq %cl, %r9, %rsi
+; AVX512VBMI-NEXT: shrdq %cl, %r8, %r9
+; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VBMI-NEXT: shrdq %cl, %rax, %rdx
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movq %rdx, 24(%rdi)
+; AVX512VBMI-NEXT: movq %r8, 16(%rdi)
+; AVX512VBMI-NEXT: movq %r9, 8(%rdi)
+; AVX512VBMI-NEXT: movq %rsi, (%rdi)
+; AVX512VBMI-NEXT: popq %rbx
+; AVX512VBMI-NEXT: vzeroupper
+; AVX512VBMI-NEXT: retq
+ %a0 = bitcast <4 x i64> %v0 to i256
+ %a1 = bitcast <4 x i64> %v1 to i256
+ %r = call i256 @llvm.fshr.i256(i256 %a0, i256 %a1, i256 %a2)
+ ret i256 %r
+}
+
+define i256 @fshl_rot_i256_vector(<4 x i64> %v0, i256 %a2) nounwind {
+; SSE2-LABEL: fshl_rot_i256_vector:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movq %rsi, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm2, %rax
+; SSE2-NEXT: movq %xmm1, %rdx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rsi
+; SSE2-NEXT: testb $-128, %cl
+; SSE2-NEXT: movq %rsi, %r8
+; SSE2-NEXT: cmovneq %rax, %r8
+; SSE2-NEXT: movq %xmm0, %r9
+; SSE2-NEXT: movq %r9, %r10
+; SSE2-NEXT: cmovneq %rdx, %r10
+; SSE2-NEXT: cmovneq %rsi, %rax
+; SSE2-NEXT: cmovneq %r9, %rdx
+; SSE2-NEXT: testb $64, %cl
+; SSE2-NEXT: movq %rdx, %rsi
+; SSE2-NEXT: cmovneq %rax, %rsi
+; SSE2-NEXT: cmovneq %r10, %rax
+; SSE2-NEXT: cmoveq %r8, %rdx
+; SSE2-NEXT: cmovneq %r8, %r10
+; SSE2-NEXT: movq %r10, %r8
+; SSE2-NEXT: shldq %cl, %rdx, %r8
+; SSE2-NEXT: movq %rax, %r9
+; SSE2-NEXT: shldq %cl, %r10, %r9
+; SSE2-NEXT: movq %rsi, %r10
+; SSE2-NEXT: shldq %cl, %rax, %r10
+; SSE2-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE2-NEXT: shldq %cl, %rsi, %rdx
+; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: movq %rdx, 24(%rdi)
+; SSE2-NEXT: movq %r10, 16(%rdi)
+; SSE2-NEXT: movq %r9, 8(%rdi)
+; SSE2-NEXT: movq %r8, (%rdi)
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: fshl_rot_i256_vector:
+; SSE42: # %bb.0:
+; SSE42-NEXT: movq %rsi, %rcx
+; SSE42-NEXT: movq %xmm1, %rdx
+; SSE42-NEXT: pextrq $1, %xmm0, %rax
+; SSE42-NEXT: pextrq $1, %xmm1, %rsi
+; SSE42-NEXT: testb $-128, %cl
+; SSE42-NEXT: movq %rsi, %r8
+; SSE42-NEXT: cmovneq %rax, %r8
+; SSE42-NEXT: movq %xmm0, %r9
+; SSE42-NEXT: movq %r9, %r10
+; SSE42-NEXT: cmovneq %rdx, %r10
+; SSE42-NEXT: cmovneq %rsi, %rax
+; SSE42-NEXT: cmovneq %r9, %rdx
+; SSE42-NEXT: testb $64, %cl
+; SSE42-NEXT: movq %rdx, %rsi
+; SSE42-NEXT: cmovneq %rax, %rsi
+; SSE42-NEXT: cmovneq %r10, %rax
+; SSE42-NEXT: cmoveq %r8, %rdx
+; SSE42-NEXT: cmovneq %r8, %r10
+; SSE42-NEXT: movq %r10, %r8
+; SSE42-NEXT: shldq %cl, %rdx, %r8
+; SSE42-NEXT: movq %rax, %r9
+; SSE42-NEXT: shldq %cl, %r10, %r9
+; SSE42-NEXT: movq %rsi, %r10
+; SSE42-NEXT: shldq %cl, %rax, %r10
+; SSE42-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE42-NEXT: shldq %cl, %rsi, %rdx
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: movq %rdx, 24(%rdi)
+; SSE42-NEXT: movq %r10, 16(%rdi)
+; SSE42-NEXT: movq %r9, 8(%rdi)
+; SSE42-NEXT: movq %r8, (%rdi)
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: fshl_rot_i256_vector:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vmovq %xmm1, %rdx
+; AVX2-NEXT: vpextrq $1, %xmm0, %rsi
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: vmovq %xmm0, %r8
+; AVX2-NEXT: testb $-128, %cl
+; AVX2-NEXT: movq %rax, %r9
+; AVX2-NEXT: cmovneq %rsi, %r9
+; AVX2-NEXT: movq %r8, %r10
+; AVX2-NEXT: cmovneq %rdx, %r10
+; AVX2-NEXT: cmovneq %rax, %rsi
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: cmovneq %r8, %rdx
+; AVX2-NEXT: testb $64, %cl
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: cmovneq %rsi, %rdi
+; AVX2-NEXT: cmovneq %r10, %rsi
+; AVX2-NEXT: cmoveq %r9, %rdx
+; AVX2-NEXT: cmovneq %r9, %r10
+; AVX2-NEXT: movq %r10, %r8
+; AVX2-NEXT: shldq %cl, %rdx, %r8
+; AVX2-NEXT: movq %rsi, %r9
+; AVX2-NEXT: shldq %cl, %r10, %r9
+; AVX2-NEXT: movq %rdi, %r10
+; AVX2-NEXT: shldq %cl, %rsi, %r10
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shldq %cl, %rdi, %rdx
+; AVX2-NEXT: movq %rdx, 24(%rax)
+; AVX2-NEXT: movq %r10, 16(%rax)
+; AVX2-NEXT: movq %r9, 8(%rax)
+; AVX2-NEXT: movq %r8, (%rax)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: fshl_rot_i256_vector:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT: vmovq %xmm1, %rdx
+; AVX512F-NEXT: vpextrq $1, %xmm0, %rsi
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rax
+; AVX512F-NEXT: vmovq %xmm0, %r8
+; AVX512F-NEXT: testb $-128, %cl
+; AVX512F-NEXT: movq %rax, %r9
+; AVX512F-NEXT: cmovneq %rsi, %r9
+; AVX512F-NEXT: movq %r8, %r10
+; AVX512F-NEXT: cmovneq %rdx, %r10
+; AVX512F-NEXT: cmovneq %rax, %rsi
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: cmovneq %r8, %rdx
+; AVX512F-NEXT: testb $64, %cl
+; AVX512F-NEXT: movq %rdx, %rdi
+; AVX512F-NEXT: cmovneq %rsi, %rdi
+; AVX512F-NEXT: cmovneq %r10, %rsi
+; AVX512F-NEXT: cmoveq %r9, %rdx
+; AVX512F-NEXT: cmovneq %r9, %r10
+; AVX512F-NEXT: movq %r10, %r8
+; AVX512F-NEXT: shldq %cl, %rdx, %r8
+; AVX512F-NEXT: movq %rsi, %r9
+; AVX512F-NEXT: shldq %cl, %r10, %r9
+; AVX512F-NEXT: movq %rdi, %r10
+; AVX512F-NEXT: shldq %cl, %rsi, %r10
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shldq %cl, %rdi, %rdx
+; AVX512F-NEXT: movq %rdx, 24(%rax)
+; AVX512F-NEXT: movq %r10, 16(%rax)
+; AVX512F-NEXT: movq %r9, 8(%rax)
+; AVX512F-NEXT: movq %r8, (%rax)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: fshl_rot_i256_vector:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT: vmovq %xmm1, %rdx
+; AVX512VL-NEXT: vmovq %xmm0, %rsi
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %r8
+; AVX512VL-NEXT: testb $-128, %cl
+; AVX512VL-NEXT: movq %r8, %r9
+; AVX512VL-NEXT: cmovneq %rax, %r9
+; AVX512VL-NEXT: movq %rsi, %r10
+; AVX512VL-NEXT: cmovneq %rdx, %r10
+; AVX512VL-NEXT: cmovneq %r8, %rax
+; AVX512VL-NEXT: cmovneq %rsi, %rdx
+; AVX512VL-NEXT: testb $64, %cl
+; AVX512VL-NEXT: movq %rdx, %rsi
+; AVX512VL-NEXT: cmovneq %rax, %rsi
+; AVX512VL-NEXT: cmovneq %r10, %rax
+; AVX512VL-NEXT: cmoveq %r9, %rdx
+; AVX512VL-NEXT: cmovneq %r9, %r10
+; AVX512VL-NEXT: movq %r10, %r8
+; AVX512VL-NEXT: shldq %cl, %rdx, %r8
+; AVX512VL-NEXT: movq %rax, %r9
+; AVX512VL-NEXT: shldq %cl, %r10, %r9
+; AVX512VL-NEXT: movq %rsi, %r10
+; AVX512VL-NEXT: shldq %cl, %rax, %r10
+; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VL-NEXT: shldq %cl, %rsi, %rdx
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movq %rdx, 24(%rdi)
+; AVX512VL-NEXT: movq %r10, 16(%rdi)
+; AVX512VL-NEXT: movq %r9, 8(%rdi)
+; AVX512VL-NEXT: movq %r8, (%rdi)
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: fshl_rot_i256_vector:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VBMI-NEXT: vmovq %xmm1, %rdx
+; AVX512VBMI-NEXT: vmovq %xmm0, %rsi
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %r8
+; AVX512VBMI-NEXT: testb $-128, %cl
+; AVX512VBMI-NEXT: movq %r8, %r9
+; AVX512VBMI-NEXT: cmovneq %rax, %r9
+; AVX512VBMI-NEXT: movq %rsi, %r10
+; AVX512VBMI-NEXT: cmovneq %rdx, %r10
+; AVX512VBMI-NEXT: cmovneq %r8, %rax
+; AVX512VBMI-NEXT: cmovneq %rsi, %rdx
+; AVX512VBMI-NEXT: testb $64, %cl
+; AVX512VBMI-NEXT: movq %rdx, %rsi
+; AVX512VBMI-NEXT: cmovneq %rax, %rsi
+; AVX512VBMI-NEXT: cmovneq %r10, %rax
+; AVX512VBMI-NEXT: cmoveq %r9, %rdx
+; AVX512VBMI-NEXT: cmovneq %r9, %r10
+; AVX512VBMI-NEXT: movq %r10, %r8
+; AVX512VBMI-NEXT: shldq %cl, %rdx, %r8
+; AVX512VBMI-NEXT: movq %rax, %r9
+; AVX512VBMI-NEXT: shldq %cl, %r10, %r9
+; AVX512VBMI-NEXT: movq %rsi, %r10
+; AVX512VBMI-NEXT: shldq %cl, %rax, %r10
+; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VBMI-NEXT: shldq %cl, %rsi, %rdx
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movq %rdx, 24(%rdi)
+; AVX512VBMI-NEXT: movq %r10, 16(%rdi)
+; AVX512VBMI-NEXT: movq %r9, 8(%rdi)
+; AVX512VBMI-NEXT: movq %r8, (%rdi)
+; AVX512VBMI-NEXT: vzeroupper
+; AVX512VBMI-NEXT: retq
+ %a0 = bitcast <4 x i64> %v0 to i256
+ %r = call i256 @llvm.fshl.i256(i256 %a0, i256 %a0, i256 %a2)
+ ret i256 %r
+}
+
+define i256 @fshr_rot_i256_vector(<4 x i64> %v0, i256 %a2) nounwind {
+; SSE2-LABEL: fshr_rot_i256_vector:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movq %rsi, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm2, %rdx
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %r8
+; SSE2-NEXT: testb %cl, %cl
+; SSE2-NEXT: movq %r8, %r9
+; SSE2-NEXT: cmovnsq %rdx, %r9
+; SSE2-NEXT: movq %xmm0, %r10
+; SSE2-NEXT: movq %r10, %rsi
+; SSE2-NEXT: cmovnsq %rax, %rsi
+; SSE2-NEXT: cmovnsq %r8, %rdx
+; SSE2-NEXT: cmovnsq %r10, %rax
+; SSE2-NEXT: testb $64, %cl
+; SSE2-NEXT: movq %rax, %r8
+; SSE2-NEXT: cmoveq %rdx, %r8
+; SSE2-NEXT: cmoveq %rsi, %rdx
+; SSE2-NEXT: cmoveq %r9, %rsi
+; SSE2-NEXT: cmovneq %r9, %rax
+; SSE2-NEXT: movq %rax, %r9
+; SSE2-NEXT: shrdq %cl, %rsi, %r9
+; SSE2-NEXT: shrdq %cl, %rdx, %rsi
+; SSE2-NEXT: shrdq %cl, %r8, %rdx
+; SSE2-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE2-NEXT: shrdq %cl, %rax, %r8
+; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: movq %r8, 24(%rdi)
+; SSE2-NEXT: movq %rdx, 16(%rdi)
+; SSE2-NEXT: movq %rsi, 8(%rdi)
+; SSE2-NEXT: movq %r9, (%rdi)
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: fshr_rot_i256_vector:
+; SSE42: # %bb.0:
+; SSE42-NEXT: movq %rsi, %rcx
+; SSE42-NEXT: movq %xmm1, %rax
+; SSE42-NEXT: pextrq $1, %xmm0, %rdx
+; SSE42-NEXT: pextrq $1, %xmm1, %r8
+; SSE42-NEXT: testb %cl, %cl
+; SSE42-NEXT: movq %r8, %r9
+; SSE42-NEXT: cmovnsq %rdx, %r9
+; SSE42-NEXT: movq %xmm0, %r10
+; SSE42-NEXT: movq %r10, %rsi
+; SSE42-NEXT: cmovnsq %rax, %rsi
+; SSE42-NEXT: cmovnsq %r8, %rdx
+; SSE42-NEXT: cmovnsq %r10, %rax
+; SSE42-NEXT: testb $64, %cl
+; SSE42-NEXT: movq %rax, %r8
+; SSE42-NEXT: cmoveq %rdx, %r8
+; SSE42-NEXT: cmoveq %rsi, %rdx
+; SSE42-NEXT: cmoveq %r9, %rsi
+; SSE42-NEXT: cmovneq %r9, %rax
+; SSE42-NEXT: movq %rax, %r9
+; SSE42-NEXT: shrdq %cl, %rsi, %r9
+; SSE42-NEXT: shrdq %cl, %rdx, %rsi
+; SSE42-NEXT: shrdq %cl, %r8, %rdx
+; SSE42-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE42-NEXT: shrdq %cl, %rax, %r8
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: movq %r8, 24(%rdi)
+; SSE42-NEXT: movq %rdx, 16(%rdi)
+; SSE42-NEXT: movq %rsi, 8(%rdi)
+; SSE42-NEXT: movq %r9, (%rdi)
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: fshr_rot_i256_vector:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vmovq %xmm1, %rdx
+; AVX2-NEXT: vpextrq $1, %xmm0, %rsi
+; AVX2-NEXT: vpextrq $1, %xmm1, %rax
+; AVX2-NEXT: vmovq %xmm0, %r9
+; AVX2-NEXT: testb %cl, %cl
+; AVX2-NEXT: movq %rax, %r10
+; AVX2-NEXT: cmovnsq %rsi, %r10
+; AVX2-NEXT: movq %r9, %r8
+; AVX2-NEXT: cmovnsq %rdx, %r8
+; AVX2-NEXT: cmovnsq %rax, %rsi
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: cmovnsq %r9, %rdx
+; AVX2-NEXT: testb $64, %cl
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: cmoveq %rsi, %rdi
+; AVX2-NEXT: cmoveq %r8, %rsi
+; AVX2-NEXT: cmoveq %r10, %r8
+; AVX2-NEXT: cmovneq %r10, %rdx
+; AVX2-NEXT: movq %rdx, %r9
+; AVX2-NEXT: shrdq %cl, %r8, %r9
+; AVX2-NEXT: shrdq %cl, %rsi, %r8
+; AVX2-NEXT: shrdq %cl, %rdi, %rsi
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shrdq %cl, %rdx, %rdi
+; AVX2-NEXT: movq %rdi, 24(%rax)
+; AVX2-NEXT: movq %rsi, 16(%rax)
+; AVX2-NEXT: movq %r8, 8(%rax)
+; AVX2-NEXT: movq %r9, (%rax)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: fshr_rot_i256_vector:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT: vmovq %xmm1, %rdx
+; AVX512F-NEXT: vpextrq $1, %xmm0, %rsi
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rax
+; AVX512F-NEXT: vmovq %xmm0, %r9
+; AVX512F-NEXT: testb %cl, %cl
+; AVX512F-NEXT: movq %rax, %r10
+; AVX512F-NEXT: cmovnsq %rsi, %r10
+; AVX512F-NEXT: movq %r9, %r8
+; AVX512F-NEXT: cmovnsq %rdx, %r8
+; AVX512F-NEXT: cmovnsq %rax, %rsi
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: cmovnsq %r9, %rdx
+; AVX512F-NEXT: testb $64, %cl
+; AVX512F-NEXT: movq %rdx, %rdi
+; AVX512F-NEXT: cmoveq %rsi, %rdi
+; AVX512F-NEXT: cmoveq %r8, %rsi
+; AVX512F-NEXT: cmoveq %r10, %r8
+; AVX512F-NEXT: cmovneq %r10, %rdx
+; AVX512F-NEXT: movq %rdx, %r9
+; AVX512F-NEXT: shrdq %cl, %r8, %r9
+; AVX512F-NEXT: shrdq %cl, %rsi, %r8
+; AVX512F-NEXT: shrdq %cl, %rdi, %rsi
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shrdq %cl, %rdx, %rdi
+; AVX512F-NEXT: movq %rdi, 24(%rax)
+; AVX512F-NEXT: movq %rsi, 16(%rax)
+; AVX512F-NEXT: movq %r8, 8(%rax)
+; AVX512F-NEXT: movq %r9, (%rax)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: fshr_rot_i256_vector:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT: vmovq %xmm1, %rax
+; AVX512VL-NEXT: vmovq %xmm0, %r8
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %r9
+; AVX512VL-NEXT: testb %cl, %cl
+; AVX512VL-NEXT: movq %r9, %r10
+; AVX512VL-NEXT: cmovnsq %rdx, %r10
+; AVX512VL-NEXT: movq %r8, %rsi
+; AVX512VL-NEXT: cmovnsq %rax, %rsi
+; AVX512VL-NEXT: cmovnsq %r9, %rdx
+; AVX512VL-NEXT: cmovnsq %r8, %rax
+; AVX512VL-NEXT: testb $64, %cl
+; AVX512VL-NEXT: movq %rax, %r8
+; AVX512VL-NEXT: cmoveq %rdx, %r8
+; AVX512VL-NEXT: cmoveq %rsi, %rdx
+; AVX512VL-NEXT: cmoveq %r10, %rsi
+; AVX512VL-NEXT: cmovneq %r10, %rax
+; AVX512VL-NEXT: movq %rax, %r9
+; AVX512VL-NEXT: shrdq %cl, %rsi, %r9
+; AVX512VL-NEXT: shrdq %cl, %rdx, %rsi
+; AVX512VL-NEXT: shrdq %cl, %r8, %rdx
+; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VL-NEXT: shrdq %cl, %rax, %r8
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movq %r8, 24(%rdi)
+; AVX512VL-NEXT: movq %rdx, 16(%rdi)
+; AVX512VL-NEXT: movq %rsi, 8(%rdi)
+; AVX512VL-NEXT: movq %r9, (%rdi)
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: fshr_rot_i256_vector:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VBMI-NEXT: vmovq %xmm1, %rax
+; AVX512VBMI-NEXT: vmovq %xmm0, %r8
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %r9
+; AVX512VBMI-NEXT: testb %cl, %cl
+; AVX512VBMI-NEXT: movq %r9, %r10
+; AVX512VBMI-NEXT: cmovnsq %rdx, %r10
+; AVX512VBMI-NEXT: movq %r8, %rsi
+; AVX512VBMI-NEXT: cmovnsq %rax, %rsi
+; AVX512VBMI-NEXT: cmovnsq %r9, %rdx
+; AVX512VBMI-NEXT: cmovnsq %r8, %rax
+; AVX512VBMI-NEXT: testb $64, %cl
+; AVX512VBMI-NEXT: movq %rax, %r8
+; AVX512VBMI-NEXT: cmoveq %rdx, %r8
+; AVX512VBMI-NEXT: cmoveq %rsi, %rdx
+; AVX512VBMI-NEXT: cmoveq %r10, %rsi
+; AVX512VBMI-NEXT: cmovneq %r10, %rax
+; AVX512VBMI-NEXT: movq %rax, %r9
+; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r9
+; AVX512VBMI-NEXT: shrdq %cl, %rdx, %rsi
+; AVX512VBMI-NEXT: shrdq %cl, %r8, %rdx
+; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VBMI-NEXT: shrdq %cl, %rax, %r8
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movq %r8, 24(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
+; AVX512VBMI-NEXT: movq %rsi, 8(%rdi)
+; AVX512VBMI-NEXT: movq %r9, (%rdi)
+; AVX512VBMI-NEXT: vzeroupper
+; AVX512VBMI-NEXT: retq
+ %a0 = bitcast <4 x i64> %v0 to i256
+ %r = call i256 @llvm.fshr.i256(i256 %a0, i256 %a0, i256 %a2)
+ ret i256 %r
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX512: {{.*}}
diff --git a/llvm/test/CodeGen/X86/shift-i256.ll b/llvm/test/CodeGen/X86/shift-i256.ll
index 128e2199fb56f..00f6b5cf04ced 100644
--- a/llvm/test/CodeGen/X86/shift-i256.ll
+++ b/llvm/test/CodeGen/X86/shift-i256.ll
@@ -1,322 +1,3178 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-- | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-- -O0 | FileCheck %s -check-prefixes=CHECK-X64,CHECK-X64-O0
-; RUN: llc < %s -mtriple=x86_64-- -O2 | FileCheck %s -check-prefixes=CHECK-X64,CHECK-X64-O2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s -check-prefixes=CHECK,SSE
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s -check-prefixes=CHECK,SSE
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s -check-prefixes=CHECK,AVX2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=knl | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512VL
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 -mattr=+AVX512vbmi2 | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512VBMI
+; RUN: llc < %s -mtriple=i686-- | FileCheck %s -check-prefixes=X86
-; CHECK-LABEL: shift1
-define void @shift1(i256 %x, i256 %a, ptr nocapture %r) nounwind readnone {
-; CHECK-LABEL: shift1:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushl %ebp
-; CHECK-NEXT: movl %esp, %ebp
-; CHECK-NEXT: pushl %ebx
-; CHECK-NEXT: pushl %edi
-; CHECK-NEXT: pushl %esi
-; CHECK-NEXT: andl $-16, %esp
-; CHECK-NEXT: subl $112, %esp
-; CHECK-NEXT: movl 40(%ebp), %ecx
-; CHECK-NEXT: movl 8(%ebp), %eax
-; CHECK-NEXT: movl 12(%ebp), %edx
-; CHECK-NEXT: movl 16(%ebp), %esi
-; CHECK-NEXT: movl 32(%ebp), %edi
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl 28(%ebp), %edi
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl 24(%ebp), %edi
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl 20(%ebp), %edi
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl 36(%ebp), %edi
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; CHECK-NEXT: sarl $31, %edi
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl %ecx, %eax
-; CHECK-NEXT: shrb $5, %al
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: movl 40(%esp,%eax,4), %edx
-; CHECK-NEXT: movl 36(%esp,%eax,4), %esi
-; CHECK-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: shrdl %cl, %edx, %esi
-; CHECK-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl 44(%esp,%eax,4), %esi
-; CHECK-NEXT: shrdl %cl, %esi, %edx
-; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl 48(%esp,%eax,4), %ebx
-; CHECK-NEXT: shrdl %cl, %ebx, %esi
-; CHECK-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl 52(%esp,%eax,4), %esi
-; CHECK-NEXT: shrdl %cl, %esi, %ebx
-; CHECK-NEXT: movl 56(%esp,%eax,4), %edx
-; CHECK-NEXT: shrdl %cl, %edx, %esi
-; CHECK-NEXT: movl 32(%esp,%eax,4), %edi
-; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl 60(%esp,%eax,4), %eax
-; CHECK-NEXT: shrdl %cl, %eax, %edx
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; CHECK-NEXT: shrdl %cl, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT: sarl %cl, %eax
-; CHECK-NEXT: movl 72(%ebp), %ecx
-; CHECK-NEXT: movl %eax, 28(%ecx)
-; CHECK-NEXT: movl %edx, 24(%ecx)
-; CHECK-NEXT: movl %esi, 20(%ecx)
-; CHECK-NEXT: movl %ebx, 16(%ecx)
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-NEXT: movl %eax, 12(%ecx)
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-NEXT: movl %eax, 8(%ecx)
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-NEXT: movl %eax, 4(%ecx)
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; CHECK-NEXT: movl %eax, (%ecx)
-; CHECK-NEXT: leal -12(%ebp), %esp
-; CHECK-NEXT: popl %esi
-; CHECK-NEXT: popl %edi
-; CHECK-NEXT: popl %ebx
-; CHECK-NEXT: popl %ebp
-; CHECK-NEXT: retl
-;
-; CHECK-X64-O0-LABEL: shift1:
-; CHECK-X64-O0: # %bb.0: # %entry
-; CHECK-X64-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-X64-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-X64-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-X64-O0-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: sarq $63, %rcx
-; CHECK-X64-O0-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movb %r8b, %cl
-; CHECK-X64-O0-NEXT: movb %cl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; CHECK-X64-O0-NEXT: movb %cl, %dl
-; CHECK-X64-O0-NEXT: shrb $6, %dl
-; CHECK-X64-O0-NEXT: movzbl %dl, %edx
-; CHECK-X64-O0-NEXT: movl %edx, %edi
-; CHECK-X64-O0-NEXT: movq -56(%rsp,%rdi,8), %rsi
-; CHECK-X64-O0-NEXT: movq -72(%rsp,%rdi,8), %r8
-; CHECK-X64-O0-NEXT: movq -64(%rsp,%rdi,8), %r9
-; CHECK-X64-O0-NEXT: movq %r9, %rdx
-; CHECK-X64-O0-NEXT: shrdq %cl, %rsi, %rdx
-; CHECK-X64-O0-NEXT: movb {{[-0-9]+}}(%r{{[sb]}}p), %cl # 1-byte Reload
-; CHECK-X64-O0-NEXT: movq -48(%rsp,%rdi,8), %rdi
-; CHECK-X64-O0-NEXT: shrdq %cl, %rdi, %rsi
-; CHECK-X64-O0-NEXT: movb {{[-0-9]+}}(%r{{[sb]}}p), %cl # 1-byte Reload
-; CHECK-X64-O0-NEXT: shrdq %cl, %r9, %r8
-; CHECK-X64-O0-NEXT: movb {{[-0-9]+}}(%r{{[sb]}}p), %cl # 1-byte Reload
-; CHECK-X64-O0-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-X64-O0-NEXT: sarq %cl, %rdi
-; CHECK-X64-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-X64-O0-NEXT: movq %rdi, 24(%rax)
-; CHECK-X64-O0-NEXT: movq %rsi, 16(%rax)
-; CHECK-X64-O0-NEXT: movq %rdx, 8(%rax)
-; CHECK-X64-O0-NEXT: movq %rcx, (%rax)
-; CHECK-X64-O0-NEXT: retq
-;
-; CHECK-X64-O2-LABEL: shift1:
-; CHECK-X64-O2: # %bb.0: # %entry
-; CHECK-X64-O2-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-X64-O2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O2-NEXT: sarq $63, %rcx
-; CHECK-X64-O2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O2-NEXT: movl %r8d, %ecx
-; CHECK-X64-O2-NEXT: shrb $6, %cl
-; CHECK-X64-O2-NEXT: movzbl %cl, %edx
-; CHECK-X64-O2-NEXT: movq -56(%rsp,%rdx,8), %rsi
-; CHECK-X64-O2-NEXT: movq -72(%rsp,%rdx,8), %rdi
-; CHECK-X64-O2-NEXT: movq -64(%rsp,%rdx,8), %r9
-; CHECK-X64-O2-NEXT: movq %r9, %r10
-; CHECK-X64-O2-NEXT: movl %r8d, %ecx
-; CHECK-X64-O2-NEXT: shrdq %cl, %rsi, %r10
-; CHECK-X64-O2-NEXT: movq -48(%rsp,%rdx,8), %rdx
-; CHECK-X64-O2-NEXT: shrdq %cl, %rdx, %rsi
-; CHECK-X64-O2-NEXT: shrdq %cl, %r9, %rdi
-; CHECK-X64-O2-NEXT: sarq %cl, %rdx
-; CHECK-X64-O2-NEXT: movq %rdx, 24(%rax)
-; CHECK-X64-O2-NEXT: movq %rsi, 16(%rax)
-; CHECK-X64-O2-NEXT: movq %r10, 8(%rax)
-; CHECK-X64-O2-NEXT: movq %rdi, (%rax)
-; CHECK-X64-O2-NEXT: retq
-entry:
- %0 = ashr i256 %x, %a
- store i256 %0, ptr %r
- ret void
+define i256 @shl_i256(i256 %a0, i256 %a1) nounwind {
+; SSE-LABEL: shl_i256:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: xorps %xmm0, %xmm0
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %r9d, %eax
+; SSE-NEXT: shrb $3, %al
+; SSE-NEXT: andb $24, %al
+; SSE-NEXT: negb %al
+; SSE-NEXT: movsbq %al, %rax
+; SSE-NEXT: movq -32(%rsp,%rax), %rdx
+; SSE-NEXT: movq -24(%rsp,%rax), %rsi
+; SSE-NEXT: movq %rsi, %r8
+; SSE-NEXT: movl %r9d, %ecx
+; SSE-NEXT: shldq %cl, %rdx, %r8
+; SSE-NEXT: movq -16(%rsp,%rax), %r10
+; SSE-NEXT: shldq %cl, %rsi, %r10
+; SSE-NEXT: movq -40(%rsp,%rax), %rax
+; SSE-NEXT: movq %rax, %rsi
+; SSE-NEXT: shlq %cl, %rsi
+; SSE-NEXT: shldq %cl, %rax, %rdx
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq %r10, 24(%rdi)
+; SSE-NEXT: movq %r8, 16(%rdi)
+; SSE-NEXT: movq %rdx, 8(%rdi)
+; SSE-NEXT: movq %rsi, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: shl_i256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %r9d, %eax
+; AVX2-NEXT: shrb $3, %al
+; AVX2-NEXT: andb $24, %al
+; AVX2-NEXT: negb %al
+; AVX2-NEXT: movsbq %al, %rdx
+; AVX2-NEXT: movq -32(%rsp,%rdx), %rsi
+; AVX2-NEXT: movq -24(%rsp,%rdx), %rax
+; AVX2-NEXT: movq %rax, %r8
+; AVX2-NEXT: movl %r9d, %ecx
+; AVX2-NEXT: shldq %cl, %rsi, %r8
+; AVX2-NEXT: movq -16(%rsp,%rdx), %r10
+; AVX2-NEXT: shldq %cl, %rax, %r10
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq -40(%rsp,%rdx), %rdx
+; AVX2-NEXT: shlxq %r9, %rdx, %rdi
+; AVX2-NEXT: shldq %cl, %rdx, %rsi
+; AVX2-NEXT: movq %r10, 24(%rax)
+; AVX2-NEXT: movq %r8, 16(%rax)
+; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: shl_i256:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %r9d, %eax
+; AVX512F-NEXT: shrb $3, %al
+; AVX512F-NEXT: andb $24, %al
+; AVX512F-NEXT: negb %al
+; AVX512F-NEXT: movsbq %al, %rdx
+; AVX512F-NEXT: movq -32(%rsp,%rdx), %rsi
+; AVX512F-NEXT: movq -24(%rsp,%rdx), %rax
+; AVX512F-NEXT: movq %rax, %r8
+; AVX512F-NEXT: movl %r9d, %ecx
+; AVX512F-NEXT: shldq %cl, %rsi, %r8
+; AVX512F-NEXT: movq -16(%rsp,%rdx), %r10
+; AVX512F-NEXT: shldq %cl, %rax, %r10
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: movq -40(%rsp,%rdx), %rdx
+; AVX512F-NEXT: shlxq %r9, %rdx, %rdi
+; AVX512F-NEXT: shldq %cl, %rdx, %rsi
+; AVX512F-NEXT: movq %r10, 24(%rax)
+; AVX512F-NEXT: movq %r8, 16(%rax)
+; AVX512F-NEXT: movq %rsi, 8(%rax)
+; AVX512F-NEXT: movq %rdi, (%rax)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: shl_i256:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %r9d, %eax
+; AVX512VL-NEXT: shrb $3, %al
+; AVX512VL-NEXT: andb $24, %al
+; AVX512VL-NEXT: negb %al
+; AVX512VL-NEXT: movsbq %al, %rax
+; AVX512VL-NEXT: movq -32(%rsp,%rax), %rdx
+; AVX512VL-NEXT: movq -24(%rsp,%rax), %rsi
+; AVX512VL-NEXT: movq %rsi, %r8
+; AVX512VL-NEXT: movl %r9d, %ecx
+; AVX512VL-NEXT: shldq %cl, %rdx, %r8
+; AVX512VL-NEXT: movq -16(%rsp,%rax), %r10
+; AVX512VL-NEXT: shldq %cl, %rsi, %r10
+; AVX512VL-NEXT: movq -40(%rsp,%rax), %rsi
+; AVX512VL-NEXT: shldq %cl, %rsi, %rdx
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: shlxq %r9, %rsi, %rcx
+; AVX512VL-NEXT: movq %r10, 24(%rdi)
+; AVX512VL-NEXT: movq %r8, 16(%rdi)
+; AVX512VL-NEXT: movq %rdx, 8(%rdi)
+; AVX512VL-NEXT: movq %rcx, (%rdi)
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: shl_i256:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %r9d, %eax
+; AVX512VBMI-NEXT: shrb $3, %al
+; AVX512VBMI-NEXT: andb $24, %al
+; AVX512VBMI-NEXT: negb %al
+; AVX512VBMI-NEXT: movsbq %al, %rax
+; AVX512VBMI-NEXT: movq -32(%rsp,%rax), %rdx
+; AVX512VBMI-NEXT: movq -24(%rsp,%rax), %rsi
+; AVX512VBMI-NEXT: movq %rsi, %r8
+; AVX512VBMI-NEXT: movl %r9d, %ecx
+; AVX512VBMI-NEXT: shldq %cl, %rdx, %r8
+; AVX512VBMI-NEXT: movq -16(%rsp,%rax), %r10
+; AVX512VBMI-NEXT: shldq %cl, %rsi, %r10
+; AVX512VBMI-NEXT: movq -40(%rsp,%rax), %rsi
+; AVX512VBMI-NEXT: shldq %cl, %rsi, %rdx
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: shlxq %r9, %rsi, %rcx
+; AVX512VBMI-NEXT: movq %r10, 24(%rdi)
+; AVX512VBMI-NEXT: movq %r8, 16(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, 8(%rdi)
+; AVX512VBMI-NEXT: movq %rcx, (%rdi)
+; AVX512VBMI-NEXT: vzeroupper
+; AVX512VBMI-NEXT: retq
+;
+; X86-LABEL: shl_i256:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $112, %esp
+; X86-NEXT: movl 44(%ebp), %ecx
+; X86-NEXT: movl 12(%ebp), %eax
+; X86-NEXT: movl 16(%ebp), %edx
+; X86-NEXT: movl 20(%ebp), %esi
+; X86-NEXT: movl 40(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $3, %al
+; X86-NEXT: andb $28, %al
+; X86-NEXT: negb %al
+; X86-NEXT: movsbl %al, %eax
+; X86-NEXT: movl 68(%esp,%eax), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 72(%esp,%eax), %edx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 76(%esp,%eax), %esi
+; X86-NEXT: movl %esi, %edi
+; X86-NEXT: shldl %cl, %edx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 80(%esp,%eax), %edx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 84(%esp,%eax), %esi
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: shldl %cl, %edx, %ebx
+; X86-NEXT: movl 88(%esp,%eax), %edi
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shldl %cl, %esi, %edx
+; X86-NEXT: movl 64(%esp,%eax), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 92(%esp,%eax), %esi
+; X86-NEXT: shldl %cl, %edi, %esi
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl %esi, 28(%eax)
+; X86-NEXT: movl %edx, 24(%eax)
+; X86-NEXT: movl %ebx, 20(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 16(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 12(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 8(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shll %cl, %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl %cl, %edi, %esi
+; X86-NEXT: movl %esi, 4(%eax)
+; X86-NEXT: movl %edx, (%eax)
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+ %r = shl i256 %a0, %a1
+ ret i256 %r
+}
+
+define i256 @lshr_i256(i256 %a0, i256 %a1) nounwind {
+; SSE-LABEL: lshr_i256:
+; SSE: # %bb.0:
+; SSE-NEXT: xorps %xmm0, %xmm0
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %r9d, %eax
+; SSE-NEXT: shrb $6, %al
+; SSE-NEXT: movzbl %al, %eax
+; SSE-NEXT: movq -56(%rsp,%rax,8), %rdx
+; SSE-NEXT: movq -64(%rsp,%rax,8), %rsi
+; SSE-NEXT: movq %rsi, %r8
+; SSE-NEXT: movl %r9d, %ecx
+; SSE-NEXT: shrdq %cl, %rdx, %r8
+; SSE-NEXT: movq -48(%rsp,%rax,8), %r10
+; SSE-NEXT: shrdq %cl, %r10, %rdx
+; SSE-NEXT: movq -72(%rsp,%rax,8), %r11
+; SSE-NEXT: shrdq %cl, %rsi, %r11
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: shrq %cl, %r10
+; SSE-NEXT: movq %r10, 24(%rdi)
+; SSE-NEXT: movq %rdx, 16(%rdi)
+; SSE-NEXT: movq %r8, 8(%rdi)
+; SSE-NEXT: movq %r11, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: lshr_i256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %r9d, %eax
+; AVX2-NEXT: shrb $6, %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX2-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX2-NEXT: movq %rsi, %r8
+; AVX2-NEXT: movl %r9d, %ecx
+; AVX2-NEXT: shrdq %cl, %rdx, %r8
+; AVX2-NEXT: movq -72(%rsp,%rax,8), %r10
+; AVX2-NEXT: movq -48(%rsp,%rax,8), %r11
+; AVX2-NEXT: shrdq %cl, %r11, %rdx
+; AVX2-NEXT: shrdq %cl, %rsi, %r10
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: shrxq %r9, %r11, %rcx
+; AVX2-NEXT: movq %rcx, 24(%rdi)
+; AVX2-NEXT: movq %rdx, 16(%rdi)
+; AVX2-NEXT: movq %r8, 8(%rdi)
+; AVX2-NEXT: movq %r10, (%rdi)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: lshr_i256:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %r9d, %eax
+; AVX512F-NEXT: shrb $6, %al
+; AVX512F-NEXT: movzbl %al, %eax
+; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512F-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512F-NEXT: movq %rsi, %r8
+; AVX512F-NEXT: movl %r9d, %ecx
+; AVX512F-NEXT: shrdq %cl, %rdx, %r8
+; AVX512F-NEXT: movq -72(%rsp,%rax,8), %r10
+; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r11
+; AVX512F-NEXT: shrdq %cl, %r11, %rdx
+; AVX512F-NEXT: shrdq %cl, %rsi, %r10
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: shrxq %r9, %r11, %rcx
+; AVX512F-NEXT: movq %rcx, 24(%rdi)
+; AVX512F-NEXT: movq %rdx, 16(%rdi)
+; AVX512F-NEXT: movq %r8, 8(%rdi)
+; AVX512F-NEXT: movq %r10, (%rdi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: lshr_i256:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %r9d, %eax
+; AVX512VL-NEXT: shrb $6, %al
+; AVX512VL-NEXT: movzbl %al, %eax
+; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512VL-NEXT: movq %rsi, %r8
+; AVX512VL-NEXT: movl %r9d, %ecx
+; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r10
+; AVX512VL-NEXT: shrdq %cl, %r10, %rdx
+; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %r11
+; AVX512VL-NEXT: shrdq %cl, %rsi, %r11
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: shrxq %r9, %r10, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rdi)
+; AVX512VL-NEXT: movq %rdx, 16(%rdi)
+; AVX512VL-NEXT: movq %r8, 8(%rdi)
+; AVX512VL-NEXT: movq %r11, (%rdi)
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: lshr_i256:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %r9d, %eax
+; AVX512VBMI-NEXT: shrb $6, %al
+; AVX512VBMI-NEXT: movzbl %al, %eax
+; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512VBMI-NEXT: movq %rsi, %r8
+; AVX512VBMI-NEXT: movl %r9d, %ecx
+; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r10
+; AVX512VBMI-NEXT: shrdq %cl, %r10, %rdx
+; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %r11
+; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r11
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: shrxq %r9, %r10, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
+; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
+; AVX512VBMI-NEXT: movq %r11, (%rdi)
+; AVX512VBMI-NEXT: vzeroupper
+; AVX512VBMI-NEXT: retq
+;
+; X86-LABEL: lshr_i256:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $112, %esp
+; X86-NEXT: movl 44(%ebp), %ecx
+; X86-NEXT: movl 12(%ebp), %eax
+; X86-NEXT: movl 16(%ebp), %edx
+; X86-NEXT: movl 20(%ebp), %esi
+; X86-NEXT: movl 40(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $5, %al
+; X86-NEXT: movzbl %al, %eax
+; X86-NEXT: movl 40(%esp,%eax,4), %edx
+; X86-NEXT: movl 36(%esp,%eax,4), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 44(%esp,%eax,4), %esi
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 48(%esp,%eax,4), %ebx
+; X86-NEXT: shrdl %cl, %ebx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 52(%esp,%eax,4), %edi
+; X86-NEXT: shrdl %cl, %edi, %ebx
+; X86-NEXT: movl 56(%esp,%eax,4), %esi
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl 32(%esp,%eax,4), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 60(%esp,%eax,4), %edx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrl %cl, %edx
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl %edx, 28(%eax)
+; X86-NEXT: movl %esi, 24(%eax)
+; X86-NEXT: movl %edi, 20(%eax)
+; X86-NEXT: movl %ebx, 16(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 12(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+ %r = lshr i256 %a0, %a1
+ ret i256 %r
+}
+
+define i256 @ashr_i256(i256 %a0, i256 %a1) nounwind {
+; SSE-LABEL: ashr_i256:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: sarq $63, %r8
+; SSE-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %r9d, %eax
+; SSE-NEXT: shrb $6, %al
+; SSE-NEXT: movzbl %al, %eax
+; SSE-NEXT: movq -56(%rsp,%rax,8), %rdx
+; SSE-NEXT: movq -64(%rsp,%rax,8), %rsi
+; SSE-NEXT: movq %rsi, %r8
+; SSE-NEXT: movl %r9d, %ecx
+; SSE-NEXT: shrdq %cl, %rdx, %r8
+; SSE-NEXT: movq -48(%rsp,%rax,8), %r10
+; SSE-NEXT: shrdq %cl, %r10, %rdx
+; SSE-NEXT: movq -72(%rsp,%rax,8), %r11
+; SSE-NEXT: shrdq %cl, %rsi, %r11
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: sarq %cl, %r10
+; SSE-NEXT: movq %r10, 24(%rdi)
+; SSE-NEXT: movq %rdx, 16(%rdi)
+; SSE-NEXT: movq %r8, 8(%rdi)
+; SSE-NEXT: movq %r11, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: ashr_i256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: sarq $63, %r8
+; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %r9d, %eax
+; AVX2-NEXT: shrb $6, %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX2-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX2-NEXT: movq %rsi, %r8
+; AVX2-NEXT: movl %r9d, %ecx
+; AVX2-NEXT: shrdq %cl, %rdx, %r8
+; AVX2-NEXT: movq -72(%rsp,%rax,8), %r10
+; AVX2-NEXT: movq -48(%rsp,%rax,8), %r11
+; AVX2-NEXT: shrdq %cl, %r11, %rdx
+; AVX2-NEXT: shrdq %cl, %rsi, %r10
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: sarxq %r9, %r11, %rcx
+; AVX2-NEXT: movq %rcx, 24(%rdi)
+; AVX2-NEXT: movq %rdx, 16(%rdi)
+; AVX2-NEXT: movq %r8, 8(%rdi)
+; AVX2-NEXT: movq %r10, (%rdi)
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: ashr_i256:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: sarq $63, %r8
+; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %r9d, %eax
+; AVX512F-NEXT: shrb $6, %al
+; AVX512F-NEXT: movzbl %al, %eax
+; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512F-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512F-NEXT: movq %rsi, %r8
+; AVX512F-NEXT: movl %r9d, %ecx
+; AVX512F-NEXT: shrdq %cl, %rdx, %r8
+; AVX512F-NEXT: movq -72(%rsp,%rax,8), %r10
+; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r11
+; AVX512F-NEXT: shrdq %cl, %r11, %rdx
+; AVX512F-NEXT: shrdq %cl, %rsi, %r10
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: sarxq %r9, %r11, %rcx
+; AVX512F-NEXT: movq %rcx, 24(%rdi)
+; AVX512F-NEXT: movq %rdx, 16(%rdi)
+; AVX512F-NEXT: movq %r8, 8(%rdi)
+; AVX512F-NEXT: movq %r10, (%rdi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: ashr_i256:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: sarq $63, %r8
+; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %r9d, %eax
+; AVX512VL-NEXT: shrb $6, %al
+; AVX512VL-NEXT: movzbl %al, %eax
+; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512VL-NEXT: movq %rsi, %r8
+; AVX512VL-NEXT: movl %r9d, %ecx
+; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r10
+; AVX512VL-NEXT: shrdq %cl, %r10, %rdx
+; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %r11
+; AVX512VL-NEXT: shrdq %cl, %rsi, %r11
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: sarxq %r9, %r10, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rdi)
+; AVX512VL-NEXT: movq %rdx, 16(%rdi)
+; AVX512VL-NEXT: movq %r8, 8(%rdi)
+; AVX512VL-NEXT: movq %r11, (%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: ashr_i256:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: sarq $63, %r8
+; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %r9d, %eax
+; AVX512VBMI-NEXT: shrb $6, %al
+; AVX512VBMI-NEXT: movzbl %al, %eax
+; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512VBMI-NEXT: movq %rsi, %r8
+; AVX512VBMI-NEXT: movl %r9d, %ecx
+; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r10
+; AVX512VBMI-NEXT: shrdq %cl, %r10, %rdx
+; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %r11
+; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r11
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: sarxq %r9, %r10, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
+; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
+; AVX512VBMI-NEXT: movq %r11, (%rdi)
+; AVX512VBMI-NEXT: retq
+;
+; X86-LABEL: ashr_i256:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $112, %esp
+; X86-NEXT: movl 44(%ebp), %ecx
+; X86-NEXT: movl 12(%ebp), %eax
+; X86-NEXT: movl 16(%ebp), %edx
+; X86-NEXT: movl 20(%ebp), %esi
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 40(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: sarl $31, %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $5, %al
+; X86-NEXT: movzbl %al, %eax
+; X86-NEXT: movl 40(%esp,%eax,4), %edx
+; X86-NEXT: movl 36(%esp,%eax,4), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 44(%esp,%eax,4), %esi
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 48(%esp,%eax,4), %ebx
+; X86-NEXT: shrdl %cl, %ebx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 52(%esp,%eax,4), %edi
+; X86-NEXT: shrdl %cl, %edi, %ebx
+; X86-NEXT: movl 56(%esp,%eax,4), %esi
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl 32(%esp,%eax,4), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 60(%esp,%eax,4), %edx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: sarl %cl, %edx
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl %edx, 28(%eax)
+; X86-NEXT: movl %esi, 24(%eax)
+; X86-NEXT: movl %edi, 20(%eax)
+; X86-NEXT: movl %ebx, 16(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 12(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+ %r = ashr i256 %a0, %a1
+ ret i256 %r
+}
+
+define i256 @shl_i256_load(ptr %p0, i256 %a1) nounwind {
+; SSE-LABEL: shl_i256_load:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdx, %rcx
+; SSE-NEXT: movaps (%rsi), %xmm0
+; SSE-NEXT: movaps 16(%rsi), %xmm1
+; SSE-NEXT: xorps %xmm2, %xmm2
+; SSE-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %ecx, %eax
+; SSE-NEXT: shrb $3, %al
+; SSE-NEXT: andb $24, %al
+; SSE-NEXT: negb %al
+; SSE-NEXT: movsbq %al, %rax
+; SSE-NEXT: movq -32(%rsp,%rax), %rdx
+; SSE-NEXT: movq -24(%rsp,%rax), %rsi
+; SSE-NEXT: movq %rsi, %r8
+; SSE-NEXT: shldq %cl, %rdx, %r8
+; SSE-NEXT: movq -16(%rsp,%rax), %r9
+; SSE-NEXT: shldq %cl, %rsi, %r9
+; SSE-NEXT: movq -40(%rsp,%rax), %rax
+; SSE-NEXT: movq %rax, %rsi
+; SSE-NEXT: shlq %cl, %rsi
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shldq %cl, %rax, %rdx
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq %r9, 24(%rdi)
+; SSE-NEXT: movq %r8, 16(%rdi)
+; SSE-NEXT: movq %rdx, 8(%rdi)
+; SSE-NEXT: movq %rsi, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: shl_i256_load:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: vmovups (%rsi), %ymm0
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: shrb $3, %al
+; AVX2-NEXT: andb $24, %al
+; AVX2-NEXT: negb %al
+; AVX2-NEXT: movsbq %al, %rdx
+; AVX2-NEXT: movq -32(%rsp,%rdx), %rsi
+; AVX2-NEXT: movq -24(%rsp,%rdx), %rax
+; AVX2-NEXT: movq %rax, %r8
+; AVX2-NEXT: shldq %cl, %rsi, %r8
+; AVX2-NEXT: movq -16(%rsp,%rdx), %r9
+; AVX2-NEXT: shldq %cl, %rax, %r9
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq -40(%rsp,%rdx), %rdx
+; AVX2-NEXT: shlxq %rcx, %rdx, %rdi
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shldq %cl, %rdx, %rsi
+; AVX2-NEXT: movq %r9, 24(%rax)
+; AVX2-NEXT: movq %r8, 16(%rax)
+; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: shl_i256_load:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rdx, %rcx
+; AVX512F-NEXT: vmovups (%rsi), %ymm0
+; AVX512F-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %ecx, %eax
+; AVX512F-NEXT: shrb $3, %al
+; AVX512F-NEXT: andb $24, %al
+; AVX512F-NEXT: negb %al
+; AVX512F-NEXT: movsbq %al, %rdx
+; AVX512F-NEXT: movq -32(%rsp,%rdx), %rsi
+; AVX512F-NEXT: movq -24(%rsp,%rdx), %rax
+; AVX512F-NEXT: movq %rax, %r8
+; AVX512F-NEXT: shldq %cl, %rsi, %r8
+; AVX512F-NEXT: movq -16(%rsp,%rdx), %r9
+; AVX512F-NEXT: shldq %cl, %rax, %r9
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: movq -40(%rsp,%rdx), %rdx
+; AVX512F-NEXT: shlxq %rcx, %rdx, %rdi
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shldq %cl, %rdx, %rsi
+; AVX512F-NEXT: movq %r9, 24(%rax)
+; AVX512F-NEXT: movq %r8, 16(%rax)
+; AVX512F-NEXT: movq %rsi, 8(%rax)
+; AVX512F-NEXT: movq %rdi, (%rax)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: shl_i256_load:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rdx, %rcx
+; AVX512VL-NEXT: vmovups (%rsi), %ymm0
+; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %ecx, %eax
+; AVX512VL-NEXT: shrb $3, %al
+; AVX512VL-NEXT: andb $24, %al
+; AVX512VL-NEXT: negb %al
+; AVX512VL-NEXT: movsbq %al, %rax
+; AVX512VL-NEXT: movq -32(%rsp,%rax), %rdx
+; AVX512VL-NEXT: movq -24(%rsp,%rax), %rsi
+; AVX512VL-NEXT: movq %rsi, %r8
+; AVX512VL-NEXT: shldq %cl, %rdx, %r8
+; AVX512VL-NEXT: movq -16(%rsp,%rax), %r9
+; AVX512VL-NEXT: shldq %cl, %rsi, %r9
+; AVX512VL-NEXT: movq -40(%rsp,%rax), %rsi
+; AVX512VL-NEXT: shldq %cl, %rsi, %rdx
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: shlxq %rcx, %rsi, %rcx
+; AVX512VL-NEXT: movq %r9, 24(%rdi)
+; AVX512VL-NEXT: movq %r8, 16(%rdi)
+; AVX512VL-NEXT: movq %rdx, 8(%rdi)
+; AVX512VL-NEXT: movq %rcx, (%rdi)
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: shl_i256_load:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rdx, %rcx
+; AVX512VBMI-NEXT: vmovups (%rsi), %ymm0
+; AVX512VBMI-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %ecx, %eax
+; AVX512VBMI-NEXT: shrb $3, %al
+; AVX512VBMI-NEXT: andb $24, %al
+; AVX512VBMI-NEXT: negb %al
+; AVX512VBMI-NEXT: movsbq %al, %rax
+; AVX512VBMI-NEXT: movq -32(%rsp,%rax), %rdx
+; AVX512VBMI-NEXT: movq -24(%rsp,%rax), %rsi
+; AVX512VBMI-NEXT: movq %rsi, %r8
+; AVX512VBMI-NEXT: shldq %cl, %rdx, %r8
+; AVX512VBMI-NEXT: movq -16(%rsp,%rax), %r9
+; AVX512VBMI-NEXT: shldq %cl, %rsi, %r9
+; AVX512VBMI-NEXT: movq -40(%rsp,%rax), %rsi
+; AVX512VBMI-NEXT: shldq %cl, %rsi, %rdx
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: shlxq %rcx, %rsi, %rcx
+; AVX512VBMI-NEXT: movq %r9, 24(%rdi)
+; AVX512VBMI-NEXT: movq %r8, 16(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, 8(%rdi)
+; AVX512VBMI-NEXT: movq %rcx, (%rdi)
+; AVX512VBMI-NEXT: vzeroupper
+; AVX512VBMI-NEXT: retq
+;
+; X86-LABEL: shl_i256_load:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $112, %esp
+; X86-NEXT: movl 12(%ebp), %ecx
+; X86-NEXT: movl (%ecx), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 4(%ecx), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 8(%ecx), %esi
+; X86-NEXT: movl 12(%ecx), %edi
+; X86-NEXT: movl 16(%ecx), %ebx
+; X86-NEXT: movl 20(%ecx), %edx
+; X86-NEXT: movl 24(%ecx), %eax
+; X86-NEXT: movl 28(%ecx), %ecx
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 16(%ebp), %ecx
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $3, %al
+; X86-NEXT: andb $28, %al
+; X86-NEXT: negb %al
+; X86-NEXT: movsbl %al, %eax
+; X86-NEXT: movl 68(%esp,%eax), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 72(%esp,%eax), %edx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 76(%esp,%eax), %esi
+; X86-NEXT: movl %esi, %edi
+; X86-NEXT: shldl %cl, %edx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 80(%esp,%eax), %edx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 84(%esp,%eax), %esi
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: shldl %cl, %edx, %ebx
+; X86-NEXT: movl 88(%esp,%eax), %edi
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shldl %cl, %esi, %edx
+; X86-NEXT: movl 64(%esp,%eax), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 92(%esp,%eax), %esi
+; X86-NEXT: shldl %cl, %edi, %esi
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl %esi, 28(%eax)
+; X86-NEXT: movl %edx, 24(%eax)
+; X86-NEXT: movl %ebx, 20(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 16(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 12(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 8(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shll %cl, %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl %cl, %edi, %esi
+; X86-NEXT: movl %esi, 4(%eax)
+; X86-NEXT: movl %edx, (%eax)
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+ %a0 = load i256, ptr %p0
+ %r = shl i256 %a0, %a1
+ ret i256 %r
+}
+
+define i256 @lshr_i256_load(ptr %p0, i256 %a1) nounwind {
+; SSE-LABEL: lshr_i256_load:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdx, %rcx
+; SSE-NEXT: movaps (%rsi), %xmm0
+; SSE-NEXT: movaps 16(%rsi), %xmm1
+; SSE-NEXT: xorps %xmm2, %xmm2
+; SSE-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %ecx, %eax
+; SSE-NEXT: shrb $6, %al
+; SSE-NEXT: movzbl %al, %eax
+; SSE-NEXT: movq -56(%rsp,%rax,8), %rdx
+; SSE-NEXT: movq -64(%rsp,%rax,8), %rsi
+; SSE-NEXT: movq %rsi, %r8
+; SSE-NEXT: shrdq %cl, %rdx, %r8
+; SSE-NEXT: movq -48(%rsp,%rax,8), %r9
+; SSE-NEXT: shrdq %cl, %r9, %rdx
+; SSE-NEXT: movq -72(%rsp,%rax,8), %r10
+; SSE-NEXT: shrdq %cl, %rsi, %r10
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shrq %cl, %r9
+; SSE-NEXT: movq %r9, 24(%rdi)
+; SSE-NEXT: movq %rdx, 16(%rdi)
+; SSE-NEXT: movq %r8, 8(%rdi)
+; SSE-NEXT: movq %r10, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: lshr_i256_load:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: vmovups (%rsi), %ymm0
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: shrb $6, %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX2-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX2-NEXT: movq %rsi, %r8
+; AVX2-NEXT: shrdq %cl, %rdx, %r8
+; AVX2-NEXT: movq -72(%rsp,%rax,8), %r9
+; AVX2-NEXT: movq -48(%rsp,%rax,8), %r10
+; AVX2-NEXT: shrdq %cl, %r10, %rdx
+; AVX2-NEXT: shrdq %cl, %rsi, %r9
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: shrxq %rcx, %r10, %rcx
+; AVX2-NEXT: movq %rcx, 24(%rdi)
+; AVX2-NEXT: movq %rdx, 16(%rdi)
+; AVX2-NEXT: movq %r8, 8(%rdi)
+; AVX2-NEXT: movq %r9, (%rdi)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: lshr_i256_load:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rdx, %rcx
+; AVX512F-NEXT: vmovups (%rsi), %ymm0
+; AVX512F-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %ecx, %eax
+; AVX512F-NEXT: shrb $6, %al
+; AVX512F-NEXT: movzbl %al, %eax
+; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512F-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512F-NEXT: movq %rsi, %r8
+; AVX512F-NEXT: shrdq %cl, %rdx, %r8
+; AVX512F-NEXT: movq -72(%rsp,%rax,8), %r9
+; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r10
+; AVX512F-NEXT: shrdq %cl, %r10, %rdx
+; AVX512F-NEXT: shrdq %cl, %rsi, %r9
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: shrxq %rcx, %r10, %rcx
+; AVX512F-NEXT: movq %rcx, 24(%rdi)
+; AVX512F-NEXT: movq %rdx, 16(%rdi)
+; AVX512F-NEXT: movq %r8, 8(%rdi)
+; AVX512F-NEXT: movq %r9, (%rdi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: lshr_i256_load:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rdx, %rcx
+; AVX512VL-NEXT: vmovups (%rsi), %ymm0
+; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %ecx, %eax
+; AVX512VL-NEXT: shrb $6, %al
+; AVX512VL-NEXT: movzbl %al, %eax
+; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512VL-NEXT: movq %rsi, %r8
+; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r9
+; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
+; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %r10
+; AVX512VL-NEXT: shrdq %cl, %rsi, %r10
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: shrxq %rcx, %r9, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rdi)
+; AVX512VL-NEXT: movq %rdx, 16(%rdi)
+; AVX512VL-NEXT: movq %r8, 8(%rdi)
+; AVX512VL-NEXT: movq %r10, (%rdi)
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: lshr_i256_load:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rdx, %rcx
+; AVX512VBMI-NEXT: vmovups (%rsi), %ymm0
+; AVX512VBMI-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %ecx, %eax
+; AVX512VBMI-NEXT: shrb $6, %al
+; AVX512VBMI-NEXT: movzbl %al, %eax
+; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512VBMI-NEXT: movq %rsi, %r8
+; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r9
+; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
+; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %r10
+; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r10
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: shrxq %rcx, %r9, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
+; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
+; AVX512VBMI-NEXT: movq %r10, (%rdi)
+; AVX512VBMI-NEXT: vzeroupper
+; AVX512VBMI-NEXT: retq
+;
+; X86-LABEL: lshr_i256_load:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $112, %esp
+; X86-NEXT: movl 12(%ebp), %ecx
+; X86-NEXT: movl (%ecx), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 4(%ecx), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 8(%ecx), %esi
+; X86-NEXT: movl 12(%ecx), %edi
+; X86-NEXT: movl 16(%ecx), %ebx
+; X86-NEXT: movl 20(%ecx), %edx
+; X86-NEXT: movl 24(%ecx), %eax
+; X86-NEXT: movl 28(%ecx), %ecx
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 16(%ebp), %ecx
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $5, %al
+; X86-NEXT: movzbl %al, %eax
+; X86-NEXT: movl 40(%esp,%eax,4), %edx
+; X86-NEXT: movl 36(%esp,%eax,4), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 44(%esp,%eax,4), %esi
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 48(%esp,%eax,4), %ebx
+; X86-NEXT: shrdl %cl, %ebx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 52(%esp,%eax,4), %edi
+; X86-NEXT: shrdl %cl, %edi, %ebx
+; X86-NEXT: movl 56(%esp,%eax,4), %esi
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl 32(%esp,%eax,4), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 60(%esp,%eax,4), %edx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrl %cl, %edx
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl %edx, 28(%eax)
+; X86-NEXT: movl %esi, 24(%eax)
+; X86-NEXT: movl %edi, 20(%eax)
+; X86-NEXT: movl %ebx, 16(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 12(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+ %a0 = load i256, ptr %p0
+ %r = lshr i256 %a0, %a1
+ ret i256 %r
+}
+
+define i256 @ashr_i256_load(ptr %p0, i256 %a1) nounwind {
+; SSE-LABEL: ashr_i256_load:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdx, %rcx
+; SSE-NEXT: movaps (%rsi), %xmm0
+; SSE-NEXT: movq 16(%rsi), %rax
+; SSE-NEXT: movq 24(%rsi), %rdx
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: sarq $63, %rdx
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %ecx, %eax
+; SSE-NEXT: shrb $6, %al
+; SSE-NEXT: movzbl %al, %eax
+; SSE-NEXT: movq -56(%rsp,%rax,8), %rdx
+; SSE-NEXT: movq -64(%rsp,%rax,8), %rsi
+; SSE-NEXT: movq %rsi, %r8
+; SSE-NEXT: shrdq %cl, %rdx, %r8
+; SSE-NEXT: movq -48(%rsp,%rax,8), %r9
+; SSE-NEXT: shrdq %cl, %r9, %rdx
+; SSE-NEXT: movq -72(%rsp,%rax,8), %r10
+; SSE-NEXT: shrdq %cl, %rsi, %r10
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: sarq %cl, %r9
+; SSE-NEXT: movq %r9, 24(%rdi)
+; SSE-NEXT: movq %rdx, 16(%rdi)
+; SSE-NEXT: movq %r8, 8(%rdi)
+; SSE-NEXT: movq %r10, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: ashr_i256_load:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: vmovaps (%rsi), %xmm0
+; AVX2-NEXT: movq 16(%rsi), %rax
+; AVX2-NEXT: movq 24(%rsi), %rdx
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: sarq $63, %rdx
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: shrb $6, %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX2-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX2-NEXT: movq %rsi, %r8
+; AVX2-NEXT: shrdq %cl, %rdx, %r8
+; AVX2-NEXT: movq -72(%rsp,%rax,8), %r9
+; AVX2-NEXT: movq -48(%rsp,%rax,8), %r10
+; AVX2-NEXT: shrdq %cl, %r10, %rdx
+; AVX2-NEXT: shrdq %cl, %rsi, %r9
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: sarxq %rcx, %r10, %rcx
+; AVX2-NEXT: movq %rcx, 24(%rdi)
+; AVX2-NEXT: movq %rdx, 16(%rdi)
+; AVX2-NEXT: movq %r8, 8(%rdi)
+; AVX2-NEXT: movq %r9, (%rdi)
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: ashr_i256_load:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rdx, %rcx
+; AVX512F-NEXT: vmovaps (%rsi), %xmm0
+; AVX512F-NEXT: movq 16(%rsi), %rax
+; AVX512F-NEXT: movq 24(%rsi), %rdx
+; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: sarq $63, %rdx
+; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %ecx, %eax
+; AVX512F-NEXT: shrb $6, %al
+; AVX512F-NEXT: movzbl %al, %eax
+; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512F-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512F-NEXT: movq %rsi, %r8
+; AVX512F-NEXT: shrdq %cl, %rdx, %r8
+; AVX512F-NEXT: movq -72(%rsp,%rax,8), %r9
+; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r10
+; AVX512F-NEXT: shrdq %cl, %r10, %rdx
+; AVX512F-NEXT: shrdq %cl, %rsi, %r9
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: sarxq %rcx, %r10, %rcx
+; AVX512F-NEXT: movq %rcx, 24(%rdi)
+; AVX512F-NEXT: movq %rdx, 16(%rdi)
+; AVX512F-NEXT: movq %r8, 8(%rdi)
+; AVX512F-NEXT: movq %r9, (%rdi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: ashr_i256_load:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rdx, %rcx
+; AVX512VL-NEXT: vmovaps (%rsi), %xmm0
+; AVX512VL-NEXT: movq 16(%rsi), %rax
+; AVX512VL-NEXT: movq 24(%rsi), %rdx
+; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: sarq $63, %rdx
+; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %ecx, %eax
+; AVX512VL-NEXT: shrb $6, %al
+; AVX512VL-NEXT: movzbl %al, %eax
+; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512VL-NEXT: movq %rsi, %r8
+; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r9
+; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
+; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %r10
+; AVX512VL-NEXT: shrdq %cl, %rsi, %r10
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: sarxq %rcx, %r9, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rdi)
+; AVX512VL-NEXT: movq %rdx, 16(%rdi)
+; AVX512VL-NEXT: movq %r8, 8(%rdi)
+; AVX512VL-NEXT: movq %r10, (%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: ashr_i256_load:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rdx, %rcx
+; AVX512VBMI-NEXT: vmovaps (%rsi), %xmm0
+; AVX512VBMI-NEXT: movq 16(%rsi), %rax
+; AVX512VBMI-NEXT: movq 24(%rsi), %rdx
+; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: sarq $63, %rdx
+; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %ecx, %eax
+; AVX512VBMI-NEXT: shrb $6, %al
+; AVX512VBMI-NEXT: movzbl %al, %eax
+; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512VBMI-NEXT: movq %rsi, %r8
+; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r9
+; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
+; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %r10
+; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r10
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: sarxq %rcx, %r9, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
+; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
+; AVX512VBMI-NEXT: movq %r10, (%rdi)
+; AVX512VBMI-NEXT: retq
+;
+; X86-LABEL: ashr_i256_load:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $112, %esp
+; X86-NEXT: movl 12(%ebp), %eax
+; X86-NEXT: movl (%eax), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 4(%eax), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 8(%eax), %edi
+; X86-NEXT: movl 12(%eax), %esi
+; X86-NEXT: movl 16(%eax), %ebx
+; X86-NEXT: movl 20(%eax), %edx
+; X86-NEXT: movl 24(%eax), %ecx
+; X86-NEXT: movl 28(%eax), %eax
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 16(%ebp), %ecx
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: sarl $31, %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $5, %al
+; X86-NEXT: movzbl %al, %eax
+; X86-NEXT: movl 40(%esp,%eax,4), %edx
+; X86-NEXT: movl 36(%esp,%eax,4), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 44(%esp,%eax,4), %esi
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 48(%esp,%eax,4), %ebx
+; X86-NEXT: shrdl %cl, %ebx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 52(%esp,%eax,4), %edi
+; X86-NEXT: shrdl %cl, %edi, %ebx
+; X86-NEXT: movl 56(%esp,%eax,4), %esi
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl 32(%esp,%eax,4), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 60(%esp,%eax,4), %edx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: sarl %cl, %edx
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl %edx, 28(%eax)
+; X86-NEXT: movl %esi, 24(%eax)
+; X86-NEXT: movl %edi, 20(%eax)
+; X86-NEXT: movl %ebx, 16(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 12(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+ %a0 = load i256, ptr %p0
+ %r = ashr i256 %a0, %a1
+ ret i256 %r
+}
+
+define i256 @shl_i256_1(i256 %a0) nounwind {
+; CHECK-LABEL: shl_i256_1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shldq $1, %rcx, %r8
+; CHECK-NEXT: shldq $1, %rdx, %rcx
+; CHECK-NEXT: shldq $1, %rsi, %rdx
+; CHECK-NEXT: addq %rsi, %rsi
+; CHECK-NEXT: movq %r8, 24(%rdi)
+; CHECK-NEXT: movq %rcx, 16(%rdi)
+; CHECK-NEXT: movq %rdx, 8(%rdi)
+; CHECK-NEXT: movq %rsi, (%rdi)
+; CHECK-NEXT: retq
+;
+; X86-LABEL: shl_i256_1:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $1, %ecx, %edx
+; X86-NEXT: movl %edx, 28(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $1, %edx, %ecx
+; X86-NEXT: movl %ecx, 24(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shldl $1, %ecx, %edx
+; X86-NEXT: movl %edx, 20(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $1, %edx, %ecx
+; X86-NEXT: movl %ecx, 16(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shldl $1, %ecx, %edx
+; X86-NEXT: movl %edx, 12(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $1, %edx, %ecx
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shldl $1, %ecx, %edx
+; X86-NEXT: movl %edx, 4(%eax)
+; X86-NEXT: addl %ecx, %ecx
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: retl $4
+ %r = shl i256 %a0, 1
+ ret i256 %r
+}
+
+define i256 @lshr_i256_1(i256 %a0) nounwind {
+; CHECK-LABEL: lshr_i256_1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shrdq $1, %rdx, %rsi
+; CHECK-NEXT: shrdq $1, %rcx, %rdx
+; CHECK-NEXT: shrdq $1, %r8, %rcx
+; CHECK-NEXT: shrq %r8
+; CHECK-NEXT: movq %r8, 24(%rdi)
+; CHECK-NEXT: movq %rcx, 16(%rdi)
+; CHECK-NEXT: movq %rdx, 8(%rdi)
+; CHECK-NEXT: movq %rsi, (%rdi)
+; CHECK-NEXT: retq
+;
+; X86-LABEL: lshr_i256_1:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $8, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %ebp, %esi
+; X86-NEXT: shldl $31, %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $31, %ebx, %edx
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NEXT: shldl $31, %ecx, %ebx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: shldl $31, %edi, %esi
+; X86-NEXT: shldl $31, %eax, %edi
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shldl $31, %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shrdl $1, %eax, %ecx
+; X86-NEXT: shrl %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ebp, 28(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NEXT: movl %ebp, 24(%eax)
+; X86-NEXT: movl (%esp), %ebp # 4-byte Reload
+; X86-NEXT: movl %ebp, 20(%eax)
+; X86-NEXT: movl %ebx, 16(%eax)
+; X86-NEXT: movl %esi, 12(%eax)
+; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %edx, 4(%eax)
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: addl $8, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+ %r = lshr i256 %a0, 1
+ ret i256 %r
+}
+
+define i256 @ashr_i256_1(i256 %a0) nounwind {
+; CHECK-LABEL: ashr_i256_1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shrdq $1, %rdx, %rsi
+; CHECK-NEXT: shrdq $1, %rcx, %rdx
+; CHECK-NEXT: shrdq $1, %r8, %rcx
+; CHECK-NEXT: sarq %r8
+; CHECK-NEXT: movq %r8, 24(%rdi)
+; CHECK-NEXT: movq %rcx, 16(%rdi)
+; CHECK-NEXT: movq %rdx, 8(%rdi)
+; CHECK-NEXT: movq %rsi, (%rdi)
+; CHECK-NEXT: retq
+;
+; X86-LABEL: ashr_i256_1:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $8, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %ebp, %esi
+; X86-NEXT: shldl $31, %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $31, %ebx, %edx
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NEXT: shldl $31, %ecx, %ebx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: shldl $31, %edi, %esi
+; X86-NEXT: shldl $31, %eax, %edi
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shldl $31, %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shrdl $1, %eax, %ecx
+; X86-NEXT: sarl %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ebp, 28(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NEXT: movl %ebp, 24(%eax)
+; X86-NEXT: movl (%esp), %ebp # 4-byte Reload
+; X86-NEXT: movl %ebp, 20(%eax)
+; X86-NEXT: movl %ebx, 16(%eax)
+; X86-NEXT: movl %esi, 12(%eax)
+; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %edx, 4(%eax)
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: addl $8, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+ %r = ashr i256 %a0, 1
+ ret i256 %r
+}
+
+define i256 @shl_i256_200(i256 %a0) nounwind {
+; SSE-LABEL: shl_i256_200:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: shlq $8, %rsi
+; SSE-NEXT: movq %rsi, 24(%rdi)
+; SSE-NEXT: xorps %xmm0, %xmm0
+; SSE-NEXT: movaps %xmm0, (%rdi)
+; SSE-NEXT: movq $0, 16(%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: shl_i256_200:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: shlq $8, %rsi
+; AVX2-NEXT: movq %rsi, 24(%rdi)
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovaps %xmm0, (%rdi)
+; AVX2-NEXT: movq $0, 16(%rdi)
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: shl_i256_200:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: shlq $8, %rsi
+; AVX512-NEXT: movq %rsi, 24(%rdi)
+; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vmovaps %xmm0, (%rdi)
+; AVX512-NEXT: movq $0, 16(%rdi)
+; AVX512-NEXT: retq
+;
+; X86-LABEL: shl_i256_200:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shll $8, %ecx
+; X86-NEXT: movl %edx, 28(%eax)
+; X86-NEXT: movl %ecx, 24(%eax)
+; X86-NEXT: movl $0, 20(%eax)
+; X86-NEXT: movl $0, 16(%eax)
+; X86-NEXT: movl $0, 12(%eax)
+; X86-NEXT: movl $0, 8(%eax)
+; X86-NEXT: movl $0, 4(%eax)
+; X86-NEXT: movl $0, (%eax)
+; X86-NEXT: retl $4
+ %r = shl i256 %a0, 200
+ ret i256 %r
+}
+
+define i256 @lshr_i256_200(i256 %a0) nounwind {
+; SSE-LABEL: lshr_i256_200:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: shrq $8, %r8
+; SSE-NEXT: xorps %xmm0, %xmm0
+; SSE-NEXT: movups %xmm0, 8(%rdi)
+; SSE-NEXT: movq %r8, (%rdi)
+; SSE-NEXT: movq $0, 24(%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: lshr_i256_200:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: shrq $8, %r8
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %xmm0, 8(%rdi)
+; AVX2-NEXT: movq %r8, (%rdi)
+; AVX2-NEXT: movq $0, 24(%rdi)
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: lshr_i256_200:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: shrq $8, %r8
+; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vmovups %xmm0, 8(%rdi)
+; AVX512-NEXT: movq %r8, (%rdi)
+; AVX512-NEXT: movq $0, 24(%rdi)
+; AVX512-NEXT: retq
+;
+; X86-LABEL: lshr_i256_200:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shrl $8, %edx
+; X86-NEXT: movl %edx, 4(%eax)
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: movl $0, 28(%eax)
+; X86-NEXT: movl $0, 24(%eax)
+; X86-NEXT: movl $0, 20(%eax)
+; X86-NEXT: movl $0, 16(%eax)
+; X86-NEXT: movl $0, 12(%eax)
+; X86-NEXT: movl $0, 8(%eax)
+; X86-NEXT: retl $4
+ %r = lshr i256 %a0, 200
+ ret i256 %r
+}
+
+define i256 @ashr_i256_200(i256 %a0) nounwind {
+; CHECK-LABEL: ashr_i256_200:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: movq %r8, %rcx
+; CHECK-NEXT: sarq $8, %rcx
+; CHECK-NEXT: sarq $63, %r8
+; CHECK-NEXT: movq %r8, 24(%rdi)
+; CHECK-NEXT: movq %r8, 16(%rdi)
+; CHECK-NEXT: movq %r8, 8(%rdi)
+; CHECK-NEXT: movq %rcx, (%rdi)
+; CHECK-NEXT: retq
+;
+; X86-LABEL: ashr_i256_200:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: sarl $8, %esi
+; X86-NEXT: sarl $31, %edx
+; X86-NEXT: movl %edx, 28(%eax)
+; X86-NEXT: movl %edx, 24(%eax)
+; X86-NEXT: movl %edx, 20(%eax)
+; X86-NEXT: movl %edx, 16(%eax)
+; X86-NEXT: movl %edx, 12(%eax)
+; X86-NEXT: movl %edx, 8(%eax)
+; X86-NEXT: movl %esi, 4(%eax)
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: popl %esi
+; X86-NEXT: retl $4
+ %r = ashr i256 %a0, 200
+ ret i256 %r
}
-define i256 @shift2(i256 %c) nounwind
-; CHECK-LABEL: shift2:
+define i256 @shl_i256_255(i256 %a0) nounwind {
+; SSE-LABEL: shl_i256_255:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: shlq $63, %rsi
+; SSE-NEXT: movq %rsi, 24(%rdi)
+; SSE-NEXT: xorps %xmm0, %xmm0
+; SSE-NEXT: movaps %xmm0, (%rdi)
+; SSE-NEXT: movq $0, 16(%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: shl_i256_255:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: shlq $63, %rsi
+; AVX2-NEXT: movq %rsi, 24(%rdi)
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovaps %xmm0, (%rdi)
+; AVX2-NEXT: movq $0, 16(%rdi)
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: shl_i256_255:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: shlq $63, %rsi
+; AVX512-NEXT: movq %rsi, 24(%rdi)
+; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vmovaps %xmm0, (%rdi)
+; AVX512-NEXT: movq $0, 16(%rdi)
+; AVX512-NEXT: retq
+;
+; X86-LABEL: shl_i256_255:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shll $31, %ecx
+; X86-NEXT: movl %ecx, 28(%eax)
+; X86-NEXT: movl $0, 24(%eax)
+; X86-NEXT: movl $0, 20(%eax)
+; X86-NEXT: movl $0, 16(%eax)
+; X86-NEXT: movl $0, 12(%eax)
+; X86-NEXT: movl $0, 8(%eax)
+; X86-NEXT: movl $0, 4(%eax)
+; X86-NEXT: movl $0, (%eax)
+; X86-NEXT: retl $4
+ %r = shl i256 %a0, 255
+ ret i256 %r
+}
+
+define i256 @lshr_i256_255(i256 %a0) nounwind {
+; SSE-LABEL: lshr_i256_255:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: shrq $63, %r8
+; SSE-NEXT: xorps %xmm0, %xmm0
+; SSE-NEXT: movups %xmm0, 8(%rdi)
+; SSE-NEXT: movq %r8, (%rdi)
+; SSE-NEXT: movq $0, 24(%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: lshr_i256_255:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: shrq $63, %r8
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %xmm0, 8(%rdi)
+; AVX2-NEXT: movq %r8, (%rdi)
+; AVX2-NEXT: movq $0, 24(%rdi)
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: lshr_i256_255:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: shrq $63, %r8
+; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vmovups %xmm0, 8(%rdi)
+; AVX512-NEXT: movq %r8, (%rdi)
+; AVX512-NEXT: movq $0, 24(%rdi)
+; AVX512-NEXT: retq
+;
+; X86-LABEL: lshr_i256_255:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shrl $31, %ecx
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: movl $0, 28(%eax)
+; X86-NEXT: movl $0, 24(%eax)
+; X86-NEXT: movl $0, 20(%eax)
+; X86-NEXT: movl $0, 16(%eax)
+; X86-NEXT: movl $0, 12(%eax)
+; X86-NEXT: movl $0, 8(%eax)
+; X86-NEXT: movl $0, 4(%eax)
+; X86-NEXT: retl $4
+ %r = lshr i256 %a0, 255
+ ret i256 %r
+}
+
+define i256 @ashr_i256_255(i256 %a0) nounwind {
+; CHECK-LABEL: ashr_i256_255:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushl %ebp
-; CHECK-NEXT: movl %esp, %ebp
-; CHECK-NEXT: pushl %ebx
-; CHECK-NEXT: pushl %edi
-; CHECK-NEXT: pushl %esi
-; CHECK-NEXT: andl $-16, %esp
-; CHECK-NEXT: subl $112, %esp
-; CHECK-NEXT: movl 12(%ebp), %ecx
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $1, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp)
-; CHECK-NEXT: movl %ecx, %eax
-; CHECK-NEXT: shrb $3, %al
-; CHECK-NEXT: andb $28, %al
-; CHECK-NEXT: negb %al
-; CHECK-NEXT: movsbl %al, %eax
-; CHECK-NEXT: movl 68(%esp,%eax), %esi
-; CHECK-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl 72(%esp,%eax), %edx
-; CHECK-NEXT: movl %edx, %edi
-; CHECK-NEXT: shldl %cl, %esi, %edi
-; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl 76(%esp,%eax), %esi
-; CHECK-NEXT: movl %esi, %edi
-; CHECK-NEXT: shldl %cl, %edx, %edi
-; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl 80(%esp,%eax), %edx
-; CHECK-NEXT: movl %edx, %edi
-; CHECK-NEXT: shldl %cl, %esi, %edi
-; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl 84(%esp,%eax), %esi
-; CHECK-NEXT: movl %esi, %ebx
-; CHECK-NEXT: shldl %cl, %edx, %ebx
-; CHECK-NEXT: movl 88(%esp,%eax), %edi
-; CHECK-NEXT: movl %edi, %edx
-; CHECK-NEXT: shldl %cl, %esi, %edx
-; CHECK-NEXT: movl 64(%esp,%eax), %esi
-; CHECK-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl 92(%esp,%eax), %esi
-; CHECK-NEXT: shldl %cl, %edi, %esi
-; CHECK-NEXT: movl 8(%ebp), %eax
-; CHECK-NEXT: movl %esi, 28(%eax)
-; CHECK-NEXT: movl %edx, 24(%eax)
-; CHECK-NEXT: movl %ebx, 20(%eax)
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; CHECK-NEXT: movl %edx, 16(%eax)
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; CHECK-NEXT: movl %edx, 12(%eax)
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; CHECK-NEXT: movl %edx, 8(%eax)
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; CHECK-NEXT: movl %edi, %edx
-; CHECK-NEXT: shll %cl, %edx
-; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; CHECK-NEXT: shldl %cl, %edi, %esi
-; CHECK-NEXT: movl %esi, 4(%eax)
-; CHECK-NEXT: movl %edx, (%eax)
-; CHECK-NEXT: leal -12(%ebp), %esp
-; CHECK-NEXT: popl %esi
-; CHECK-NEXT: popl %edi
-; CHECK-NEXT: popl %ebx
-; CHECK-NEXT: popl %ebp
-; CHECK-NEXT: retl $4
-;
-; CHECK-X64-O0-LABEL: shift2:
-; CHECK-X64-O0: # %bb.0:
-; CHECK-X64-O0-NEXT: movq %rdi, %rax
-; CHECK-X64-O0-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movq $1, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O0-NEXT: movb %sil, %cl
-; CHECK-X64-O0-NEXT: movb %cl, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
-; CHECK-X64-O0-NEXT: movb %cl, %dl
-; CHECK-X64-O0-NEXT: shrb $3, %dl
-; CHECK-X64-O0-NEXT: andb $24, %dl
-; CHECK-X64-O0-NEXT: negb %dl
-; CHECK-X64-O0-NEXT: movsbq %dl, %r8
-; CHECK-X64-O0-NEXT: movq -40(%rsp,%r8), %r9
-; CHECK-X64-O0-NEXT: movq -32(%rsp,%r8), %rdx
-; CHECK-X64-O0-NEXT: movq -24(%rsp,%r8), %r10
-; CHECK-X64-O0-NEXT: movq %r10, %rsi
-; CHECK-X64-O0-NEXT: shldq %cl, %rdx, %rsi
-; CHECK-X64-O0-NEXT: movb {{[-0-9]+}}(%r{{[sb]}}p), %cl # 1-byte Reload
-; CHECK-X64-O0-NEXT: movq -16(%rsp,%r8), %r8
-; CHECK-X64-O0-NEXT: shldq %cl, %r10, %r8
-; CHECK-X64-O0-NEXT: movb {{[-0-9]+}}(%r{{[sb]}}p), %cl # 1-byte Reload
-; CHECK-X64-O0-NEXT: movq %r9, %r10
-; CHECK-X64-O0-NEXT: shlq %cl, %r10
-; CHECK-X64-O0-NEXT: movb {{[-0-9]+}}(%r{{[sb]}}p), %cl # 1-byte Reload
-; CHECK-X64-O0-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-X64-O0-NEXT: shldq %cl, %r9, %rdx
-; CHECK-X64-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-X64-O0-NEXT: movq %r8, 24(%rdi)
-; CHECK-X64-O0-NEXT: movq %rsi, 16(%rdi)
-; CHECK-X64-O0-NEXT: movq %rdx, 8(%rdi)
-; CHECK-X64-O0-NEXT: movq %rcx, (%rdi)
-; CHECK-X64-O0-NEXT: retq
-;
-; CHECK-X64-O2-LABEL: shift2:
-; CHECK-X64-O2: # %bb.0:
-; CHECK-X64-O2-NEXT: movq %rsi, %rcx
-; CHECK-X64-O2-NEXT: movq %rdi, %rax
-; CHECK-X64-O2-NEXT: xorps %xmm0, %xmm0
-; CHECK-X64-O2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O2-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O2-NEXT: movq $1, -{{[0-9]+}}(%rsp)
-; CHECK-X64-O2-NEXT: movl %ecx, %edx
-; CHECK-X64-O2-NEXT: shrb $3, %dl
-; CHECK-X64-O2-NEXT: andb $24, %dl
-; CHECK-X64-O2-NEXT: negb %dl
-; CHECK-X64-O2-NEXT: movsbq %dl, %rdx
-; CHECK-X64-O2-NEXT: movq -40(%rsp,%rdx), %rsi
-; CHECK-X64-O2-NEXT: movq -32(%rsp,%rdx), %rdi
-; CHECK-X64-O2-NEXT: movq -24(%rsp,%rdx), %r8
-; CHECK-X64-O2-NEXT: movq %r8, %r9
-; CHECK-X64-O2-NEXT: shldq %cl, %rdi, %r9
-; CHECK-X64-O2-NEXT: movq -16(%rsp,%rdx), %rdx
-; CHECK-X64-O2-NEXT: shldq %cl, %r8, %rdx
-; CHECK-X64-O2-NEXT: movq %rsi, %r8
-; CHECK-X64-O2-NEXT: shlq %cl, %r8
-; CHECK-X64-O2-NEXT: # kill: def $cl killed $cl killed $rcx
-; CHECK-X64-O2-NEXT: shldq %cl, %rsi, %rdi
-; CHECK-X64-O2-NEXT: movq %rdx, 24(%rax)
-; CHECK-X64-O2-NEXT: movq %r9, 16(%rax)
-; CHECK-X64-O2-NEXT: movq %rdi, 8(%rax)
-; CHECK-X64-O2-NEXT: movq %r8, (%rax)
-; CHECK-X64-O2-NEXT: retq
-{
- %b = shl i256 1, %c ; %c must not be a constant
- ; Special case when %c is 0:
- ret i256 %b
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: sarq $63, %r8
+; CHECK-NEXT: movq %r8, 24(%rdi)
+; CHECK-NEXT: movq %r8, 16(%rdi)
+; CHECK-NEXT: movq %r8, 8(%rdi)
+; CHECK-NEXT: movq %r8, (%rdi)
+; CHECK-NEXT: retq
+;
+; X86-LABEL: ashr_i256_255:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: sarl $31, %ecx
+; X86-NEXT: movl %ecx, 28(%eax)
+; X86-NEXT: movl %ecx, 24(%eax)
+; X86-NEXT: movl %ecx, 20(%eax)
+; X86-NEXT: movl %ecx, 16(%eax)
+; X86-NEXT: movl %ecx, 12(%eax)
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: retl $4
+ %r = ashr i256 %a0, 255
+ ret i256 %r
+}
+
+define i256 @shl_1_i256(i256 %a0) nounwind {
+; SSE-LABEL: shl_1_i256:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rsi, %rcx
+; SSE-NEXT: xorps %xmm0, %xmm0
+; SSE-NEXT: movups %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq $0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq $1, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %ecx, %eax
+; SSE-NEXT: shrb $3, %al
+; SSE-NEXT: andb $24, %al
+; SSE-NEXT: negb %al
+; SSE-NEXT: movsbq %al, %rax
+; SSE-NEXT: movq -32(%rsp,%rax), %rdx
+; SSE-NEXT: movq -24(%rsp,%rax), %rsi
+; SSE-NEXT: movq %rsi, %r8
+; SSE-NEXT: shldq %cl, %rdx, %r8
+; SSE-NEXT: movq -16(%rsp,%rax), %r9
+; SSE-NEXT: shldq %cl, %rsi, %r9
+; SSE-NEXT: movq -40(%rsp,%rax), %rax
+; SSE-NEXT: movq %rax, %rsi
+; SSE-NEXT: shlq %cl, %rsi
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shldq %cl, %rax, %rdx
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq %r9, 24(%rdi)
+; SSE-NEXT: movq %r8, 16(%rdi)
+; SSE-NEXT: movq %rdx, 8(%rdi)
+; SSE-NEXT: movq %rsi, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: shl_1_i256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovss {{.*#+}} xmm0 = [1,0,0,0]
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: shrb $3, %al
+; AVX2-NEXT: andb $24, %al
+; AVX2-NEXT: negb %al
+; AVX2-NEXT: movsbq %al, %rdx
+; AVX2-NEXT: movq -32(%rsp,%rdx), %rsi
+; AVX2-NEXT: movq -24(%rsp,%rdx), %rax
+; AVX2-NEXT: movq %rax, %r8
+; AVX2-NEXT: shldq %cl, %rsi, %r8
+; AVX2-NEXT: movq -16(%rsp,%rdx), %r9
+; AVX2-NEXT: shldq %cl, %rax, %r9
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq -40(%rsp,%rdx), %rdx
+; AVX2-NEXT: shlxq %rcx, %rdx, %rdi
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shldq %cl, %rdx, %rsi
+; AVX2-NEXT: movq %r9, 24(%rax)
+; AVX2-NEXT: movq %r8, 16(%rax)
+; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: shl_1_i256:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: vmovaps {{.*#+}} zmm0 = [0,0,0,0,1,0,0,0]
+; AVX512F-NEXT: vmovups %zmm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %ecx, %eax
+; AVX512F-NEXT: shrb $3, %al
+; AVX512F-NEXT: andb $24, %al
+; AVX512F-NEXT: negb %al
+; AVX512F-NEXT: movsbq %al, %rdx
+; AVX512F-NEXT: movq -32(%rsp,%rdx), %rsi
+; AVX512F-NEXT: movq -24(%rsp,%rdx), %rax
+; AVX512F-NEXT: movq %rax, %r8
+; AVX512F-NEXT: shldq %cl, %rsi, %r8
+; AVX512F-NEXT: movq -16(%rsp,%rdx), %r9
+; AVX512F-NEXT: shldq %cl, %rax, %r9
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: movq -40(%rsp,%rdx), %rdx
+; AVX512F-NEXT: shlxq %rcx, %rdx, %rdi
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shldq %cl, %rdx, %rsi
+; AVX512F-NEXT: movq %r9, 24(%rax)
+; AVX512F-NEXT: movq %r8, 16(%rax)
+; AVX512F-NEXT: movq %rsi, 8(%rax)
+; AVX512F-NEXT: movq %rdi, (%rax)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: shl_1_i256:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %ecx, %eax
+; AVX512VL-NEXT: shrb $3, %al
+; AVX512VL-NEXT: andb $24, %al
+; AVX512VL-NEXT: negb %al
+; AVX512VL-NEXT: movsbq %al, %rax
+; AVX512VL-NEXT: movq -32(%rsp,%rax), %rdx
+; AVX512VL-NEXT: movq -24(%rsp,%rax), %rsi
+; AVX512VL-NEXT: movq %rsi, %r8
+; AVX512VL-NEXT: shldq %cl, %rdx, %r8
+; AVX512VL-NEXT: movq -16(%rsp,%rax), %r9
+; AVX512VL-NEXT: shldq %cl, %rsi, %r9
+; AVX512VL-NEXT: movq -40(%rsp,%rax), %rsi
+; AVX512VL-NEXT: shldq %cl, %rsi, %rdx
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: shlxq %rcx, %rsi, %rcx
+; AVX512VL-NEXT: movq %r9, 24(%rdi)
+; AVX512VL-NEXT: movq %r8, 16(%rdi)
+; AVX512VL-NEXT: movq %rdx, 8(%rdi)
+; AVX512VL-NEXT: movq %rcx, (%rdi)
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: shl_1_i256:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %ecx, %eax
+; AVX512VBMI-NEXT: shrb $3, %al
+; AVX512VBMI-NEXT: andb $24, %al
+; AVX512VBMI-NEXT: negb %al
+; AVX512VBMI-NEXT: movsbq %al, %rax
+; AVX512VBMI-NEXT: movq -32(%rsp,%rax), %rdx
+; AVX512VBMI-NEXT: movq -24(%rsp,%rax), %rsi
+; AVX512VBMI-NEXT: movq %rsi, %r8
+; AVX512VBMI-NEXT: shldq %cl, %rdx, %r8
+; AVX512VBMI-NEXT: movq -16(%rsp,%rax), %r9
+; AVX512VBMI-NEXT: shldq %cl, %rsi, %r9
+; AVX512VBMI-NEXT: movq -40(%rsp,%rax), %rsi
+; AVX512VBMI-NEXT: shldq %cl, %rsi, %rdx
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: shlxq %rcx, %rsi, %rcx
+; AVX512VBMI-NEXT: movq %r9, 24(%rdi)
+; AVX512VBMI-NEXT: movq %r8, 16(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, 8(%rdi)
+; AVX512VBMI-NEXT: movq %rcx, (%rdi)
+; AVX512VBMI-NEXT: vzeroupper
+; AVX512VBMI-NEXT: retq
+;
+; X86-LABEL: shl_1_i256:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $112, %esp
+; X86-NEXT: movl 12(%ebp), %ecx
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $1, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $3, %al
+; X86-NEXT: andb $28, %al
+; X86-NEXT: negb %al
+; X86-NEXT: movsbl %al, %eax
+; X86-NEXT: movl 68(%esp,%eax), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 72(%esp,%eax), %edx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 76(%esp,%eax), %esi
+; X86-NEXT: movl %esi, %edi
+; X86-NEXT: shldl %cl, %edx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 80(%esp,%eax), %edx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 84(%esp,%eax), %esi
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: shldl %cl, %edx, %ebx
+; X86-NEXT: movl 88(%esp,%eax), %edi
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shldl %cl, %esi, %edx
+; X86-NEXT: movl 64(%esp,%eax), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 92(%esp,%eax), %esi
+; X86-NEXT: shldl %cl, %edi, %esi
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl %esi, 28(%eax)
+; X86-NEXT: movl %edx, 24(%eax)
+; X86-NEXT: movl %ebx, 20(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 16(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 12(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 8(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shll %cl, %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl %cl, %edi, %esi
+; X86-NEXT: movl %esi, 4(%eax)
+; X86-NEXT: movl %edx, (%eax)
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+ %r = shl i256 1, %a0
+ ret i256 %r
+}
+
+define i256 @lshr_signbit_i256(i256 %a0) nounwind {
+; SSE-LABEL: lshr_signbit_i256:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rsi, %rcx
+; SSE-NEXT: xorps %xmm0, %xmm0
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; SSE-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq $0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %ecx, %eax
+; SSE-NEXT: shrb $6, %al
+; SSE-NEXT: movzbl %al, %eax
+; SSE-NEXT: movq -56(%rsp,%rax,8), %rdx
+; SSE-NEXT: movq -64(%rsp,%rax,8), %rsi
+; SSE-NEXT: movq %rsi, %r8
+; SSE-NEXT: shrdq %cl, %rdx, %r8
+; SSE-NEXT: movq -48(%rsp,%rax,8), %r9
+; SSE-NEXT: shrdq %cl, %r9, %rdx
+; SSE-NEXT: movq -72(%rsp,%rax,8), %r10
+; SSE-NEXT: shrdq %cl, %rsi, %r10
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shrq %cl, %r9
+; SSE-NEXT: movq %r9, 24(%rdi)
+; SSE-NEXT: movq %rdx, 16(%rdi)
+; SSE-NEXT: movq %r8, 8(%rdi)
+; SSE-NEXT: movq %r10, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: lshr_signbit_i256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: shrb $6, %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX2-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX2-NEXT: movq %rsi, %r8
+; AVX2-NEXT: shrdq %cl, %rdx, %r8
+; AVX2-NEXT: movq -72(%rsp,%rax,8), %r9
+; AVX2-NEXT: movq -48(%rsp,%rax,8), %r10
+; AVX2-NEXT: shrdq %cl, %r10, %rdx
+; AVX2-NEXT: shrdq %cl, %rsi, %r9
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: shrxq %rcx, %r10, %rcx
+; AVX2-NEXT: movq %rcx, 24(%rdi)
+; AVX2-NEXT: movq %rdx, 16(%rdi)
+; AVX2-NEXT: movq %r8, 8(%rdi)
+; AVX2-NEXT: movq %r9, (%rdi)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: lshr_signbit_i256:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: vmovaps {{.*#+}} zmm0 = [0,0,0,9223372036854775808,0,0,0,0]
+; AVX512F-NEXT: vmovups %zmm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %ecx, %eax
+; AVX512F-NEXT: shrb $6, %al
+; AVX512F-NEXT: movzbl %al, %eax
+; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512F-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512F-NEXT: movq %rsi, %r8
+; AVX512F-NEXT: shrdq %cl, %rdx, %r8
+; AVX512F-NEXT: movq -72(%rsp,%rax,8), %r9
+; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r10
+; AVX512F-NEXT: shrdq %cl, %r10, %rdx
+; AVX512F-NEXT: shrdq %cl, %rsi, %r9
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: shrxq %rcx, %r10, %rcx
+; AVX512F-NEXT: movq %rcx, 24(%rdi)
+; AVX512F-NEXT: movq %rdx, 16(%rdi)
+; AVX512F-NEXT: movq %r8, 8(%rdi)
+; AVX512F-NEXT: movq %r9, (%rdi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: lshr_signbit_i256:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %ecx, %eax
+; AVX512VL-NEXT: shrb $6, %al
+; AVX512VL-NEXT: movzbl %al, %eax
+; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512VL-NEXT: movq %rsi, %r8
+; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r9
+; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
+; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %r10
+; AVX512VL-NEXT: shrdq %cl, %rsi, %r10
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: shrxq %rcx, %r9, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rdi)
+; AVX512VL-NEXT: movq %rdx, 16(%rdi)
+; AVX512VL-NEXT: movq %r8, 8(%rdi)
+; AVX512VL-NEXT: movq %r10, (%rdi)
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: lshr_signbit_i256:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %ecx, %eax
+; AVX512VBMI-NEXT: shrb $6, %al
+; AVX512VBMI-NEXT: movzbl %al, %eax
+; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512VBMI-NEXT: movq %rsi, %r8
+; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r9
+; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
+; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %r10
+; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r10
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: shrxq %rcx, %r9, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
+; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
+; AVX512VBMI-NEXT: movq %r10, (%rdi)
+; AVX512VBMI-NEXT: vzeroupper
+; AVX512VBMI-NEXT: retq
+;
+; X86-LABEL: lshr_signbit_i256:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $112, %esp
+; X86-NEXT: movl 12(%ebp), %ecx
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $-2147483648, {{[0-9]+}}(%esp) # imm = 0x80000000
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $5, %al
+; X86-NEXT: movzbl %al, %eax
+; X86-NEXT: movl 40(%esp,%eax,4), %edx
+; X86-NEXT: movl 36(%esp,%eax,4), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 44(%esp,%eax,4), %esi
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 48(%esp,%eax,4), %ebx
+; X86-NEXT: shrdl %cl, %ebx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 52(%esp,%eax,4), %edi
+; X86-NEXT: shrdl %cl, %edi, %ebx
+; X86-NEXT: movl 56(%esp,%eax,4), %esi
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl 32(%esp,%eax,4), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 60(%esp,%eax,4), %edx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrl %cl, %edx
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl %edx, 28(%eax)
+; X86-NEXT: movl %esi, 24(%eax)
+; X86-NEXT: movl %edi, 20(%eax)
+; X86-NEXT: movl %ebx, 16(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 12(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+ %s = shl i256 1, 255
+ %r = lshr i256 %s, %a0
+ ret i256 %r
+}
+
+define i256 @ashr_signbit_i256(i256 %a0) nounwind {
+; SSE-LABEL: ashr_signbit_i256:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rsi, %rcx
+; SSE-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; SSE-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: xorps %xmm0, %xmm0
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq $-1, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq $-1, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq $-1, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq $-1, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq $0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %ecx, %eax
+; SSE-NEXT: shrb $6, %al
+; SSE-NEXT: movzbl %al, %eax
+; SSE-NEXT: movq -56(%rsp,%rax,8), %rdx
+; SSE-NEXT: movq -64(%rsp,%rax,8), %rsi
+; SSE-NEXT: movq %rsi, %r8
+; SSE-NEXT: shrdq %cl, %rdx, %r8
+; SSE-NEXT: movq -48(%rsp,%rax,8), %r9
+; SSE-NEXT: shrdq %cl, %r9, %rdx
+; SSE-NEXT: movq -72(%rsp,%rax,8), %r10
+; SSE-NEXT: shrdq %cl, %rsi, %r10
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: sarq %cl, %r9
+; SSE-NEXT: movq %r9, 24(%rdi)
+; SSE-NEXT: movq %rdx, 16(%rdi)
+; SSE-NEXT: movq %r8, 8(%rdi)
+; SSE-NEXT: movq %r10, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: ashr_signbit_i256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: shrb $6, %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX2-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX2-NEXT: movq %rsi, %r8
+; AVX2-NEXT: shrdq %cl, %rdx, %r8
+; AVX2-NEXT: movq -72(%rsp,%rax,8), %r9
+; AVX2-NEXT: movq -48(%rsp,%rax,8), %r10
+; AVX2-NEXT: shrdq %cl, %r10, %rdx
+; AVX2-NEXT: shrdq %cl, %rsi, %r9
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: sarxq %rcx, %r10, %rcx
+; AVX2-NEXT: movq %rcx, 24(%rdi)
+; AVX2-NEXT: movq %rdx, 16(%rdi)
+; AVX2-NEXT: movq %r8, 8(%rdi)
+; AVX2-NEXT: movq %r9, (%rdi)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: ashr_signbit_i256:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: vmovaps {{.*#+}} zmm0 = [0,0,0,9223372036854775808,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615]
+; AVX512F-NEXT: vmovups %zmm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %ecx, %eax
+; AVX512F-NEXT: shrb $6, %al
+; AVX512F-NEXT: movzbl %al, %eax
+; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512F-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512F-NEXT: movq %rsi, %r8
+; AVX512F-NEXT: shrdq %cl, %rdx, %r8
+; AVX512F-NEXT: movq -72(%rsp,%rax,8), %r9
+; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r10
+; AVX512F-NEXT: shrdq %cl, %r10, %rdx
+; AVX512F-NEXT: shrdq %cl, %rsi, %r9
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: sarxq %rcx, %r10, %rcx
+; AVX512F-NEXT: movq %rcx, 24(%rdi)
+; AVX512F-NEXT: movq %rdx, 16(%rdi)
+; AVX512F-NEXT: movq %r8, 8(%rdi)
+; AVX512F-NEXT: movq %r9, (%rdi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: ashr_signbit_i256:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %ecx, %eax
+; AVX512VL-NEXT: shrb $6, %al
+; AVX512VL-NEXT: movzbl %al, %eax
+; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512VL-NEXT: movq %rsi, %r8
+; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r9
+; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
+; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %r10
+; AVX512VL-NEXT: shrdq %cl, %rsi, %r10
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: sarxq %rcx, %r9, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rdi)
+; AVX512VL-NEXT: movq %rdx, 16(%rdi)
+; AVX512VL-NEXT: movq %r8, 8(%rdi)
+; AVX512VL-NEXT: movq %r10, (%rdi)
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: ashr_signbit_i256:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %ecx, %eax
+; AVX512VBMI-NEXT: shrb $6, %al
+; AVX512VBMI-NEXT: movzbl %al, %eax
+; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %rdx
+; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %rsi
+; AVX512VBMI-NEXT: movq %rsi, %r8
+; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
+; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r9
+; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
+; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %r10
+; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r10
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: sarxq %rcx, %r9, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rdi)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
+; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
+; AVX512VBMI-NEXT: movq %r10, (%rdi)
+; AVX512VBMI-NEXT: vzeroupper
+; AVX512VBMI-NEXT: retq
+;
+; X86-LABEL: ashr_signbit_i256:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $112, %esp
+; X86-NEXT: movl 12(%ebp), %ecx
+; X86-NEXT: movl $-1, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $-1, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $-1, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $-1, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $-1, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $-1, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $-1, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $-1, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $-2147483648, {{[0-9]+}}(%esp) # imm = 0x80000000
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $5, %al
+; X86-NEXT: movzbl %al, %eax
+; X86-NEXT: movl 40(%esp,%eax,4), %edx
+; X86-NEXT: movl 36(%esp,%eax,4), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 44(%esp,%eax,4), %esi
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 48(%esp,%eax,4), %ebx
+; X86-NEXT: shrdl %cl, %ebx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 52(%esp,%eax,4), %edi
+; X86-NEXT: shrdl %cl, %edi, %ebx
+; X86-NEXT: movl 56(%esp,%eax,4), %esi
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl 32(%esp,%eax,4), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 60(%esp,%eax,4), %edx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: sarl %cl, %edx
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl %edx, 28(%eax)
+; X86-NEXT: movl %esi, 24(%eax)
+; X86-NEXT: movl %edi, 20(%eax)
+; X86-NEXT: movl %ebx, 16(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 12(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+ %s = shl i256 1, 255
+ %r = ashr i256 %s, %a0
+ ret i256 %r
+}
+
+define i64 @lshr_extract_i256_i64(i256 %a0, i256 %a1) nounwind {
+; SSE-LABEL: lshr_extract_i256_i64:
+; SSE: # %bb.0:
+; SSE-NEXT: xorps %xmm0, %xmm0
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %r8d, %eax
+; SSE-NEXT: shrb $6, %al
+; SSE-NEXT: movzbl %al, %ecx
+; SSE-NEXT: movq -72(%rsp,%rcx,8), %rax
+; SSE-NEXT: movq -64(%rsp,%rcx,8), %rdx
+; SSE-NEXT: movl %r8d, %ecx
+; SSE-NEXT: shrdq %cl, %rdx, %rax
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: lshr_extract_i256_i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %r8d, %eax
+; AVX2-NEXT: shrb $6, %al
+; AVX2-NEXT: movzbl %al, %ecx
+; AVX2-NEXT: movq -72(%rsp,%rcx,8), %rax
+; AVX2-NEXT: movq -64(%rsp,%rcx,8), %rdx
+; AVX2-NEXT: movl %r8d, %ecx
+; AVX2-NEXT: shrdq %cl, %rdx, %rax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: lshr_extract_i256_i64:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %r8d, %eax
+; AVX512F-NEXT: shrb $6, %al
+; AVX512F-NEXT: movzbl %al, %ecx
+; AVX512F-NEXT: movq -72(%rsp,%rcx,8), %rax
+; AVX512F-NEXT: movq -64(%rsp,%rcx,8), %rdx
+; AVX512F-NEXT: movl %r8d, %ecx
+; AVX512F-NEXT: shrdq %cl, %rdx, %rax
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: lshr_extract_i256_i64:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %r8d, %eax
+; AVX512VL-NEXT: shrb $6, %al
+; AVX512VL-NEXT: movzbl %al, %ecx
+; AVX512VL-NEXT: movq -72(%rsp,%rcx,8), %rax
+; AVX512VL-NEXT: movq -64(%rsp,%rcx,8), %rdx
+; AVX512VL-NEXT: movl %r8d, %ecx
+; AVX512VL-NEXT: shrdq %cl, %rdx, %rax
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: lshr_extract_i256_i64:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %r8d, %eax
+; AVX512VBMI-NEXT: shrb $6, %al
+; AVX512VBMI-NEXT: movzbl %al, %ecx
+; AVX512VBMI-NEXT: movq -72(%rsp,%rcx,8), %rax
+; AVX512VBMI-NEXT: movq -64(%rsp,%rcx,8), %rdx
+; AVX512VBMI-NEXT: movl %r8d, %ecx
+; AVX512VBMI-NEXT: shrdq %cl, %rdx, %rax
+; AVX512VBMI-NEXT: vzeroupper
+; AVX512VBMI-NEXT: retq
+;
+; X86-LABEL: lshr_extract_i256_i64:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $64, %esp
+; X86-NEXT: movl 40(%ebp), %ecx
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl 12(%ebp), %edx
+; X86-NEXT: movl 16(%ebp), %esi
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 20(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, (%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $5, %al
+; X86-NEXT: movzbl %al, %edx
+; X86-NEXT: movl 8(%esp,%edx,4), %esi
+; X86-NEXT: movl (%esp,%edx,4), %eax
+; X86-NEXT: movl 4(%esp,%edx,4), %edi
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edi, %eax
+; X86-NEXT: leal -8(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl
+ %b = lshr i256 %a0, %a1
+ %r = trunc i256 %b to i64
+ ret i64 %r
+}
+
+define i64 @ashr_extract_i256_i64(i256 %a0, i256 %a1) nounwind {
+; CHECK-LABEL: ashr_extract_i256_i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT: sarq $63, %rcx
+; CHECK-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT: movl %r8d, %eax
+; CHECK-NEXT: shrb $6, %al
+; CHECK-NEXT: movzbl %al, %ecx
+; CHECK-NEXT: movq -72(%rsp,%rcx,8), %rax
+; CHECK-NEXT: movq -64(%rsp,%rcx,8), %rdx
+; CHECK-NEXT: movl %r8d, %ecx
+; CHECK-NEXT: shrdq %cl, %rdx, %rax
+; CHECK-NEXT: retq
+;
+; X86-LABEL: ashr_extract_i256_i64:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $64, %esp
+; X86-NEXT: movl 40(%ebp), %ecx
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl 12(%ebp), %edx
+; X86-NEXT: movl 16(%ebp), %esi
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 20(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, (%esp)
+; X86-NEXT: sarl $31, %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $5, %al
+; X86-NEXT: movzbl %al, %edx
+; X86-NEXT: movl 8(%esp,%edx,4), %esi
+; X86-NEXT: movl (%esp,%edx,4), %eax
+; X86-NEXT: movl 4(%esp,%edx,4), %edi
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edi, %eax
+; X86-NEXT: leal -8(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl
+ %b = ashr i256 %a0, %a1
+ %r = trunc i256 %b to i64
+ ret i64 %r
+}
+
+define i64 @lshr_extract_load_i256_i64(ptr %p0, i256 %a1) nounwind {
+; SSE-LABEL: lshr_extract_load_i256_i64:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rsi, %rcx
+; SSE-NEXT: movaps (%rdi), %xmm0
+; SSE-NEXT: movaps 16(%rdi), %xmm1
+; SSE-NEXT: xorps %xmm2, %xmm2
+; SSE-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %ecx, %eax
+; SSE-NEXT: shrb $6, %al
+; SSE-NEXT: movzbl %al, %edx
+; SSE-NEXT: movq -72(%rsp,%rdx,8), %rax
+; SSE-NEXT: movq -64(%rsp,%rdx,8), %rdx
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shrdq %cl, %rdx, %rax
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: lshr_extract_load_i256_i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: vmovups (%rdi), %ymm0
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: shrb $6, %al
+; AVX2-NEXT: movzbl %al, %edx
+; AVX2-NEXT: movq -72(%rsp,%rdx,8), %rax
+; AVX2-NEXT: movq -64(%rsp,%rdx,8), %rdx
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shrdq %cl, %rdx, %rax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: lshr_extract_load_i256_i64:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: vmovups (%rdi), %ymm0
+; AVX512F-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %ecx, %eax
+; AVX512F-NEXT: shrb $6, %al
+; AVX512F-NEXT: movzbl %al, %edx
+; AVX512F-NEXT: movq -72(%rsp,%rdx,8), %rax
+; AVX512F-NEXT: movq -64(%rsp,%rdx,8), %rdx
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shrdq %cl, %rdx, %rax
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: lshr_extract_load_i256_i64:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: vmovups (%rdi), %ymm0
+; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %ecx, %eax
+; AVX512VL-NEXT: shrb $6, %al
+; AVX512VL-NEXT: movzbl %al, %edx
+; AVX512VL-NEXT: movq -72(%rsp,%rdx,8), %rax
+; AVX512VL-NEXT: movq -64(%rsp,%rdx,8), %rdx
+; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VL-NEXT: shrdq %cl, %rdx, %rax
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: lshr_extract_load_i256_i64:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: vmovups (%rdi), %ymm0
+; AVX512VBMI-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %ecx, %eax
+; AVX512VBMI-NEXT: shrb $6, %al
+; AVX512VBMI-NEXT: movzbl %al, %edx
+; AVX512VBMI-NEXT: movq -72(%rsp,%rdx,8), %rax
+; AVX512VBMI-NEXT: movq -64(%rsp,%rdx,8), %rdx
+; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VBMI-NEXT: shrdq %cl, %rdx, %rax
+; AVX512VBMI-NEXT: vzeroupper
+; AVX512VBMI-NEXT: retq
+;
+; X86-LABEL: lshr_extract_load_i256_i64:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $96, %esp
+; X86-NEXT: movl 8(%ebp), %ecx
+; X86-NEXT: movl (%ecx), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 4(%ecx), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 8(%ecx), %esi
+; X86-NEXT: movl 12(%ecx), %edi
+; X86-NEXT: movl 16(%ecx), %ebx
+; X86-NEXT: movl 20(%ecx), %edx
+; X86-NEXT: movl 24(%ecx), %eax
+; X86-NEXT: movl 28(%ecx), %ecx
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 12(%ebp), %ecx
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $5, %al
+; X86-NEXT: movzbl %al, %edx
+; X86-NEXT: movl 24(%esp,%edx,4), %esi
+; X86-NEXT: movl 16(%esp,%edx,4), %eax
+; X86-NEXT: movl 20(%esp,%edx,4), %edi
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edi, %eax
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl
+ %a0 = load i256, ptr %p0
+ %b = lshr i256 %a0, %a1
+ %r = trunc i256 %b to i64
+ ret i64 %r
+}
+
+define i64 @ashr_extract_load_i256_i64(ptr %p0, i256 %a1) nounwind {
+; SSE-LABEL: ashr_extract_load_i256_i64:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rsi, %rcx
+; SSE-NEXT: movaps (%rdi), %xmm0
+; SSE-NEXT: movq 16(%rdi), %rax
+; SSE-NEXT: movq 24(%rdi), %rdx
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: sarq $63, %rdx
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %ecx, %eax
+; SSE-NEXT: shrb $6, %al
+; SSE-NEXT: movzbl %al, %edx
+; SSE-NEXT: movq -72(%rsp,%rdx,8), %rax
+; SSE-NEXT: movq -64(%rsp,%rdx,8), %rdx
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shrdq %cl, %rdx, %rax
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: ashr_extract_load_i256_i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: vmovaps (%rdi), %xmm0
+; AVX2-NEXT: movq 16(%rdi), %rax
+; AVX2-NEXT: movq 24(%rdi), %rdx
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: sarq $63, %rdx
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: shrb $6, %al
+; AVX2-NEXT: movzbl %al, %edx
+; AVX2-NEXT: movq -72(%rsp,%rdx,8), %rax
+; AVX2-NEXT: movq -64(%rsp,%rdx,8), %rdx
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shrdq %cl, %rdx, %rax
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: ashr_extract_load_i256_i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movq %rsi, %rcx
+; AVX512-NEXT: vmovaps (%rdi), %xmm0
+; AVX512-NEXT: movq 16(%rdi), %rax
+; AVX512-NEXT: movq 24(%rdi), %rdx
+; AVX512-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: sarq $63, %rdx
+; AVX512-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: movl %ecx, %eax
+; AVX512-NEXT: shrb $6, %al
+; AVX512-NEXT: movzbl %al, %edx
+; AVX512-NEXT: movq -72(%rsp,%rdx,8), %rax
+; AVX512-NEXT: movq -64(%rsp,%rdx,8), %rdx
+; AVX512-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512-NEXT: shrdq %cl, %rdx, %rax
+; AVX512-NEXT: retq
+;
+; X86-LABEL: ashr_extract_load_i256_i64:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $96, %esp
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl (%eax), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 4(%eax), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 8(%eax), %edi
+; X86-NEXT: movl 12(%eax), %esi
+; X86-NEXT: movl 16(%eax), %ebx
+; X86-NEXT: movl 20(%eax), %edx
+; X86-NEXT: movl 24(%eax), %ecx
+; X86-NEXT: movl 28(%eax), %eax
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 12(%ebp), %ecx
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: sarl $31, %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $5, %al
+; X86-NEXT: movzbl %al, %edx
+; X86-NEXT: movl 24(%esp,%edx,4), %esi
+; X86-NEXT: movl 16(%esp,%edx,4), %eax
+; X86-NEXT: movl 20(%esp,%edx,4), %edi
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edi, %eax
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl
+ %a0 = load i256, ptr %p0
+ %b = ashr i256 %a0, %a1
+ %r = trunc i256 %b to i64
+ ret i64 %r
+}
+
+define i64 @lshr_extract_idx_load_i256_i64(ptr %p0, i256 %a1) nounwind {
+; CHECK-LABEL: lshr_extract_idx_load_i256_i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: andl $3, %esi
+; CHECK-NEXT: movq (%rdi,%rsi,8), %rax
+; CHECK-NEXT: retq
+;
+; X86-LABEL: lshr_extract_idx_load_i256_i64:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: andl $3, %edx
+; X86-NEXT: movl (%ecx,%edx,8), %eax
+; X86-NEXT: movl 4(%ecx,%edx,8), %edx
+; X86-NEXT: retl
+ %a0 = load i256, ptr %p0
+ %m1 = mul i256 %a1, 64
+ %b = lshr i256 %a0, %m1
+ %r = trunc i256 %b to i64
+ ret i64 %r
+}
+
+define i64 @ashr_extract_idx_load_i256_i64(ptr %p0, i256 %a1) nounwind {
+; SSE-LABEL: ashr_extract_idx_load_i256_i64:
+; SSE: # %bb.0:
+; SSE-NEXT: movaps (%rdi), %xmm0
+; SSE-NEXT: movq 16(%rdi), %rax
+; SSE-NEXT: movq 24(%rdi), %rcx
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: sarq $63, %rcx
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: andb $3, %sil
+; SSE-NEXT: movzbl %sil, %eax
+; SSE-NEXT: movq -72(%rsp,%rax,8), %rax
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: ashr_extract_idx_load_i256_i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovaps (%rdi), %xmm0
+; AVX2-NEXT: movq 16(%rdi), %rax
+; AVX2-NEXT: movq 24(%rdi), %rcx
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: sarq $63, %rcx
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: andb $3, %sil
+; AVX2-NEXT: movzbl %sil, %eax
+; AVX2-NEXT: movq -72(%rsp,%rax,8), %rax
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: ashr_extract_idx_load_i256_i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps (%rdi), %xmm0
+; AVX512-NEXT: movq 16(%rdi), %rax
+; AVX512-NEXT: movq 24(%rdi), %rcx
+; AVX512-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: sarq $63, %rcx
+; AVX512-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT: andb $3, %sil
+; AVX512-NEXT: movzbl %sil, %eax
+; AVX512-NEXT: movq -72(%rsp,%rax,8), %rax
+; AVX512-NEXT: retq
+;
+; X86-LABEL: ashr_extract_idx_load_i256_i64:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $96, %esp
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl (%eax), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 4(%eax), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 8(%eax), %esi
+; X86-NEXT: movl 12(%eax), %edi
+; X86-NEXT: movl 16(%eax), %ebx
+; X86-NEXT: movl 20(%eax), %edx
+; X86-NEXT: movl 24(%eax), %ecx
+; X86-NEXT: movl 28(%eax), %eax
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movzbl 12(%ebp), %ecx
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: sarl $31, %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: andl $3, %ecx
+; X86-NEXT: movl 16(%esp,%ecx,8), %eax
+; X86-NEXT: movl 20(%esp,%ecx,8), %edx
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl
+ %a0 = load i256, ptr %p0
+ %m1 = mul i256 %a1, 64
+ %b = ashr i256 %a0, %m1
+ %r = trunc i256 %b to i64
+ ret i64 %r
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-X64: {{.*}}
More information about the llvm-commits
mailing list