[llvm] 307890f - [X86] Freeze vXi8 shl(x, 1) -> add(x, x) vector fold (PR50468)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 24 08:08:37 PDT 2021
Author: Simon Pilgrim
Date: 2021-08-24T16:08:24+01:00
New Revision: 307890f85b33bc190b563bcb3daeb232b8ecaa73
URL: https://github.com/llvm/llvm-project/commit/307890f85b33bc190b563bcb3daeb232b8ecaa73
DIFF: https://github.com/llvm/llvm-project/commit/307890f85b33bc190b563bcb3daeb232b8ecaa73.diff
LOG: [X86] Freeze vXi8 shl(x,1) -> add(x,x) vector fold (PR50468)
We don't have any vXi8 shift instructions (other than on XOP which is handled separately), so replace the shl(x,1) -> add(x,x) fold with shl(x,1) -> add(freeze(x),freeze(x)) to avoid the undef issues identified in PR50468.
Split off from D106675 as I'm still looking at whether we can fix the vXi16/i32/i64 issues with the D106679 alternative.
Differential Revision: https://reviews.llvm.org/D108139
Added:
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
llvm/test/CodeGen/X86/bitreverse.ll
llvm/test/CodeGen/X86/combine-bitreverse.ll
llvm/test/CodeGen/X86/vector-bitreverse.ll
llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
llvm/test/CodeGen/X86/vector-fshr-128.ll
llvm/test/CodeGen/X86/vector-fshr-256.ll
llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
llvm/test/CodeGen/X86/vector-rotate-128.ll
llvm/test/CodeGen/X86/vector-rotate-256.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 5096d45ad10d..e69109669248 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -28731,8 +28731,15 @@ static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
MVT ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
// Simple i8 add case
- if (Op.getOpcode() == ISD::SHL && ShiftAmt == 1)
+ if (Op.getOpcode() == ISD::SHL && ShiftAmt == 1) {
+ // R may be undef at run-time, but (shl R, 1) must be an even number (LSB
+ // must be 0). (add undef, undef) however can be any value. To make this
+ // safe, we must freeze R to ensure that register allocation uses the same
+ // register for an undefined value. This ensures that the result will
+ // still be even and preserves the original semantics.
+ R = DAG.getNode(ISD::FREEZE, dl, VT, R);
return DAG.getNode(ISD::ADD, dl, VT, R, R);
+ }
// ashr(R, 7) === cmp_slt(R, 0)
if (Op.getOpcode() == ISD::SRA && ShiftAmt == 7) {
diff --git a/llvm/test/CodeGen/X86/bitreverse.ll b/llvm/test/CodeGen/X86/bitreverse.ll
index 3a98ff23c397..b1c4cbead6c1 100644
--- a/llvm/test/CodeGen/X86/bitreverse.ll
+++ b/llvm/test/CodeGen/X86/bitreverse.ll
@@ -69,11 +69,11 @@ define <2 x i16> @test_bitreverse_v2i16(<2 x i16> %a) nounwind {
; X64-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; X64-NEXT: psrlw $2, %xmm0
; X64-NEXT: por %xmm1, %xmm0
-; X64-NEXT: movdqa {{.*#+}} xmm1 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; X64-NEXT: movdqa {{.*#+}} xmm1 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; X64-NEXT: pand %xmm0, %xmm1
-; X64-NEXT: paddb %xmm1, %xmm1
+; X64-NEXT: psrlw $1, %xmm1
; X64-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; X64-NEXT: psrlw $1, %xmm0
+; X64-NEXT: paddb %xmm0, %xmm0
; X64-NEXT: por %xmm1, %xmm0
; X64-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/combine-bitreverse.ll b/llvm/test/CodeGen/X86/combine-bitreverse.ll
index a9bcf394048b..8c41f533fd6b 100644
--- a/llvm/test/CodeGen/X86/combine-bitreverse.ll
+++ b/llvm/test/CodeGen/X86/combine-bitreverse.ll
@@ -61,11 +61,11 @@ define <4 x i32> @test_demandedbits_bitreverse(<4 x i32> %a0) nounwind {
; X86-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
; X86-NEXT: psrlw $2, %xmm0
; X86-NEXT: por %xmm1, %xmm0
-; X86-NEXT: movdqa {{.*#+}} xmm1 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; X86-NEXT: movdqa {{.*#+}} xmm1 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; X86-NEXT: pand %xmm0, %xmm1
-; X86-NEXT: paddb %xmm1, %xmm1
+; X86-NEXT: psrlw $1, %xmm1
; X86-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-NEXT: psrlw $1, %xmm0
+; X86-NEXT: paddb %xmm0, %xmm0
; X86-NEXT: por %xmm1, %xmm0
; X86-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
; X86-NEXT: retl
diff --git a/llvm/test/CodeGen/X86/vector-bitreverse.ll b/llvm/test/CodeGen/X86/vector-bitreverse.ll
index 4f73bdac52de..651418d271be 100644
--- a/llvm/test/CodeGen/X86/vector-bitreverse.ll
+++ b/llvm/test/CodeGen/X86/vector-bitreverse.ll
@@ -693,11 +693,11 @@ define <16 x i8> @test_bitreverse_v16i8(<16 x i8> %a) nounwind {
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE2-NEXT: psrlw $2, %xmm0
; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: paddb %xmm1, %xmm1
+; SSE2-NEXT: psrlw $1, %xmm1
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: psrlw $1, %xmm0
+; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: por %xmm1, %xmm0
; SSE2-NEXT: retq
;
@@ -781,11 +781,11 @@ define <8 x i16> @test_bitreverse_v8i16(<8 x i16> %a) nounwind {
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE2-NEXT: psrlw $2, %xmm0
; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: paddb %xmm1, %xmm1
+; SSE2-NEXT: psrlw $1, %xmm1
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: psrlw $1, %xmm0
+; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: por %xmm1, %xmm0
; SSE2-NEXT: retq
;
@@ -881,11 +881,11 @@ define <4 x i32> @test_bitreverse_v4i32(<4 x i32> %a) nounwind {
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE2-NEXT: psrlw $2, %xmm0
; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: paddb %xmm1, %xmm1
+; SSE2-NEXT: psrlw $1, %xmm1
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: psrlw $1, %xmm0
+; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: por %xmm1, %xmm0
; SSE2-NEXT: retq
;
@@ -983,11 +983,11 @@ define <2 x i64> @test_bitreverse_v2i64(<2 x i64> %a) nounwind {
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE2-NEXT: psrlw $2, %xmm0
; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; SSE2-NEXT: pand %xmm0, %xmm1
-; SSE2-NEXT: paddb %xmm1, %xmm1
+; SSE2-NEXT: psrlw $1, %xmm1
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: psrlw $1, %xmm0
+; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: por %xmm1, %xmm0
; SSE2-NEXT: retq
;
@@ -1079,13 +1079,13 @@ define <32 x i8> @test_bitreverse_v32i8(<32 x i8> %a) nounwind {
; SSE2-NEXT: pand %xmm5, %xmm0
; SSE2-NEXT: psrlw $2, %xmm0
; SSE2-NEXT: por %xmm4, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; SSE2-NEXT: movdqa %xmm0, %xmm6
; SSE2-NEXT: pand %xmm4, %xmm6
-; SSE2-NEXT: paddb %xmm6, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
+; SSE2-NEXT: psrlw $1, %xmm6
+; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; SSE2-NEXT: pand %xmm7, %xmm0
-; SSE2-NEXT: psrlw $1, %xmm0
+; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: por %xmm6, %xmm0
; SSE2-NEXT: movdqa %xmm2, %xmm6
; SSE2-NEXT: psllw $4, %xmm6
@@ -1099,9 +1099,9 @@ define <32 x i8> @test_bitreverse_v32i8(<32 x i8> %a) nounwind {
; SSE2-NEXT: psrlw $2, %xmm1
; SSE2-NEXT: por %xmm3, %xmm1
; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: paddb %xmm4, %xmm4
+; SSE2-NEXT: psrlw $1, %xmm4
; SSE2-NEXT: pand %xmm7, %xmm1
-; SSE2-NEXT: psrlw $1, %xmm1
+; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: retq
;
@@ -1256,13 +1256,13 @@ define <16 x i16> @test_bitreverse_v16i16(<16 x i16> %a) nounwind {
; SSE2-NEXT: pand %xmm5, %xmm0
; SSE2-NEXT: psrlw $2, %xmm0
; SSE2-NEXT: por %xmm4, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; SSE2-NEXT: movdqa %xmm0, %xmm7
; SSE2-NEXT: pand %xmm4, %xmm7
-; SSE2-NEXT: paddb %xmm7, %xmm7
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
+; SSE2-NEXT: psrlw $1, %xmm7
+; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; SSE2-NEXT: pand %xmm6, %xmm0
-; SSE2-NEXT: psrlw $1, %xmm0
+; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: por %xmm7, %xmm0
; SSE2-NEXT: movdqa %xmm2, %xmm7
; SSE2-NEXT: psrlw $8, %xmm7
@@ -1280,9 +1280,9 @@ define <16 x i16> @test_bitreverse_v16i16(<16 x i16> %a) nounwind {
; SSE2-NEXT: psrlw $2, %xmm1
; SSE2-NEXT: por %xmm3, %xmm1
; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: paddb %xmm4, %xmm4
+; SSE2-NEXT: psrlw $1, %xmm4
; SSE2-NEXT: pand %xmm6, %xmm1
-; SSE2-NEXT: psrlw $1, %xmm1
+; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: retq
;
@@ -1459,13 +1459,13 @@ define <8 x i32> @test_bitreverse_v8i32(<8 x i32> %a) nounwind {
; SSE2-NEXT: pand %xmm8, %xmm0
; SSE2-NEXT: psrlw $2, %xmm0
; SSE2-NEXT: por %xmm5, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; SSE2-NEXT: movdqa %xmm0, %xmm6
; SSE2-NEXT: pand %xmm5, %xmm6
-; SSE2-NEXT: paddb %xmm6, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
+; SSE2-NEXT: psrlw $1, %xmm6
+; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; SSE2-NEXT: pand %xmm7, %xmm0
-; SSE2-NEXT: psrlw $1, %xmm0
+; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: por %xmm6, %xmm0
; SSE2-NEXT: movdqa %xmm2, %xmm6
; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
@@ -1487,9 +1487,9 @@ define <8 x i32> @test_bitreverse_v8i32(<8 x i32> %a) nounwind {
; SSE2-NEXT: psrlw $2, %xmm1
; SSE2-NEXT: por %xmm3, %xmm1
; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: paddb %xmm5, %xmm5
+; SSE2-NEXT: psrlw $1, %xmm5
; SSE2-NEXT: pand %xmm7, %xmm1
-; SSE2-NEXT: psrlw $1, %xmm1
+; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: por %xmm5, %xmm1
; SSE2-NEXT: retq
;
@@ -1668,13 +1668,13 @@ define <4 x i64> @test_bitreverse_v4i64(<4 x i64> %a) nounwind {
; SSE2-NEXT: pand %xmm8, %xmm0
; SSE2-NEXT: psrlw $2, %xmm0
; SSE2-NEXT: por %xmm5, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; SSE2-NEXT: movdqa %xmm0, %xmm6
; SSE2-NEXT: pand %xmm5, %xmm6
-; SSE2-NEXT: paddb %xmm6, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
+; SSE2-NEXT: psrlw $1, %xmm6
+; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; SSE2-NEXT: pand %xmm7, %xmm0
-; SSE2-NEXT: psrlw $1, %xmm0
+; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: por %xmm6, %xmm0
; SSE2-NEXT: movdqa %xmm2, %xmm6
; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
@@ -1698,9 +1698,9 @@ define <4 x i64> @test_bitreverse_v4i64(<4 x i64> %a) nounwind {
; SSE2-NEXT: psrlw $2, %xmm1
; SSE2-NEXT: por %xmm3, %xmm1
; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: paddb %xmm5, %xmm5
+; SSE2-NEXT: psrlw $1, %xmm5
; SSE2-NEXT: pand %xmm7, %xmm1
-; SSE2-NEXT: psrlw $1, %xmm1
+; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: por %xmm5, %xmm1
; SSE2-NEXT: retq
;
@@ -1868,13 +1868,13 @@ define <64 x i8> @test_bitreverse_v64i8(<64 x i8> %a) nounwind {
; SSE2-NEXT: pand %xmm8, %xmm0
; SSE2-NEXT: psrlw $2, %xmm0
; SSE2-NEXT: por %xmm6, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; SSE2-NEXT: movdqa %xmm0, %xmm7
; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: paddb %xmm7, %xmm7
-; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
+; SSE2-NEXT: psrlw $1, %xmm7
+; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; SSE2-NEXT: pand %xmm9, %xmm0
-; SSE2-NEXT: psrlw $1, %xmm0
+; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: por %xmm7, %xmm0
; SSE2-NEXT: movdqa %xmm1, %xmm7
; SSE2-NEXT: psllw $4, %xmm7
@@ -1891,9 +1891,9 @@ define <64 x i8> @test_bitreverse_v64i8(<64 x i8> %a) nounwind {
; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: movdqa %xmm1, %xmm4
; SSE2-NEXT: pand %xmm6, %xmm4
-; SSE2-NEXT: paddb %xmm4, %xmm4
+; SSE2-NEXT: psrlw $1, %xmm4
; SSE2-NEXT: pand %xmm9, %xmm1
-; SSE2-NEXT: psrlw $1, %xmm1
+; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: movdqa %xmm2, %xmm4
; SSE2-NEXT: psllw $4, %xmm4
@@ -1910,9 +1910,9 @@ define <64 x i8> @test_bitreverse_v64i8(<64 x i8> %a) nounwind {
; SSE2-NEXT: por %xmm4, %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm4
; SSE2-NEXT: pand %xmm6, %xmm4
-; SSE2-NEXT: paddb %xmm4, %xmm4
+; SSE2-NEXT: psrlw $1, %xmm4
; SSE2-NEXT: pand %xmm9, %xmm2
-; SSE2-NEXT: psrlw $1, %xmm2
+; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: por %xmm4, %xmm2
; SSE2-NEXT: movdqa %xmm10, %xmm4
; SSE2-NEXT: psllw $4, %xmm4
@@ -1926,9 +1926,9 @@ define <64 x i8> @test_bitreverse_v64i8(<64 x i8> %a) nounwind {
; SSE2-NEXT: psrlw $2, %xmm3
; SSE2-NEXT: por %xmm5, %xmm3
; SSE2-NEXT: pand %xmm3, %xmm6
-; SSE2-NEXT: paddb %xmm6, %xmm6
+; SSE2-NEXT: psrlw $1, %xmm6
; SSE2-NEXT: pand %xmm9, %xmm3
-; SSE2-NEXT: psrlw $1, %xmm3
+; SSE2-NEXT: paddb %xmm3, %xmm3
; SSE2-NEXT: por %xmm6, %xmm3
; SSE2-NEXT: retq
;
@@ -2160,13 +2160,13 @@ define <32 x i16> @test_bitreverse_v32i16(<32 x i16> %a) nounwind {
; SSE2-NEXT: pand %xmm8, %xmm0
; SSE2-NEXT: psrlw $2, %xmm0
; SSE2-NEXT: por %xmm6, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; SSE2-NEXT: movdqa %xmm0, %xmm7
; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: paddb %xmm7, %xmm7
-; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
+; SSE2-NEXT: psrlw $1, %xmm7
+; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; SSE2-NEXT: pand %xmm9, %xmm0
-; SSE2-NEXT: psrlw $1, %xmm0
+; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: por %xmm7, %xmm0
; SSE2-NEXT: movdqa %xmm1, %xmm7
; SSE2-NEXT: psrlw $8, %xmm7
@@ -2187,9 +2187,9 @@ define <32 x i16> @test_bitreverse_v32i16(<32 x i16> %a) nounwind {
; SSE2-NEXT: por %xmm5, %xmm1
; SSE2-NEXT: movdqa %xmm1, %xmm5
; SSE2-NEXT: pand %xmm6, %xmm5
-; SSE2-NEXT: paddb %xmm5, %xmm5
+; SSE2-NEXT: psrlw $1, %xmm5
; SSE2-NEXT: pand %xmm9, %xmm1
-; SSE2-NEXT: psrlw $1, %xmm1
+; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: por %xmm5, %xmm1
; SSE2-NEXT: movdqa %xmm2, %xmm5
; SSE2-NEXT: psrlw $8, %xmm5
@@ -2210,9 +2210,9 @@ define <32 x i16> @test_bitreverse_v32i16(<32 x i16> %a) nounwind {
; SSE2-NEXT: por %xmm5, %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm5
; SSE2-NEXT: pand %xmm6, %xmm5
-; SSE2-NEXT: paddb %xmm5, %xmm5
+; SSE2-NEXT: psrlw $1, %xmm5
; SSE2-NEXT: pand %xmm9, %xmm2
-; SSE2-NEXT: psrlw $1, %xmm2
+; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: por %xmm5, %xmm2
; SSE2-NEXT: movdqa %xmm4, %xmm5
; SSE2-NEXT: psrlw $8, %xmm5
@@ -2230,9 +2230,9 @@ define <32 x i16> @test_bitreverse_v32i16(<32 x i16> %a) nounwind {
; SSE2-NEXT: psrlw $2, %xmm3
; SSE2-NEXT: por %xmm10, %xmm3
; SSE2-NEXT: pand %xmm3, %xmm6
-; SSE2-NEXT: paddb %xmm6, %xmm6
+; SSE2-NEXT: psrlw $1, %xmm6
; SSE2-NEXT: pand %xmm9, %xmm3
-; SSE2-NEXT: psrlw $1, %xmm3
+; SSE2-NEXT: paddb %xmm3, %xmm3
; SSE2-NEXT: por %xmm6, %xmm3
; SSE2-NEXT: retq
;
@@ -2504,13 +2504,13 @@ define <16 x i32> @test_bitreverse_v16i32(<16 x i32> %a) nounwind {
; SSE2-NEXT: pand %xmm8, %xmm0
; SSE2-NEXT: psrlw $2, %xmm0
; SSE2-NEXT: por %xmm7, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; SSE2-NEXT: movdqa %xmm0, %xmm6
; SSE2-NEXT: pand %xmm7, %xmm6
-; SSE2-NEXT: paddb %xmm6, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
+; SSE2-NEXT: psrlw $1, %xmm6
+; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; SSE2-NEXT: pand %xmm9, %xmm0
-; SSE2-NEXT: psrlw $1, %xmm0
+; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: por %xmm6, %xmm0
; SSE2-NEXT: movdqa %xmm1, %xmm6
; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm10[8],xmm6[9],xmm10[9],xmm6[10],xmm10[10],xmm6[11],xmm10[11],xmm6[12],xmm10[12],xmm6[13],xmm10[13],xmm6[14],xmm10[14],xmm6[15],xmm10[15]
@@ -2535,9 +2535,9 @@ define <16 x i32> @test_bitreverse_v16i32(<16 x i32> %a) nounwind {
; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: movdqa %xmm1, %xmm4
; SSE2-NEXT: pand %xmm7, %xmm4
-; SSE2-NEXT: paddb %xmm4, %xmm4
+; SSE2-NEXT: psrlw $1, %xmm4
; SSE2-NEXT: pand %xmm9, %xmm1
-; SSE2-NEXT: psrlw $1, %xmm1
+; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: movdqa %xmm2, %xmm4
; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm10[8],xmm4[9],xmm10[9],xmm4[10],xmm10[10],xmm4[11],xmm10[11],xmm4[12],xmm10[12],xmm4[13],xmm10[13],xmm4[14],xmm10[14],xmm4[15],xmm10[15]
@@ -2562,9 +2562,9 @@ define <16 x i32> @test_bitreverse_v16i32(<16 x i32> %a) nounwind {
; SSE2-NEXT: por %xmm4, %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm4
; SSE2-NEXT: pand %xmm7, %xmm4
-; SSE2-NEXT: paddb %xmm4, %xmm4
+; SSE2-NEXT: psrlw $1, %xmm4
; SSE2-NEXT: pand %xmm9, %xmm2
-; SSE2-NEXT: psrlw $1, %xmm2
+; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: por %xmm4, %xmm2
; SSE2-NEXT: movdqa %xmm11, %xmm4
; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm10[8],xmm4[9],xmm10[9],xmm4[10],xmm10[10],xmm4[11],xmm10[11],xmm4[12],xmm10[12],xmm4[13],xmm10[13],xmm4[14],xmm10[14],xmm4[15],xmm10[15]
@@ -2586,9 +2586,9 @@ define <16 x i32> @test_bitreverse_v16i32(<16 x i32> %a) nounwind {
; SSE2-NEXT: psrlw $2, %xmm3
; SSE2-NEXT: por %xmm5, %xmm3
; SSE2-NEXT: pand %xmm3, %xmm7
-; SSE2-NEXT: paddb %xmm7, %xmm7
+; SSE2-NEXT: psrlw $1, %xmm7
; SSE2-NEXT: pand %xmm9, %xmm3
-; SSE2-NEXT: psrlw $1, %xmm3
+; SSE2-NEXT: paddb %xmm3, %xmm3
; SSE2-NEXT: por %xmm7, %xmm3
; SSE2-NEXT: retq
;
@@ -2862,13 +2862,13 @@ define <8 x i64> @test_bitreverse_v8i64(<8 x i64> %a) nounwind {
; SSE2-NEXT: pand %xmm8, %xmm0
; SSE2-NEXT: psrlw $2, %xmm0
; SSE2-NEXT: por %xmm7, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
; SSE2-NEXT: movdqa %xmm0, %xmm6
; SSE2-NEXT: pand %xmm7, %xmm6
-; SSE2-NEXT: paddb %xmm6, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
+; SSE2-NEXT: psrlw $1, %xmm6
+; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; SSE2-NEXT: pand %xmm9, %xmm0
-; SSE2-NEXT: psrlw $1, %xmm0
+; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: por %xmm6, %xmm0
; SSE2-NEXT: movdqa %xmm1, %xmm6
; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm10[8],xmm6[9],xmm10[9],xmm6[10],xmm10[10],xmm6[11],xmm10[11],xmm6[12],xmm10[12],xmm6[13],xmm10[13],xmm6[14],xmm10[14],xmm6[15],xmm10[15]
@@ -2895,9 +2895,9 @@ define <8 x i64> @test_bitreverse_v8i64(<8 x i64> %a) nounwind {
; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: movdqa %xmm1, %xmm4
; SSE2-NEXT: pand %xmm7, %xmm4
-; SSE2-NEXT: paddb %xmm4, %xmm4
+; SSE2-NEXT: psrlw $1, %xmm4
; SSE2-NEXT: pand %xmm9, %xmm1
-; SSE2-NEXT: psrlw $1, %xmm1
+; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: movdqa %xmm2, %xmm4
; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm10[8],xmm4[9],xmm10[9],xmm4[10],xmm10[10],xmm4[11],xmm10[11],xmm4[12],xmm10[12],xmm4[13],xmm10[13],xmm4[14],xmm10[14],xmm4[15],xmm10[15]
@@ -2924,9 +2924,9 @@ define <8 x i64> @test_bitreverse_v8i64(<8 x i64> %a) nounwind {
; SSE2-NEXT: por %xmm4, %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm4
; SSE2-NEXT: pand %xmm7, %xmm4
-; SSE2-NEXT: paddb %xmm4, %xmm4
+; SSE2-NEXT: psrlw $1, %xmm4
; SSE2-NEXT: pand %xmm9, %xmm2
-; SSE2-NEXT: psrlw $1, %xmm2
+; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: por %xmm4, %xmm2
; SSE2-NEXT: movdqa %xmm11, %xmm4
; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm10[8],xmm4[9],xmm10[9],xmm4[10],xmm10[10],xmm4[11],xmm10[11],xmm4[12],xmm10[12],xmm4[13],xmm10[13],xmm4[14],xmm10[14],xmm4[15],xmm10[15]
@@ -2950,9 +2950,9 @@ define <8 x i64> @test_bitreverse_v8i64(<8 x i64> %a) nounwind {
; SSE2-NEXT: psrlw $2, %xmm3
; SSE2-NEXT: por %xmm5, %xmm3
; SSE2-NEXT: pand %xmm3, %xmm7
-; SSE2-NEXT: paddb %xmm7, %xmm7
+; SSE2-NEXT: psrlw $1, %xmm7
; SSE2-NEXT: pand %xmm9, %xmm3
-; SSE2-NEXT: psrlw $1, %xmm3
+; SSE2-NEXT: paddb %xmm3, %xmm3
; SSE2-NEXT: por %xmm7, %xmm3
; SSE2-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
index 665ab72d6aa3..1e41aa019f7c 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
@@ -517,10 +517,10 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
; SSE2-NEXT: pandn %xmm3, %xmm2
; SSE2-NEXT: por %xmm4, %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm3
-; SSE2-NEXT: paddb %xmm2, %xmm3
+; SSE2-NEXT: psrlw $7, %xmm3
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
; SSE2-NEXT: movdqa %xmm2, %xmm4
-; SSE2-NEXT: psrlw $7, %xmm4
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
+; SSE2-NEXT: paddb %xmm2, %xmm4
; SSE2-NEXT: por %xmm3, %xmm4
; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pcmpgtb %xmm1, %xmm0
@@ -553,10 +553,10 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
; SSE41-NEXT: movdqa %xmm2, %xmm0
; SSE41-NEXT: pblendvb %xmm0, %xmm3, %xmm1
; SSE41-NEXT: movdqa %xmm1, %xmm0
-; SSE41-NEXT: paddb %xmm1, %xmm0
+; SSE41-NEXT: psrlw $7, %xmm0
+; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE41-NEXT: movdqa %xmm1, %xmm3
-; SSE41-NEXT: psrlw $7, %xmm3
-; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE41-NEXT: paddb %xmm1, %xmm3
; SSE41-NEXT: por %xmm0, %xmm3
; SSE41-NEXT: paddb %xmm2, %xmm2
; SSE41-NEXT: movdqa %xmm2, %xmm0
@@ -580,10 +580,10 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
; AVX-NEXT: vpor %xmm2, %xmm3, %xmm2
; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1
; AVX-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; AVX-NEXT: vpaddb %xmm0, %xmm0, %xmm2
-; AVX-NEXT: vpsrlw $7, %xmm0, %xmm3
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3
-; AVX-NEXT: vpor %xmm3, %xmm2, %xmm2
+; AVX-NEXT: vpsrlw $7, %xmm0, %xmm2
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX-NEXT: vpaddb %xmm0, %xmm0, %xmm3
+; AVX-NEXT: vpor %xmm2, %xmm3, %xmm2
; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1
; AVX-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
; AVX-NEXT: retq
@@ -728,10 +728,10 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
; X86-SSE2-NEXT: pandn %xmm3, %xmm2
; X86-SSE2-NEXT: por %xmm4, %xmm2
; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
-; X86-SSE2-NEXT: paddb %xmm2, %xmm3
+; X86-SSE2-NEXT: psrlw $7, %xmm3
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm3
; X86-SSE2-NEXT: movdqa %xmm2, %xmm4
-; X86-SSE2-NEXT: psrlw $7, %xmm4
-; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm4
+; X86-SSE2-NEXT: paddb %xmm2, %xmm4
; X86-SSE2-NEXT: por %xmm3, %xmm4
; X86-SSE2-NEXT: paddb %xmm1, %xmm1
; X86-SSE2-NEXT: pcmpgtb %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
index 2f9cd77992ce..37343d66bf08 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
@@ -443,10 +443,10 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
; AVX2-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1
; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpaddb %ymm0, %ymm0, %ymm2
-; AVX2-NEXT: vpsrlw $7, %ymm0, %ymm3
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
-; AVX2-NEXT: vpor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpsrlw $7, %ymm0, %ymm2
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT: vpaddb %ymm0, %ymm0, %ymm3
+; AVX2-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1
; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
; AVX2-NEXT: retq
@@ -467,10 +467,10 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
; AVX512F-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1
; AVX512F-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT: vpaddb %ymm0, %ymm0, %ymm2
-; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm3
-; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
-; AVX512F-NEXT: vpor %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm2
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT: vpaddb %ymm0, %ymm0, %ymm3
+; AVX512F-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1
; AVX512F-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
; AVX512F-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
index 7dcafe8685cd..488861ad53b3 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
@@ -272,154 +272,154 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
;
; AVX512BW-LABEL: var_funnnel_v64i8:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-NEXT: vpsubb %zmm1, %zmm2, %zmm2
-; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm2
-; AVX512BW-NEXT: vpsllw $5, %zmm2, %zmm2
-; AVX512BW-NEXT: vpaddb %zmm2, %zmm2, %zmm4
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512BW-NEXT: vpandq %zmm2, %zmm1, %zmm3
+; AVX512BW-NEXT: vpsllw $5, %zmm3, %zmm3
+; AVX512BW-NEXT: vpaddb %zmm3, %zmm3, %zmm4
; AVX512BW-NEXT: vpmovb2m %zmm4, %k1
-; AVX512BW-NEXT: vpmovb2m %zmm2, %k2
-; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm2
-; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2
-; AVX512BW-NEXT: vpblendmb %zmm2, %zmm0, %zmm2 {%k2}
-; AVX512BW-NEXT: vpsrlw $2, %zmm2, %zmm5
-; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
-; AVX512BW-NEXT: vmovdqu8 %zmm5, %zmm2 {%k1}
-; AVX512BW-NEXT: vpsrlw $1, %zmm2, %zmm5
+; AVX512BW-NEXT: vpmovb2m %zmm3, %k2
+; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm3
+; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm3
+; AVX512BW-NEXT: vpblendmb %zmm3, %zmm0, %zmm3 {%k2}
+; AVX512BW-NEXT: vpsllw $2, %zmm3, %zmm5
; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
+; AVX512BW-NEXT: vmovdqu8 %zmm5, %zmm3 {%k1}
; AVX512BW-NEXT: vpaddb %zmm4, %zmm4, %zmm4
; AVX512BW-NEXT: vpmovb2m %zmm4, %k1
-; AVX512BW-NEXT: vmovdqu8 %zmm5, %zmm2 {%k1}
-; AVX512BW-NEXT: vpandq %zmm3, %zmm1, %zmm1
+; AVX512BW-NEXT: vpaddb %zmm3, %zmm3, %zmm3 {%k1}
+; AVX512BW-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512BW-NEXT: vpsubb %zmm1, %zmm4, %zmm1
+; AVX512BW-NEXT: vpandq %zmm2, %zmm1, %zmm1
; AVX512BW-NEXT: vpsllw $5, %zmm1, %zmm1
-; AVX512BW-NEXT: vpaddb %zmm1, %zmm1, %zmm3
-; AVX512BW-NEXT: vpmovb2m %zmm3, %k1
+; AVX512BW-NEXT: vpaddb %zmm1, %zmm1, %zmm2
+; AVX512BW-NEXT: vpmovb2m %zmm2, %k1
; AVX512BW-NEXT: vpmovb2m %zmm1, %k2
-; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm1
+; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm1
; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512BW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k2}
-; AVX512BW-NEXT: vpsllw $2, %zmm0, %zmm1
+; AVX512BW-NEXT: vpsrlw $2, %zmm0, %zmm1
; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512BW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
-; AVX512BW-NEXT: vpaddb %zmm3, %zmm3, %zmm1
-; AVX512BW-NEXT: vpmovb2m %zmm1, %k1
-; AVX512BW-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1}
-; AVX512BW-NEXT: vporq %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT: vpsrlw $1, %zmm0, %zmm1
+; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT: vpaddb %zmm2, %zmm2, %zmm2
+; AVX512BW-NEXT: vpmovb2m %zmm2, %k1
+; AVX512BW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512BW-NEXT: vporq %zmm0, %zmm3, %zmm0
; AVX512BW-NEXT: retq
;
; AVX512VLBW-LABEL: var_funnnel_v64i8:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLBW-NEXT: vpsubb %zmm1, %zmm2, %zmm2
-; AVX512VLBW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VLBW-NEXT: vpandq %zmm3, %zmm2, %zmm2
-; AVX512VLBW-NEXT: vpsllw $5, %zmm2, %zmm2
-; AVX512VLBW-NEXT: vpaddb %zmm2, %zmm2, %zmm4
+; AVX512VLBW-NEXT: vmovdqa64 {{.*#+}} zmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLBW-NEXT: vpandq %zmm2, %zmm1, %zmm3
+; AVX512VLBW-NEXT: vpsllw $5, %zmm3, %zmm3
+; AVX512VLBW-NEXT: vpaddb %zmm3, %zmm3, %zmm4
; AVX512VLBW-NEXT: vpmovb2m %zmm4, %k1
-; AVX512VLBW-NEXT: vpmovb2m %zmm2, %k2
-; AVX512VLBW-NEXT: vpsrlw $4, %zmm0, %zmm2
-; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2
-; AVX512VLBW-NEXT: vpblendmb %zmm2, %zmm0, %zmm2 {%k2}
-; AVX512VLBW-NEXT: vpsrlw $2, %zmm2, %zmm5
-; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
-; AVX512VLBW-NEXT: vmovdqu8 %zmm5, %zmm2 {%k1}
-; AVX512VLBW-NEXT: vpsrlw $1, %zmm2, %zmm5
+; AVX512VLBW-NEXT: vpmovb2m %zmm3, %k2
+; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm3
+; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm3
+; AVX512VLBW-NEXT: vpblendmb %zmm3, %zmm0, %zmm3 {%k2}
+; AVX512VLBW-NEXT: vpsllw $2, %zmm3, %zmm5
; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
+; AVX512VLBW-NEXT: vmovdqu8 %zmm5, %zmm3 {%k1}
; AVX512VLBW-NEXT: vpaddb %zmm4, %zmm4, %zmm4
; AVX512VLBW-NEXT: vpmovb2m %zmm4, %k1
-; AVX512VLBW-NEXT: vmovdqu8 %zmm5, %zmm2 {%k1}
-; AVX512VLBW-NEXT: vpandq %zmm3, %zmm1, %zmm1
+; AVX512VLBW-NEXT: vpaddb %zmm3, %zmm3, %zmm3 {%k1}
+; AVX512VLBW-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLBW-NEXT: vpsubb %zmm1, %zmm4, %zmm1
+; AVX512VLBW-NEXT: vpandq %zmm2, %zmm1, %zmm1
; AVX512VLBW-NEXT: vpsllw $5, %zmm1, %zmm1
-; AVX512VLBW-NEXT: vpaddb %zmm1, %zmm1, %zmm3
-; AVX512VLBW-NEXT: vpmovb2m %zmm3, %k1
+; AVX512VLBW-NEXT: vpaddb %zmm1, %zmm1, %zmm2
+; AVX512VLBW-NEXT: vpmovb2m %zmm2, %k1
; AVX512VLBW-NEXT: vpmovb2m %zmm1, %k2
-; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm1
+; AVX512VLBW-NEXT: vpsrlw $4, %zmm0, %zmm1
; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLBW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k2}
-; AVX512VLBW-NEXT: vpsllw $2, %zmm0, %zmm1
+; AVX512VLBW-NEXT: vpsrlw $2, %zmm0, %zmm1
; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLBW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
-; AVX512VLBW-NEXT: vpaddb %zmm3, %zmm3, %zmm1
-; AVX512VLBW-NEXT: vpmovb2m %zmm1, %k1
-; AVX512VLBW-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1}
-; AVX512VLBW-NEXT: vporq %zmm2, %zmm0, %zmm0
+; AVX512VLBW-NEXT: vpsrlw $1, %zmm0, %zmm1
+; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VLBW-NEXT: vpaddb %zmm2, %zmm2, %zmm2
+; AVX512VLBW-NEXT: vpmovb2m %zmm2, %k1
+; AVX512VLBW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VLBW-NEXT: vporq %zmm0, %zmm3, %zmm0
; AVX512VLBW-NEXT: retq
;
; AVX512VBMI2-LABEL: var_funnnel_v64i8:
; AVX512VBMI2: # %bb.0:
-; AVX512VBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VBMI2-NEXT: vpsubb %zmm1, %zmm2, %zmm2
-; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VBMI2-NEXT: vpandq %zmm3, %zmm2, %zmm2
-; AVX512VBMI2-NEXT: vpsllw $5, %zmm2, %zmm2
-; AVX512VBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm4
+; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT: vpandq %zmm2, %zmm1, %zmm3
+; AVX512VBMI2-NEXT: vpsllw $5, %zmm3, %zmm3
+; AVX512VBMI2-NEXT: vpaddb %zmm3, %zmm3, %zmm4
; AVX512VBMI2-NEXT: vpmovb2m %zmm4, %k1
-; AVX512VBMI2-NEXT: vpmovb2m %zmm2, %k2
-; AVX512VBMI2-NEXT: vpsrlw $4, %zmm0, %zmm2
-; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2
-; AVX512VBMI2-NEXT: vpblendmb %zmm2, %zmm0, %zmm2 {%k2}
-; AVX512VBMI2-NEXT: vpsrlw $2, %zmm2, %zmm5
-; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
-; AVX512VBMI2-NEXT: vmovdqu8 %zmm5, %zmm2 {%k1}
-; AVX512VBMI2-NEXT: vpsrlw $1, %zmm2, %zmm5
+; AVX512VBMI2-NEXT: vpmovb2m %zmm3, %k2
+; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm3
+; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm3
+; AVX512VBMI2-NEXT: vpblendmb %zmm3, %zmm0, %zmm3 {%k2}
+; AVX512VBMI2-NEXT: vpsllw $2, %zmm3, %zmm5
; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
+; AVX512VBMI2-NEXT: vmovdqu8 %zmm5, %zmm3 {%k1}
; AVX512VBMI2-NEXT: vpaddb %zmm4, %zmm4, %zmm4
; AVX512VBMI2-NEXT: vpmovb2m %zmm4, %k1
-; AVX512VBMI2-NEXT: vmovdqu8 %zmm5, %zmm2 {%k1}
-; AVX512VBMI2-NEXT: vpandq %zmm3, %zmm1, %zmm1
+; AVX512VBMI2-NEXT: vpaddb %zmm3, %zmm3, %zmm3 {%k1}
+; AVX512VBMI2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT: vpsubb %zmm1, %zmm4, %zmm1
+; AVX512VBMI2-NEXT: vpandq %zmm2, %zmm1, %zmm1
; AVX512VBMI2-NEXT: vpsllw $5, %zmm1, %zmm1
-; AVX512VBMI2-NEXT: vpaddb %zmm1, %zmm1, %zmm3
-; AVX512VBMI2-NEXT: vpmovb2m %zmm3, %k1
+; AVX512VBMI2-NEXT: vpaddb %zmm1, %zmm1, %zmm2
+; AVX512VBMI2-NEXT: vpmovb2m %zmm2, %k1
; AVX512VBMI2-NEXT: vpmovb2m %zmm1, %k2
-; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm1
+; AVX512VBMI2-NEXT: vpsrlw $4, %zmm0, %zmm1
; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k2}
-; AVX512VBMI2-NEXT: vpsllw $2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT: vpsrlw $2, %zmm0, %zmm1
; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
-; AVX512VBMI2-NEXT: vpaddb %zmm3, %zmm3, %zmm1
-; AVX512VBMI2-NEXT: vpmovb2m %zmm1, %k1
-; AVX512VBMI2-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1}
-; AVX512VBMI2-NEXT: vporq %zmm2, %zmm0, %zmm0
+; AVX512VBMI2-NEXT: vpsrlw $1, %zmm0, %zmm1
+; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm2
+; AVX512VBMI2-NEXT: vpmovb2m %zmm2, %k1
+; AVX512VBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT: vporq %zmm0, %zmm3, %zmm0
; AVX512VBMI2-NEXT: retq
;
; AVX512VLVBMI2-LABEL: var_funnnel_v64i8:
; AVX512VLVBMI2: # %bb.0:
-; AVX512VLVBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLVBMI2-NEXT: vpsubb %zmm1, %zmm2, %zmm2
-; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VLVBMI2-NEXT: vpandq %zmm3, %zmm2, %zmm2
-; AVX512VLVBMI2-NEXT: vpsllw $5, %zmm2, %zmm2
-; AVX512VLVBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm4
+; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT: vpandq %zmm2, %zmm1, %zmm3
+; AVX512VLVBMI2-NEXT: vpsllw $5, %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT: vpaddb %zmm3, %zmm3, %zmm4
; AVX512VLVBMI2-NEXT: vpmovb2m %zmm4, %k1
-; AVX512VLVBMI2-NEXT: vpmovb2m %zmm2, %k2
-; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm0, %zmm2
-; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2
-; AVX512VLVBMI2-NEXT: vpblendmb %zmm2, %zmm0, %zmm2 {%k2}
-; AVX512VLVBMI2-NEXT: vpsrlw $2, %zmm2, %zmm5
-; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
-; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm5, %zmm2 {%k1}
-; AVX512VLVBMI2-NEXT: vpsrlw $1, %zmm2, %zmm5
+; AVX512VLVBMI2-NEXT: vpmovb2m %zmm3, %k2
+; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT: vpblendmb %zmm3, %zmm0, %zmm3 {%k2}
+; AVX512VLVBMI2-NEXT: vpsllw $2, %zmm3, %zmm5
; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm5, %zmm3 {%k1}
; AVX512VLVBMI2-NEXT: vpaddb %zmm4, %zmm4, %zmm4
; AVX512VLVBMI2-NEXT: vpmovb2m %zmm4, %k1
-; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm5, %zmm2 {%k1}
-; AVX512VLVBMI2-NEXT: vpandq %zmm3, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT: vpaddb %zmm3, %zmm3, %zmm3 {%k1}
+; AVX512VLVBMI2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT: vpsubb %zmm1, %zmm4, %zmm1
+; AVX512VLVBMI2-NEXT: vpandq %zmm2, %zmm1, %zmm1
; AVX512VLVBMI2-NEXT: vpsllw $5, %zmm1, %zmm1
-; AVX512VLVBMI2-NEXT: vpaddb %zmm1, %zmm1, %zmm3
-; AVX512VLVBMI2-NEXT: vpmovb2m %zmm3, %k1
+; AVX512VLVBMI2-NEXT: vpaddb %zmm1, %zmm1, %zmm2
+; AVX512VLVBMI2-NEXT: vpmovb2m %zmm2, %k1
; AVX512VLVBMI2-NEXT: vpmovb2m %zmm1, %k2
-; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm0, %zmm1
; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k2}
-; AVX512VLVBMI2-NEXT: vpsllw $2, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT: vpsrlw $2, %zmm0, %zmm1
; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
-; AVX512VLVBMI2-NEXT: vpaddb %zmm3, %zmm3, %zmm1
-; AVX512VLVBMI2-NEXT: vpmovb2m %zmm1, %k1
-; AVX512VLVBMI2-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1}
-; AVX512VLVBMI2-NEXT: vporq %zmm2, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT: vpsrlw $1, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT: vpmovb2m %zmm2, %k1
+; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT: vporq %zmm0, %zmm3, %zmm0
; AVX512VLVBMI2-NEXT: retq
%res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> %amt)
ret <64 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll
index 1807b8ca8fb4..8c74560cbdd9 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll
@@ -732,65 +732,65 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt) nounwind {
; SSE2-LABEL: var_funnnel_v16i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; SSE2-NEXT: movdqa %xmm2, %xmm4
-; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: psllw $5, %xmm4
+; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; SSE2-NEXT: movdqa %xmm2, %xmm5
+; SSE2-NEXT: pand %xmm4, %xmm5
+; SSE2-NEXT: psllw $5, %xmm5
; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pcmpgtb %xmm4, %xmm6
-; SSE2-NEXT: paddb %xmm0, %xmm0
+; SSE2-NEXT: pcmpgtb %xmm5, %xmm6
; SSE2-NEXT: movdqa %xmm6, %xmm7
-; SSE2-NEXT: pandn %xmm0, %xmm7
-; SSE2-NEXT: psllw $4, %xmm0
-; SSE2-NEXT: pand %xmm6, %xmm0
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: por %xmm7, %xmm0
-; SSE2-NEXT: paddb %xmm4, %xmm4
+; SSE2-NEXT: pandn %xmm1, %xmm7
+; SSE2-NEXT: psrlw $4, %xmm1
+; SSE2-NEXT: pand %xmm6, %xmm1
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT: por %xmm7, %xmm1
+; SSE2-NEXT: paddb %xmm5, %xmm5
; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pcmpgtb %xmm4, %xmm6
+; SSE2-NEXT: pcmpgtb %xmm5, %xmm6
; SSE2-NEXT: movdqa %xmm6, %xmm7
-; SSE2-NEXT: pandn %xmm0, %xmm7
-; SSE2-NEXT: psllw $2, %xmm0
-; SSE2-NEXT: pand %xmm6, %xmm0
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: por %xmm7, %xmm0
-; SSE2-NEXT: paddb %xmm4, %xmm4
+; SSE2-NEXT: pandn %xmm1, %xmm7
+; SSE2-NEXT: psrlw $2, %xmm1
+; SSE2-NEXT: pand %xmm6, %xmm1
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT: por %xmm7, %xmm1
+; SSE2-NEXT: paddb %xmm5, %xmm5
; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pcmpgtb %xmm4, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm4
-; SSE2-NEXT: pandn %xmm0, %xmm4
-; SSE2-NEXT: paddb %xmm0, %xmm0
-; SSE2-NEXT: pand %xmm6, %xmm0
-; SSE2-NEXT: pand %xmm5, %xmm2
-; SSE2-NEXT: psllw $5, %xmm2
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pandn %xmm1, %xmm6
-; SSE2-NEXT: psrlw $4, %xmm1
-; SSE2-NEXT: pand %xmm5, %xmm1
+; SSE2-NEXT: pcmpgtb %xmm5, %xmm6
+; SSE2-NEXT: movdqa %xmm6, %xmm5
+; SSE2-NEXT: pandn %xmm1, %xmm5
+; SSE2-NEXT: psrlw $1, %xmm1
+; SSE2-NEXT: pand %xmm6, %xmm1
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: por %xmm6, %xmm1
+; SSE2-NEXT: por %xmm5, %xmm1
+; SSE2-NEXT: pandn %xmm4, %xmm2
+; SSE2-NEXT: psllw $5, %xmm2
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm4
+; SSE2-NEXT: paddb %xmm0, %xmm0
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pandn %xmm0, %xmm5
+; SSE2-NEXT: psllw $4, %xmm0
+; SSE2-NEXT: pand %xmm4, %xmm0
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
; SSE2-NEXT: paddb %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pandn %xmm1, %xmm6
-; SSE2-NEXT: psrlw $2, %xmm1
-; SSE2-NEXT: pand %xmm5, %xmm1
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: por %xmm6, %xmm1
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pandn %xmm0, %xmm5
+; SSE2-NEXT: psllw $2, %xmm0
+; SSE2-NEXT: pand %xmm4, %xmm0
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: pcmpgtb %xmm2, %xmm3
; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: psrlw $1, %xmm1
-; SSE2-NEXT: pand %xmm3, %xmm1
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: por %xmm2, %xmm1
-; SSE2-NEXT: por %xmm4, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm0, %xmm2
+; SSE2-NEXT: por %xmm1, %xmm2
+; SSE2-NEXT: paddb %xmm0, %xmm0
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: por %xmm2, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: var_funnnel_v16i8:
@@ -981,77 +981,77 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
; XOP-LABEL: var_funnnel_v16i8:
; XOP: # %bb.0:
; XOP-NEXT: vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOP-NEXT: vpandn %xmm3, %xmm2, %xmm4
+; XOP-NEXT: vpand %xmm3, %xmm2, %xmm4
+; XOP-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; XOP-NEXT: vpsubb %xmm4, %xmm5, %xmm4
+; XOP-NEXT: vpshlb %xmm4, %xmm1, %xmm1
+; XOP-NEXT: vpandn %xmm3, %xmm2, %xmm2
; XOP-NEXT: vpaddb %xmm0, %xmm0, %xmm0
-; XOP-NEXT: vpshlb %xmm4, %xmm0, %xmm0
-; XOP-NEXT: vpand %xmm3, %xmm2, %xmm2
-; XOP-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; XOP-NEXT: vpsubb %xmm2, %xmm3, %xmm2
-; XOP-NEXT: vpshlb %xmm2, %xmm1, %xmm1
+; XOP-NEXT: vpshlb %xmm2, %xmm0, %xmm0
; XOP-NEXT: vpor %xmm1, %xmm0, %xmm0
; XOP-NEXT: retq
;
; X86-SSE2-LABEL: var_funnnel_v16i8:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm4
-; X86-SSE2-NEXT: pandn %xmm5, %xmm4
-; X86-SSE2-NEXT: psllw $5, %xmm4
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm5
+; X86-SSE2-NEXT: pand %xmm4, %xmm5
+; X86-SSE2-NEXT: psllw $5, %xmm5
; X86-SSE2-NEXT: pxor %xmm3, %xmm3
; X86-SSE2-NEXT: pxor %xmm6, %xmm6
-; X86-SSE2-NEXT: pcmpgtb %xmm4, %xmm6
-; X86-SSE2-NEXT: paddb %xmm0, %xmm0
+; X86-SSE2-NEXT: pcmpgtb %xmm5, %xmm6
; X86-SSE2-NEXT: movdqa %xmm6, %xmm7
-; X86-SSE2-NEXT: pandn %xmm0, %xmm7
-; X86-SSE2-NEXT: psllw $4, %xmm0
-; X86-SSE2-NEXT: pand %xmm6, %xmm0
-; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE2-NEXT: por %xmm7, %xmm0
-; X86-SSE2-NEXT: paddb %xmm4, %xmm4
+; X86-SSE2-NEXT: pandn %xmm1, %xmm7
+; X86-SSE2-NEXT: psrlw $4, %xmm1
+; X86-SSE2-NEXT: pand %xmm6, %xmm1
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT: por %xmm7, %xmm1
+; X86-SSE2-NEXT: paddb %xmm5, %xmm5
; X86-SSE2-NEXT: pxor %xmm6, %xmm6
-; X86-SSE2-NEXT: pcmpgtb %xmm4, %xmm6
+; X86-SSE2-NEXT: pcmpgtb %xmm5, %xmm6
; X86-SSE2-NEXT: movdqa %xmm6, %xmm7
-; X86-SSE2-NEXT: pandn %xmm0, %xmm7
-; X86-SSE2-NEXT: psllw $2, %xmm0
-; X86-SSE2-NEXT: pand %xmm6, %xmm0
-; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE2-NEXT: por %xmm7, %xmm0
-; X86-SSE2-NEXT: paddb %xmm4, %xmm4
+; X86-SSE2-NEXT: pandn %xmm1, %xmm7
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm6, %xmm1
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT: por %xmm7, %xmm1
+; X86-SSE2-NEXT: paddb %xmm5, %xmm5
; X86-SSE2-NEXT: pxor %xmm6, %xmm6
-; X86-SSE2-NEXT: pcmpgtb %xmm4, %xmm6
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm4
-; X86-SSE2-NEXT: pandn %xmm0, %xmm4
-; X86-SSE2-NEXT: paddb %xmm0, %xmm0
-; X86-SSE2-NEXT: pand %xmm6, %xmm0
-; X86-SSE2-NEXT: pand %xmm5, %xmm2
-; X86-SSE2-NEXT: psllw $5, %xmm2
-; X86-SSE2-NEXT: pxor %xmm5, %xmm5
-; X86-SSE2-NEXT: pcmpgtb %xmm2, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm6
-; X86-SSE2-NEXT: pandn %xmm1, %xmm6
-; X86-SSE2-NEXT: psrlw $4, %xmm1
-; X86-SSE2-NEXT: pand %xmm5, %xmm1
+; X86-SSE2-NEXT: pcmpgtb %xmm5, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm5
+; X86-SSE2-NEXT: pandn %xmm1, %xmm5
+; X86-SSE2-NEXT: psrlw $1, %xmm1
+; X86-SSE2-NEXT: pand %xmm6, %xmm1
; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
-; X86-SSE2-NEXT: por %xmm6, %xmm1
+; X86-SSE2-NEXT: por %xmm5, %xmm1
+; X86-SSE2-NEXT: pandn %xmm4, %xmm2
+; X86-SSE2-NEXT: psllw $5, %xmm2
+; X86-SSE2-NEXT: pxor %xmm4, %xmm4
+; X86-SSE2-NEXT: pcmpgtb %xmm2, %xmm4
+; X86-SSE2-NEXT: paddb %xmm0, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm5
+; X86-SSE2-NEXT: pandn %xmm0, %xmm5
+; X86-SSE2-NEXT: psllw $4, %xmm0
+; X86-SSE2-NEXT: pand %xmm4, %xmm0
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT: por %xmm5, %xmm0
; X86-SSE2-NEXT: paddb %xmm2, %xmm2
-; X86-SSE2-NEXT: pxor %xmm5, %xmm5
-; X86-SSE2-NEXT: pcmpgtb %xmm2, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm6
-; X86-SSE2-NEXT: pandn %xmm1, %xmm6
-; X86-SSE2-NEXT: psrlw $2, %xmm1
-; X86-SSE2-NEXT: pand %xmm5, %xmm1
-; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
-; X86-SSE2-NEXT: por %xmm6, %xmm1
+; X86-SSE2-NEXT: pxor %xmm4, %xmm4
+; X86-SSE2-NEXT: pcmpgtb %xmm2, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm5
+; X86-SSE2-NEXT: pandn %xmm0, %xmm5
+; X86-SSE2-NEXT: psllw $2, %xmm0
+; X86-SSE2-NEXT: pand %xmm4, %xmm0
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT: por %xmm5, %xmm0
; X86-SSE2-NEXT: paddb %xmm2, %xmm2
; X86-SSE2-NEXT: pcmpgtb %xmm2, %xmm3
; X86-SSE2-NEXT: movdqa %xmm3, %xmm2
-; X86-SSE2-NEXT: pandn %xmm1, %xmm2
-; X86-SSE2-NEXT: psrlw $1, %xmm1
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
-; X86-SSE2-NEXT: por %xmm2, %xmm1
-; X86-SSE2-NEXT: por %xmm4, %xmm1
-; X86-SSE2-NEXT: por %xmm1, %xmm0
+; X86-SSE2-NEXT: pandn %xmm0, %xmm2
+; X86-SSE2-NEXT: por %xmm1, %xmm2
+; X86-SSE2-NEXT: paddb %xmm0, %xmm0
+; X86-SSE2-NEXT: pand %xmm3, %xmm0
+; X86-SSE2-NEXT: por %xmm2, %xmm0
; X86-SSE2-NEXT: retl
%res = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
ret <16 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index 48a1d4f3209a..6df47bcc8377 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -758,45 +758,45 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
; XOPAVX1-LABEL: var_funnnel_v32i8:
; XOPAVX1: # %bb.0:
; XOPAVX1-NEXT: vmovaps {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOPAVX1-NEXT: vandnps %ymm3, %ymm2, %ymm4
+; XOPAVX1-NEXT: vandps %ymm3, %ymm2, %ymm4
; XOPAVX1-NEXT: vextractf128 $1, %ymm4, %xmm5
-; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
-; XOPAVX1-NEXT: vpaddb %xmm6, %xmm6, %xmm6
-; XOPAVX1-NEXT: vpshlb %xmm5, %xmm6, %xmm5
-; XOPAVX1-NEXT: vpaddb %xmm0, %xmm0, %xmm0
-; XOPAVX1-NEXT: vpshlb %xmm4, %xmm0, %xmm0
-; XOPAVX1-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm0
-; XOPAVX1-NEXT: vandps %ymm3, %ymm2, %ymm2
+; XOPAVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6
+; XOPAVX1-NEXT: vpsubb %xmm5, %xmm6, %xmm5
+; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm7
+; XOPAVX1-NEXT: vpshlb %xmm5, %xmm7, %xmm5
+; XOPAVX1-NEXT: vpsubb %xmm4, %xmm6, %xmm4
+; XOPAVX1-NEXT: vpshlb %xmm4, %xmm1, %xmm1
+; XOPAVX1-NEXT: vinsertf128 $1, %xmm5, %ymm1, %ymm1
+; XOPAVX1-NEXT: vandnps %ymm3, %ymm2, %ymm2
; XOPAVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
-; XOPAVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT: vpsubb %xmm3, %xmm4, %xmm3
-; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
-; XOPAVX1-NEXT: vpshlb %xmm3, %xmm5, %xmm3
-; XOPAVX1-NEXT: vpsubb %xmm2, %xmm4, %xmm2
-; XOPAVX1-NEXT: vpshlb %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; XOPAVX1-NEXT: vpaddb %xmm4, %xmm4, %xmm4
+; XOPAVX1-NEXT: vpshlb %xmm3, %xmm4, %xmm3
+; XOPAVX1-NEXT: vpaddb %xmm0, %xmm0, %xmm0
+; XOPAVX1-NEXT: vpshlb %xmm2, %xmm0, %xmm0
+; XOPAVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
; XOPAVX1-NEXT: vorps %ymm1, %ymm0, %ymm0
; XOPAVX1-NEXT: retq
;
; XOPAVX2-LABEL: var_funnnel_v32i8:
; XOPAVX2: # %bb.0:
; XOPAVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOPAVX2-NEXT: vpandn %ymm3, %ymm2, %ymm4
+; XOPAVX2-NEXT: vpand %ymm3, %ymm2, %ymm4
; XOPAVX2-NEXT: vextracti128 $1, %ymm4, %xmm5
-; XOPAVX2-NEXT: vpaddb %ymm0, %ymm0, %ymm0
-; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm6
-; XOPAVX2-NEXT: vpshlb %xmm5, %xmm6, %xmm5
-; XOPAVX2-NEXT: vpshlb %xmm4, %xmm0, %xmm0
-; XOPAVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm0
-; XOPAVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
+; XOPAVX2-NEXT: vpxor %xmm6, %xmm6, %xmm6
+; XOPAVX2-NEXT: vpsubb %xmm5, %xmm6, %xmm5
+; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm7
+; XOPAVX2-NEXT: vpshlb %xmm5, %xmm7, %xmm5
+; XOPAVX2-NEXT: vpsubb %xmm4, %xmm6, %xmm4
+; XOPAVX2-NEXT: vpshlb %xmm4, %xmm1, %xmm1
+; XOPAVX2-NEXT: vinserti128 $1, %xmm5, %ymm1, %ymm1
+; XOPAVX2-NEXT: vpandn %ymm3, %ymm2, %ymm2
; XOPAVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
-; XOPAVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT: vpsubb %xmm3, %xmm4, %xmm3
-; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
-; XOPAVX2-NEXT: vpshlb %xmm3, %xmm5, %xmm3
-; XOPAVX2-NEXT: vpsubb %xmm2, %xmm4, %xmm2
-; XOPAVX2-NEXT: vpshlb %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
+; XOPAVX2-NEXT: vpaddb %ymm0, %ymm0, %ymm0
+; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm4
+; XOPAVX2-NEXT: vpshlb %xmm3, %xmm4, %xmm3
+; XOPAVX2-NEXT: vpshlb %xmm2, %xmm0, %xmm0
+; XOPAVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
; XOPAVX2-NEXT: vpor %ymm1, %ymm0, %ymm0
; XOPAVX2-NEXT: retq
%res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
index cdb7ffbce1ac..7bd47f0c21f2 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
@@ -553,10 +553,10 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
; SSE2-NEXT: pandn %xmm1, %xmm2
; SSE2-NEXT: por %xmm4, %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm1
-; SSE2-NEXT: paddb %xmm2, %xmm1
+; SSE2-NEXT: psrlw $7, %xmm1
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; SSE2-NEXT: movdqa %xmm2, %xmm4
-; SSE2-NEXT: psrlw $7, %xmm4
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
+; SSE2-NEXT: paddb %xmm2, %xmm4
; SSE2-NEXT: por %xmm1, %xmm4
; SSE2-NEXT: paddb %xmm3, %xmm3
; SSE2-NEXT: pcmpgtb %xmm3, %xmm0
@@ -588,10 +588,10 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
; SSE41-NEXT: paddb %xmm0, %xmm0
; SSE41-NEXT: pblendvb %xmm0, %xmm3, %xmm2
; SSE41-NEXT: movdqa %xmm2, %xmm1
-; SSE41-NEXT: paddb %xmm2, %xmm1
+; SSE41-NEXT: psrlw $7, %xmm1
+; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; SSE41-NEXT: movdqa %xmm2, %xmm3
-; SSE41-NEXT: psrlw $7, %xmm3
-; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE41-NEXT: paddb %xmm2, %xmm3
; SSE41-NEXT: por %xmm1, %xmm3
; SSE41-NEXT: paddb %xmm0, %xmm0
; SSE41-NEXT: pblendvb %xmm0, %xmm3, %xmm2
@@ -616,10 +616,10 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
; AVX-NEXT: vpor %xmm2, %xmm3, %xmm2
; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1
; AVX-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; AVX-NEXT: vpaddb %xmm0, %xmm0, %xmm2
-; AVX-NEXT: vpsrlw $7, %xmm0, %xmm3
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3
-; AVX-NEXT: vpor %xmm3, %xmm2, %xmm2
+; AVX-NEXT: vpsrlw $7, %xmm0, %xmm2
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX-NEXT: vpaddb %xmm0, %xmm0, %xmm3
+; AVX-NEXT: vpor %xmm2, %xmm3, %xmm2
; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1
; AVX-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
; AVX-NEXT: retq
@@ -768,10 +768,10 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
; X86-SSE2-NEXT: pandn %xmm1, %xmm2
; X86-SSE2-NEXT: por %xmm4, %xmm2
; X86-SSE2-NEXT: movdqa %xmm2, %xmm1
-; X86-SSE2-NEXT: paddb %xmm2, %xmm1
+; X86-SSE2-NEXT: psrlw $7, %xmm1
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
; X86-SSE2-NEXT: movdqa %xmm2, %xmm4
-; X86-SSE2-NEXT: psrlw $7, %xmm4
-; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm4
+; X86-SSE2-NEXT: paddb %xmm2, %xmm4
; X86-SSE2-NEXT: por %xmm1, %xmm4
; X86-SSE2-NEXT: paddb %xmm3, %xmm3
; X86-SSE2-NEXT: pcmpgtb %xmm3, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
index 287bc1526e31..1566a18370ef 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
@@ -480,10 +480,10 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
; AVX2-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1
; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpaddb %ymm0, %ymm0, %ymm2
-; AVX2-NEXT: vpsrlw $7, %ymm0, %ymm3
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
-; AVX2-NEXT: vpor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpsrlw $7, %ymm0, %ymm2
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT: vpaddb %ymm0, %ymm0, %ymm3
+; AVX2-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1
; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
; AVX2-NEXT: retq
@@ -506,10 +506,10 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
; AVX512F-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1
; AVX512F-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT: vpaddb %ymm0, %ymm0, %ymm2
-; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm3
-; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
-; AVX512F-NEXT: vpor %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm2
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT: vpaddb %ymm0, %ymm0, %ymm3
+; AVX512F-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1
; AVX512F-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
; AVX512F-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
index ea34c30f6848..543a5cbab0d0 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
@@ -272,154 +272,154 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
;
; AVX512BW-LABEL: var_funnnel_v64i8:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512BW-NEXT: vpandq %zmm2, %zmm1, %zmm3
-; AVX512BW-NEXT: vpsllw $5, %zmm3, %zmm3
-; AVX512BW-NEXT: vpaddb %zmm3, %zmm3, %zmm4
+; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT: vpsubb %zmm1, %zmm2, %zmm2
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm2
+; AVX512BW-NEXT: vpsllw $5, %zmm2, %zmm2
+; AVX512BW-NEXT: vpaddb %zmm2, %zmm2, %zmm4
; AVX512BW-NEXT: vpmovb2m %zmm4, %k1
-; AVX512BW-NEXT: vpmovb2m %zmm3, %k2
-; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm3
-; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm3
-; AVX512BW-NEXT: vpblendmb %zmm3, %zmm0, %zmm3 {%k2}
-; AVX512BW-NEXT: vpsrlw $2, %zmm3, %zmm5
-; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
-; AVX512BW-NEXT: vmovdqu8 %zmm5, %zmm3 {%k1}
-; AVX512BW-NEXT: vpsrlw $1, %zmm3, %zmm5
+; AVX512BW-NEXT: vpmovb2m %zmm2, %k2
+; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm2
+; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2
+; AVX512BW-NEXT: vpblendmb %zmm2, %zmm0, %zmm2 {%k2}
+; AVX512BW-NEXT: vpsllw $2, %zmm2, %zmm5
; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
+; AVX512BW-NEXT: vmovdqu8 %zmm5, %zmm2 {%k1}
; AVX512BW-NEXT: vpaddb %zmm4, %zmm4, %zmm4
; AVX512BW-NEXT: vpmovb2m %zmm4, %k1
-; AVX512BW-NEXT: vmovdqu8 %zmm5, %zmm3 {%k1}
-; AVX512BW-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512BW-NEXT: vpsubb %zmm1, %zmm4, %zmm1
-; AVX512BW-NEXT: vpandq %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT: vpaddb %zmm2, %zmm2, %zmm2 {%k1}
+; AVX512BW-NEXT: vpandq %zmm3, %zmm1, %zmm1
; AVX512BW-NEXT: vpsllw $5, %zmm1, %zmm1
-; AVX512BW-NEXT: vpaddb %zmm1, %zmm1, %zmm2
-; AVX512BW-NEXT: vpmovb2m %zmm2, %k1
+; AVX512BW-NEXT: vpaddb %zmm1, %zmm1, %zmm3
+; AVX512BW-NEXT: vpmovb2m %zmm3, %k1
; AVX512BW-NEXT: vpmovb2m %zmm1, %k2
-; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm1
+; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm1
; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512BW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k2}
-; AVX512BW-NEXT: vpsllw $2, %zmm0, %zmm1
+; AVX512BW-NEXT: vpsrlw $2, %zmm0, %zmm1
; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512BW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
-; AVX512BW-NEXT: vpaddb %zmm2, %zmm2, %zmm1
-; AVX512BW-NEXT: vpmovb2m %zmm1, %k1
-; AVX512BW-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1}
-; AVX512BW-NEXT: vporq %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT: vpsrlw $1, %zmm0, %zmm1
+; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT: vpaddb %zmm3, %zmm3, %zmm3
+; AVX512BW-NEXT: vpmovb2m %zmm3, %k1
+; AVX512BW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512BW-NEXT: vporq %zmm2, %zmm0, %zmm0
; AVX512BW-NEXT: retq
;
; AVX512VLBW-LABEL: var_funnnel_v64i8:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vmovdqa64 {{.*#+}} zmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VLBW-NEXT: vpandq %zmm2, %zmm1, %zmm3
-; AVX512VLBW-NEXT: vpsllw $5, %zmm3, %zmm3
-; AVX512VLBW-NEXT: vpaddb %zmm3, %zmm3, %zmm4
+; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLBW-NEXT: vpsubb %zmm1, %zmm2, %zmm2
+; AVX512VLBW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLBW-NEXT: vpandq %zmm3, %zmm2, %zmm2
+; AVX512VLBW-NEXT: vpsllw $5, %zmm2, %zmm2
+; AVX512VLBW-NEXT: vpaddb %zmm2, %zmm2, %zmm4
; AVX512VLBW-NEXT: vpmovb2m %zmm4, %k1
-; AVX512VLBW-NEXT: vpmovb2m %zmm3, %k2
-; AVX512VLBW-NEXT: vpsrlw $4, %zmm0, %zmm3
-; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm3
-; AVX512VLBW-NEXT: vpblendmb %zmm3, %zmm0, %zmm3 {%k2}
-; AVX512VLBW-NEXT: vpsrlw $2, %zmm3, %zmm5
-; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
-; AVX512VLBW-NEXT: vmovdqu8 %zmm5, %zmm3 {%k1}
-; AVX512VLBW-NEXT: vpsrlw $1, %zmm3, %zmm5
+; AVX512VLBW-NEXT: vpmovb2m %zmm2, %k2
+; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm2
+; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2
+; AVX512VLBW-NEXT: vpblendmb %zmm2, %zmm0, %zmm2 {%k2}
+; AVX512VLBW-NEXT: vpsllw $2, %zmm2, %zmm5
; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
+; AVX512VLBW-NEXT: vmovdqu8 %zmm5, %zmm2 {%k1}
; AVX512VLBW-NEXT: vpaddb %zmm4, %zmm4, %zmm4
; AVX512VLBW-NEXT: vpmovb2m %zmm4, %k1
-; AVX512VLBW-NEXT: vmovdqu8 %zmm5, %zmm3 {%k1}
-; AVX512VLBW-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBW-NEXT: vpsubb %zmm1, %zmm4, %zmm1
-; AVX512VLBW-NEXT: vpandq %zmm2, %zmm1, %zmm1
+; AVX512VLBW-NEXT: vpaddb %zmm2, %zmm2, %zmm2 {%k1}
+; AVX512VLBW-NEXT: vpandq %zmm3, %zmm1, %zmm1
; AVX512VLBW-NEXT: vpsllw $5, %zmm1, %zmm1
-; AVX512VLBW-NEXT: vpaddb %zmm1, %zmm1, %zmm2
-; AVX512VLBW-NEXT: vpmovb2m %zmm2, %k1
+; AVX512VLBW-NEXT: vpaddb %zmm1, %zmm1, %zmm3
+; AVX512VLBW-NEXT: vpmovb2m %zmm3, %k1
; AVX512VLBW-NEXT: vpmovb2m %zmm1, %k2
-; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm1
+; AVX512VLBW-NEXT: vpsrlw $4, %zmm0, %zmm1
; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLBW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k2}
-; AVX512VLBW-NEXT: vpsllw $2, %zmm0, %zmm1
+; AVX512VLBW-NEXT: vpsrlw $2, %zmm0, %zmm1
; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLBW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
-; AVX512VLBW-NEXT: vpaddb %zmm2, %zmm2, %zmm1
-; AVX512VLBW-NEXT: vpmovb2m %zmm1, %k1
-; AVX512VLBW-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1}
-; AVX512VLBW-NEXT: vporq %zmm0, %zmm3, %zmm0
+; AVX512VLBW-NEXT: vpsrlw $1, %zmm0, %zmm1
+; AVX512VLBW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VLBW-NEXT: vpaddb %zmm3, %zmm3, %zmm3
+; AVX512VLBW-NEXT: vpmovb2m %zmm3, %k1
+; AVX512VLBW-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VLBW-NEXT: vporq %zmm2, %zmm0, %zmm0
; AVX512VLBW-NEXT: retq
;
; AVX512VBMI2-LABEL: var_funnnel_v64i8:
; AVX512VBMI2: # %bb.0:
-; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VBMI2-NEXT: vpandq %zmm2, %zmm1, %zmm3
-; AVX512VBMI2-NEXT: vpsllw $5, %zmm3, %zmm3
-; AVX512VBMI2-NEXT: vpaddb %zmm3, %zmm3, %zmm4
+; AVX512VBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VBMI2-NEXT: vpsubb %zmm1, %zmm2, %zmm2
+; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT: vpandq %zmm3, %zmm2, %zmm2
+; AVX512VBMI2-NEXT: vpsllw $5, %zmm2, %zmm2
+; AVX512VBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm4
; AVX512VBMI2-NEXT: vpmovb2m %zmm4, %k1
-; AVX512VBMI2-NEXT: vpmovb2m %zmm3, %k2
-; AVX512VBMI2-NEXT: vpsrlw $4, %zmm0, %zmm3
-; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm3
-; AVX512VBMI2-NEXT: vpblendmb %zmm3, %zmm0, %zmm3 {%k2}
-; AVX512VBMI2-NEXT: vpsrlw $2, %zmm3, %zmm5
-; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
-; AVX512VBMI2-NEXT: vmovdqu8 %zmm5, %zmm3 {%k1}
-; AVX512VBMI2-NEXT: vpsrlw $1, %zmm3, %zmm5
+; AVX512VBMI2-NEXT: vpmovb2m %zmm2, %k2
+; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm2
+; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2
+; AVX512VBMI2-NEXT: vpblendmb %zmm2, %zmm0, %zmm2 {%k2}
+; AVX512VBMI2-NEXT: vpsllw $2, %zmm2, %zmm5
; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
+; AVX512VBMI2-NEXT: vmovdqu8 %zmm5, %zmm2 {%k1}
; AVX512VBMI2-NEXT: vpaddb %zmm4, %zmm4, %zmm4
; AVX512VBMI2-NEXT: vpmovb2m %zmm4, %k1
-; AVX512VBMI2-NEXT: vmovdqu8 %zmm5, %zmm3 {%k1}
-; AVX512VBMI2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VBMI2-NEXT: vpsubb %zmm1, %zmm4, %zmm1
-; AVX512VBMI2-NEXT: vpandq %zmm2, %zmm1, %zmm1
+; AVX512VBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm2 {%k1}
+; AVX512VBMI2-NEXT: vpandq %zmm3, %zmm1, %zmm1
; AVX512VBMI2-NEXT: vpsllw $5, %zmm1, %zmm1
-; AVX512VBMI2-NEXT: vpaddb %zmm1, %zmm1, %zmm2
-; AVX512VBMI2-NEXT: vpmovb2m %zmm2, %k1
+; AVX512VBMI2-NEXT: vpaddb %zmm1, %zmm1, %zmm3
+; AVX512VBMI2-NEXT: vpmovb2m %zmm3, %k1
; AVX512VBMI2-NEXT: vpmovb2m %zmm1, %k2
-; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm1
+; AVX512VBMI2-NEXT: vpsrlw $4, %zmm0, %zmm1
; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k2}
-; AVX512VBMI2-NEXT: vpsllw $2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT: vpsrlw $2, %zmm0, %zmm1
; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
-; AVX512VBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm1
-; AVX512VBMI2-NEXT: vpmovb2m %zmm1, %k1
-; AVX512VBMI2-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1}
-; AVX512VBMI2-NEXT: vporq %zmm0, %zmm3, %zmm0
+; AVX512VBMI2-NEXT: vpsrlw $1, %zmm0, %zmm1
+; AVX512VBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VBMI2-NEXT: vpaddb %zmm3, %zmm3, %zmm3
+; AVX512VBMI2-NEXT: vpmovb2m %zmm3, %k1
+; AVX512VBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT: vporq %zmm2, %zmm0, %zmm0
; AVX512VBMI2-NEXT: retq
;
; AVX512VLVBMI2-LABEL: var_funnnel_v64i8:
; AVX512VLVBMI2: # %bb.0:
-; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VLVBMI2-NEXT: vpandq %zmm2, %zmm1, %zmm3
-; AVX512VLVBMI2-NEXT: vpsllw $5, %zmm3, %zmm3
-; AVX512VLVBMI2-NEXT: vpaddb %zmm3, %zmm3, %zmm4
+; AVX512VLVBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT: vpsubb %zmm1, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT: vpandq %zmm3, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT: vpsllw $5, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm4
; AVX512VLVBMI2-NEXT: vpmovb2m %zmm4, %k1
-; AVX512VLVBMI2-NEXT: vpmovb2m %zmm3, %k2
-; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm0, %zmm3
-; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm3
-; AVX512VLVBMI2-NEXT: vpblendmb %zmm3, %zmm0, %zmm3 {%k2}
-; AVX512VLVBMI2-NEXT: vpsrlw $2, %zmm3, %zmm5
-; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
-; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm5, %zmm3 {%k1}
-; AVX512VLVBMI2-NEXT: vpsrlw $1, %zmm3, %zmm5
+; AVX512VLVBMI2-NEXT: vpmovb2m %zmm2, %k2
+; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm2
+; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT: vpblendmb %zmm2, %zmm0, %zmm2 {%k2}
+; AVX512VLVBMI2-NEXT: vpsllw $2, %zmm2, %zmm5
; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm5, %zmm2 {%k1}
; AVX512VLVBMI2-NEXT: vpaddb %zmm4, %zmm4, %zmm4
; AVX512VLVBMI2-NEXT: vpmovb2m %zmm4, %k1
-; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm5, %zmm3 {%k1}
-; AVX512VLVBMI2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLVBMI2-NEXT: vpsubb %zmm1, %zmm4, %zmm1
-; AVX512VLVBMI2-NEXT: vpandq %zmm2, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm2 {%k1}
+; AVX512VLVBMI2-NEXT: vpandq %zmm3, %zmm1, %zmm1
; AVX512VLVBMI2-NEXT: vpsllw $5, %zmm1, %zmm1
-; AVX512VLVBMI2-NEXT: vpaddb %zmm1, %zmm1, %zmm2
-; AVX512VLVBMI2-NEXT: vpmovb2m %zmm2, %k1
+; AVX512VLVBMI2-NEXT: vpaddb %zmm1, %zmm1, %zmm3
+; AVX512VLVBMI2-NEXT: vpmovb2m %zmm3, %k1
; AVX512VLVBMI2-NEXT: vpmovb2m %zmm1, %k2
-; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm0, %zmm1
; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k2}
-; AVX512VLVBMI2-NEXT: vpsllw $2, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT: vpsrlw $2, %zmm0, %zmm1
; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
-; AVX512VLVBMI2-NEXT: vpaddb %zmm2, %zmm2, %zmm1
-; AVX512VLVBMI2-NEXT: vpmovb2m %zmm1, %k1
-; AVX512VLVBMI2-NEXT: vpaddb %zmm0, %zmm0, %zmm0 {%k1}
-; AVX512VLVBMI2-NEXT: vporq %zmm0, %zmm3, %zmm0
+; AVX512VLVBMI2-NEXT: vpsrlw $1, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT: vpaddb %zmm3, %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT: vpmovb2m %zmm3, %k1
+; AVX512VLVBMI2-NEXT: vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT: vporq %zmm2, %zmm0, %zmm0
; AVX512VLVBMI2-NEXT: retq
%res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> %amt)
ret <64 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll
index e3edd9e33ea5..d43d94548721 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-128.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll
@@ -506,10 +506,10 @@ define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
; SSE2-NEXT: pandn %xmm3, %xmm2
; SSE2-NEXT: por %xmm4, %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm3
-; SSE2-NEXT: paddb %xmm2, %xmm3
+; SSE2-NEXT: psrlw $7, %xmm3
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
; SSE2-NEXT: movdqa %xmm2, %xmm4
-; SSE2-NEXT: psrlw $7, %xmm4
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
+; SSE2-NEXT: paddb %xmm2, %xmm4
; SSE2-NEXT: por %xmm3, %xmm4
; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pcmpgtb %xmm1, %xmm0
@@ -542,10 +542,10 @@ define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
; SSE41-NEXT: movdqa %xmm2, %xmm0
; SSE41-NEXT: pblendvb %xmm0, %xmm3, %xmm1
; SSE41-NEXT: movdqa %xmm1, %xmm0
-; SSE41-NEXT: paddb %xmm1, %xmm0
+; SSE41-NEXT: psrlw $7, %xmm0
+; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE41-NEXT: movdqa %xmm1, %xmm3
-; SSE41-NEXT: psrlw $7, %xmm3
-; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE41-NEXT: paddb %xmm1, %xmm3
; SSE41-NEXT: por %xmm0, %xmm3
; SSE41-NEXT: paddb %xmm2, %xmm2
; SSE41-NEXT: movdqa %xmm2, %xmm0
@@ -569,10 +569,10 @@ define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
; AVX-NEXT: vpor %xmm2, %xmm3, %xmm2
; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1
; AVX-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; AVX-NEXT: vpaddb %xmm0, %xmm0, %xmm2
-; AVX-NEXT: vpsrlw $7, %xmm0, %xmm3
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3
-; AVX-NEXT: vpor %xmm3, %xmm2, %xmm2
+; AVX-NEXT: vpsrlw $7, %xmm0, %xmm2
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX-NEXT: vpaddb %xmm0, %xmm0, %xmm3
+; AVX-NEXT: vpor %xmm2, %xmm3, %xmm2
; AVX-NEXT: vpaddb %xmm1, %xmm1, %xmm1
; AVX-NEXT: vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
; AVX-NEXT: retq
@@ -699,10 +699,10 @@ define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
; X86-SSE2-NEXT: pandn %xmm3, %xmm2
; X86-SSE2-NEXT: por %xmm4, %xmm2
; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
-; X86-SSE2-NEXT: paddb %xmm2, %xmm3
+; X86-SSE2-NEXT: psrlw $7, %xmm3
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm3
; X86-SSE2-NEXT: movdqa %xmm2, %xmm4
-; X86-SSE2-NEXT: psrlw $7, %xmm4
-; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm4
+; X86-SSE2-NEXT: paddb %xmm2, %xmm4
; X86-SSE2-NEXT: por %xmm3, %xmm4
; X86-SSE2-NEXT: paddb %xmm1, %xmm1
; X86-SSE2-NEXT: pcmpgtb %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll
index aecf789745d1..bb853d717336 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-256.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll
@@ -439,10 +439,10 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX2-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1
; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpaddb %ymm0, %ymm0, %ymm2
-; AVX2-NEXT: vpsrlw $7, %ymm0, %ymm3
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
-; AVX2-NEXT: vpor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpsrlw $7, %ymm0, %ymm2
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT: vpaddb %ymm0, %ymm0, %ymm3
+; AVX2-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1
; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
; AVX2-NEXT: retq
@@ -463,10 +463,10 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX512F-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1
; AVX512F-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT: vpaddb %ymm0, %ymm0, %ymm2
-; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm3
-; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
-; AVX512F-NEXT: vpor %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm2
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT: vpaddb %ymm0, %ymm0, %ymm3
+; AVX512F-NEXT: vpor %ymm2, %ymm3, %ymm2
; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1
; AVX512F-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
; AVX512F-NEXT: retq
More information about the llvm-commits
mailing list