[llvm] [X86] Lower vector i8 ashr-by-1 using pavgb (PR #198487)
via llvm-commits
llvm-commits at lists.llvm.org
Tue May 19 07:50:55 PDT 2026
https://github.com/Harishankar14 updated https://github.com/llvm/llvm-project/pull/198487
>From 92b9bda1bb814c4cc454e9c876c4cc864aa1e99f Mon Sep 17 00:00:00 2001
From: Harishankar <harishankarpp7 at gmail.com>
Date: Tue, 19 May 2026 15:46:17 +0530
Subject: [PATCH] [X86] Lower vector i8 ashr-by-1 using pavgb
For vector i8 arithmetic shift right by 1, the current lowering produces
a 5-instruction sequence (psrlw + pand + xor + psubb plus a constant
load) with a 4-deep dependency chain.
This patch uses the identity
ashr(x, 1) == avgceilu(x, -1) ^ (~x & 0x80)
to lower to ISD::AVGCEILU + a short fixup, producing 4 instructions on
SSE/AVX/AVX2 and 3 on AVX-512BW (after vpternlogd fusion of the AND/XOR
pair), with two parallel dependency chains instead of one long one.
The freeze on R is required because the target reads it twice, matching
the pattern of the existing `shl R, 1 -> add R, R` case in
LowerShiftByScalarImmediate.
Alive2 proof: https://alive2.llvm.org/ce/z/LbXPhE
Fixes #198061
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 10 +
llvm/test/CodeGen/X86/avgfloors.ll | 546 +++++++++---------
.../test/CodeGen/X86/vector-shift-ashr-128.ll | 68 +--
.../test/CodeGen/X86/vector-shift-ashr-256.ll | 91 ++-
.../test/CodeGen/X86/vector-shift-ashr-512.ll | 23 +-
.../CodeGen/X86/vector-shift-ashr-sub128.ll | 204 +++----
...vector_splat-const-shift-of-constmasked.ll | 72 +--
7 files changed, 430 insertions(+), 584 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index c91143fefb6af..def772fa55808 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -31067,6 +31067,16 @@ static SDValue LowerShiftByScalarImmediate(SDValue Op, SelectionDAG &DAG,
return DAG.getNode(ISD::AND, dl, VT, SRL, DAG.getConstant(Mask, dl, VT));
}
if (Op.getOpcode() == ISD::SRA) {
+ // ashr(R, 1) === xor(avgceilu(R,-1),and(not(R),MIN_SIGNED))
+ if (ShiftAmt == 1) {
+ R = DAG.getFreeze(R);
+ SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
+ SDValue Avg = DAG.getNode(ISD::AVGCEILU, dl, VT, R, AllOnes);
+ SDValue Not = DAG.getNode(ISD::XOR, dl, VT, R, AllOnes);
+ SDValue Hi =
+ DAG.getNode(ISD::AND, dl, VT, Not, DAG.getConstant(0x80, dl, VT));
+ return DAG.getNode(ISD::XOR, dl, VT, Avg, Hi);
+ }
// ashr(R, Amt) === sub(xor(lshr(R, Amt), Mask), Mask)
SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
diff --git a/llvm/test/CodeGen/X86/avgfloors.ll b/llvm/test/CodeGen/X86/avgfloors.ll
index 9cc55c6f7a81f..284ba912532de 100644
--- a/llvm/test/CodeGen/X86/avgfloors.ll
+++ b/llvm/test/CodeGen/X86/avgfloors.ll
@@ -13,49 +13,45 @@ define <16 x i8> @test_fixed_v16i8(<16 x i8> %a0, <16 x i8> %a1) nounwind {
; SSE-LABEL: test_fixed_v16i8:
; SSE: # %bb.0:
; SSE-NEXT: movdqa %xmm0, %xmm2
-; SSE-NEXT: pand %xmm1, %xmm2
-; SSE-NEXT: pxor %xmm1, %xmm0
-; SSE-NEXT: psrlw $1, %xmm0
-; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; SSE-NEXT: pxor %xmm1, %xmm0
+; SSE-NEXT: pxor %xmm1, %xmm2
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pavgb %xmm2, %xmm3
+; SSE-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE-NEXT: pxor %xmm3, %xmm2
+; SSE-NEXT: pand %xmm1, %xmm0
; SSE-NEXT: paddb %xmm2, %xmm0
-; SSE-NEXT: psubb %xmm1, %xmm0
; SSE-NEXT: retq
;
; AVX1-LABEL: test_fixed_v16i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm2
+; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
+; AVX1-NEXT: vpavgb %xmm3, %xmm2, %xmm3
+; AVX1-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT: vpxor %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpsubb %xmm1, %xmm0, %xmm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: test_fixed_v16i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm2
+; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpavgb %xmm3, %xmm2, %xmm3
+; AVX2-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: vpsubb %xmm1, %xmm0, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_fixed_v16i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX512-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
-; AVX512-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; AVX512-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
+; AVX512-NEXT: vpavgb %xmm3, %xmm2, %xmm3
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 ^ (m32bcst & ~xmm2)
+; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddb %xmm3, %xmm0, %xmm0
; AVX512-NEXT: retq
%and = and <16 x i8> %a0, %a1
%xor = xor <16 x i8> %a0, %a1
@@ -68,49 +64,45 @@ define <16 x i8> @test_ext_v16i8(<16 x i8> %a0, <16 x i8> %a1) nounwind {
; SSE-LABEL: test_ext_v16i8:
; SSE: # %bb.0:
; SSE-NEXT: movdqa %xmm0, %xmm2
-; SSE-NEXT: pand %xmm1, %xmm2
-; SSE-NEXT: pxor %xmm1, %xmm0
-; SSE-NEXT: psrlw $1, %xmm0
-; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; SSE-NEXT: pxor %xmm1, %xmm0
+; SSE-NEXT: pxor %xmm1, %xmm2
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pavgb %xmm2, %xmm3
+; SSE-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE-NEXT: pxor %xmm3, %xmm2
+; SSE-NEXT: pand %xmm1, %xmm0
; SSE-NEXT: paddb %xmm2, %xmm0
-; SSE-NEXT: psubb %xmm1, %xmm0
; SSE-NEXT: retq
;
; AVX1-LABEL: test_ext_v16i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm2
+; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
+; AVX1-NEXT: vpavgb %xmm3, %xmm2, %xmm3
+; AVX1-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT: vpxor %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpsubb %xmm1, %xmm0, %xmm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: test_ext_v16i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm2
+; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpavgb %xmm3, %xmm2, %xmm3
+; AVX2-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: vpsubb %xmm1, %xmm0, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_ext_v16i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX512-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
-; AVX512-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; AVX512-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm0, %xmm2
+; AVX512-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
+; AVX512-NEXT: vpavgb %xmm3, %xmm2, %xmm3
+; AVX512-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 ^ (m32bcst & ~xmm2)
+; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddb %xmm3, %xmm0, %xmm0
; AVX512-NEXT: retq
%x0 = sext <16 x i8> %a0 to <16 x i16>
%x1 = sext <16 x i8> %a1 to <16 x i16>
@@ -347,67 +339,63 @@ define <32 x i8> @test_fixed_v32i8(<32 x i8> %a0, <32 x i8> %a1) nounwind {
; SSE-LABEL: test_fixed_v32i8:
; SSE: # %bb.0:
; SSE-NEXT: movdqa %xmm0, %xmm4
-; SSE-NEXT: pand %xmm2, %xmm4
-; SSE-NEXT: pxor %xmm2, %xmm0
-; SSE-NEXT: psrlw $1, %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE-NEXT: pxor %xmm2, %xmm4
+; SSE-NEXT: movdqa {{.*#+}} xmm5 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; SSE-NEXT: movdqa %xmm4, %xmm6
+; SSE-NEXT: pandn %xmm5, %xmm6
+; SSE-NEXT: pcmpeqd %xmm7, %xmm7
+; SSE-NEXT: pavgb %xmm7, %xmm4
+; SSE-NEXT: pxor %xmm6, %xmm4
; SSE-NEXT: pand %xmm2, %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm5 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; SSE-NEXT: pxor %xmm5, %xmm0
; SSE-NEXT: paddb %xmm4, %xmm0
-; SSE-NEXT: psubb %xmm5, %xmm0
-; SSE-NEXT: movdqa %xmm1, %xmm4
-; SSE-NEXT: pand %xmm3, %xmm4
-; SSE-NEXT: pxor %xmm3, %xmm1
-; SSE-NEXT: psrlw $1, %xmm1
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: pxor %xmm5, %xmm1
-; SSE-NEXT: paddb %xmm4, %xmm1
-; SSE-NEXT: psubb %xmm5, %xmm1
+; SSE-NEXT: movdqa %xmm1, %xmm2
+; SSE-NEXT: pxor %xmm3, %xmm2
+; SSE-NEXT: movdqa %xmm2, %xmm4
+; SSE-NEXT: pandn %xmm5, %xmm4
+; SSE-NEXT: pavgb %xmm7, %xmm2
+; SSE-NEXT: pxor %xmm4, %xmm2
+; SSE-NEXT: pand %xmm3, %xmm1
+; SSE-NEXT: paddb %xmm2, %xmm1
; SSE-NEXT: retq
;
; AVX1-LABEL: test_fixed_v32i8:
; AVX1: # %bb.0:
; AVX1-NEXT: vxorps %ymm1, %ymm0, %ymm2
; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
-; AVX1-NEXT: vpsrlw $1, %xmm3, %xmm3
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX1-NEXT: vpandn %xmm4, %xmm3, %xmm5
+; AVX1-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
+; AVX1-NEXT: vpavgb %xmm6, %xmm3, %xmm3
; AVX1-NEXT: vpxor %xmm5, %xmm3, %xmm3
; AVX1-NEXT: vandps %ymm1, %ymm0, %ymm0
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpaddb %xmm1, %xmm3, %xmm1
-; AVX1-NEXT: vpsubb %xmm5, %xmm1, %xmm1
-; AVX1-NEXT: vpsrlw $1, %xmm2, %xmm2
-; AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpxor %xmm5, %xmm2, %xmm2
-; AVX1-NEXT: vpaddb %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vpsubb %xmm5, %xmm0, %xmm0
+; AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT: vpandn %xmm4, %xmm2, %xmm3
+; AVX1-NEXT: vpavgb %xmm6, %xmm2, %xmm2
+; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: test_fixed_v32i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm2
-; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpsrlw $1, %ymm0, %ymm0
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm2
+; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX2-NEXT: vpavgb %ymm3, %ymm2, %ymm3
+; AVX2-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT: vpxor %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_fixed_v32i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpand %ymm1, %ymm0, %ymm2
-; AVX512-NEXT: vpxor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpsrlw $1, %ymm0, %ymm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT: vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & m32bcst)
-; AVX512-NEXT: vpaddb %ymm2, %ymm0, %ymm0
-; AVX512-NEXT: vpsubb %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpxor %ymm1, %ymm0, %ymm2
+; AVX512-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX512-NEXT: vpavgb %ymm3, %ymm2, %ymm3
+; AVX512-NEXT: vpternlogd {{.*#+}} ymm3 = ymm3 ^ (m32bcst & ~ymm2)
+; AVX512-NEXT: vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddb %ymm3, %ymm0, %ymm0
; AVX512-NEXT: retq
%and = and <32 x i8> %a0, %a1
%xor = xor <32 x i8> %a0, %a1
@@ -420,67 +408,63 @@ define <32 x i8> @test_ext_v32i8(<32 x i8> %a0, <32 x i8> %a1) nounwind {
; SSE-LABEL: test_ext_v32i8:
; SSE: # %bb.0:
; SSE-NEXT: movdqa %xmm0, %xmm4
-; SSE-NEXT: pand %xmm2, %xmm4
-; SSE-NEXT: pxor %xmm2, %xmm0
-; SSE-NEXT: psrlw $1, %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE-NEXT: pxor %xmm2, %xmm4
+; SSE-NEXT: movdqa {{.*#+}} xmm5 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; SSE-NEXT: movdqa %xmm4, %xmm6
+; SSE-NEXT: pandn %xmm5, %xmm6
+; SSE-NEXT: pcmpeqd %xmm7, %xmm7
+; SSE-NEXT: pavgb %xmm7, %xmm4
+; SSE-NEXT: pxor %xmm6, %xmm4
; SSE-NEXT: pand %xmm2, %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm5 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; SSE-NEXT: pxor %xmm5, %xmm0
; SSE-NEXT: paddb %xmm4, %xmm0
-; SSE-NEXT: psubb %xmm5, %xmm0
-; SSE-NEXT: movdqa %xmm1, %xmm4
-; SSE-NEXT: pand %xmm3, %xmm4
-; SSE-NEXT: pxor %xmm3, %xmm1
-; SSE-NEXT: psrlw $1, %xmm1
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: pxor %xmm5, %xmm1
-; SSE-NEXT: paddb %xmm4, %xmm1
-; SSE-NEXT: psubb %xmm5, %xmm1
+; SSE-NEXT: movdqa %xmm1, %xmm2
+; SSE-NEXT: pxor %xmm3, %xmm2
+; SSE-NEXT: movdqa %xmm2, %xmm4
+; SSE-NEXT: pandn %xmm5, %xmm4
+; SSE-NEXT: pavgb %xmm7, %xmm2
+; SSE-NEXT: pxor %xmm4, %xmm2
+; SSE-NEXT: pand %xmm3, %xmm1
+; SSE-NEXT: paddb %xmm2, %xmm1
; SSE-NEXT: retq
;
; AVX1-LABEL: test_ext_v32i8:
; AVX1: # %bb.0:
; AVX1-NEXT: vxorps %ymm1, %ymm0, %ymm2
; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
-; AVX1-NEXT: vpsrlw $1, %xmm3, %xmm3
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX1-NEXT: vpandn %xmm4, %xmm3, %xmm5
+; AVX1-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm6
+; AVX1-NEXT: vpavgb %xmm6, %xmm3, %xmm3
; AVX1-NEXT: vpxor %xmm5, %xmm3, %xmm3
; AVX1-NEXT: vandps %ymm1, %ymm0, %ymm0
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpaddb %xmm1, %xmm3, %xmm1
-; AVX1-NEXT: vpsubb %xmm5, %xmm1, %xmm1
-; AVX1-NEXT: vpsrlw $1, %xmm2, %xmm2
-; AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpxor %xmm5, %xmm2, %xmm2
-; AVX1-NEXT: vpaddb %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vpsubb %xmm5, %xmm0, %xmm0
+; AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT: vpandn %xmm4, %xmm2, %xmm3
+; AVX1-NEXT: vpavgb %xmm6, %xmm2, %xmm2
+; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: test_ext_v32i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm2
-; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpsrlw $1, %ymm0, %ymm0
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm2
+; AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX2-NEXT: vpavgb %ymm3, %ymm2, %ymm3
+; AVX2-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT: vpxor %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_ext_v32i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpand %ymm1, %ymm0, %ymm2
-; AVX512-NEXT: vpxor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpsrlw $1, %ymm0, %ymm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT: vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & m32bcst)
-; AVX512-NEXT: vpaddb %ymm2, %ymm0, %ymm0
-; AVX512-NEXT: vpsubb %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpxor %ymm1, %ymm0, %ymm2
+; AVX512-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX512-NEXT: vpavgb %ymm3, %ymm2, %ymm3
+; AVX512-NEXT: vpternlogd {{.*#+}} ymm3 = ymm3 ^ (m32bcst & ~ymm2)
+; AVX512-NEXT: vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddb %ymm3, %ymm0, %ymm0
; AVX512-NEXT: retq
%x0 = sext <32 x i8> %a0 to <32 x i16>
%x1 = sext <32 x i8> %a1 to <32 x i16>
@@ -868,107 +852,100 @@ define <4 x i64> @test_ext_v4i64(<4 x i64> %a0, <4 x i64> %a1) nounwind {
define <64 x i8> @test_fixed_v64i8(<64 x i8> %a0, <64 x i8> %a1) nounwind {
; SSE-LABEL: test_fixed_v64i8:
; SSE: # %bb.0:
-; SSE-NEXT: movdqa %xmm0, %xmm9
-; SSE-NEXT: pand %xmm4, %xmm9
-; SSE-NEXT: pxor %xmm4, %xmm0
-; SSE-NEXT: psrlw $1, %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm8 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE-NEXT: pand %xmm8, %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm4 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; SSE-NEXT: pxor %xmm4, %xmm0
-; SSE-NEXT: paddb %xmm9, %xmm0
-; SSE-NEXT: psubb %xmm4, %xmm0
-; SSE-NEXT: movdqa %xmm1, %xmm9
-; SSE-NEXT: pand %xmm5, %xmm9
-; SSE-NEXT: pxor %xmm5, %xmm1
-; SSE-NEXT: psrlw $1, %xmm1
-; SSE-NEXT: pand %xmm8, %xmm1
-; SSE-NEXT: pxor %xmm4, %xmm1
-; SSE-NEXT: paddb %xmm9, %xmm1
-; SSE-NEXT: psubb %xmm4, %xmm1
-; SSE-NEXT: movdqa %xmm2, %xmm5
-; SSE-NEXT: pand %xmm6, %xmm5
-; SSE-NEXT: pxor %xmm6, %xmm2
-; SSE-NEXT: psrlw $1, %xmm2
-; SSE-NEXT: pand %xmm8, %xmm2
-; SSE-NEXT: pxor %xmm4, %xmm2
-; SSE-NEXT: paddb %xmm5, %xmm2
-; SSE-NEXT: psubb %xmm4, %xmm2
-; SSE-NEXT: movdqa %xmm3, %xmm5
-; SSE-NEXT: pand %xmm7, %xmm5
-; SSE-NEXT: pxor %xmm7, %xmm3
-; SSE-NEXT: psrlw $1, %xmm3
-; SSE-NEXT: pand %xmm8, %xmm3
-; SSE-NEXT: pxor %xmm4, %xmm3
-; SSE-NEXT: paddb %xmm5, %xmm3
-; SSE-NEXT: psubb %xmm4, %xmm3
+; SSE-NEXT: movdqa %xmm0, %xmm10
+; SSE-NEXT: pxor %xmm4, %xmm10
+; SSE-NEXT: movdqa {{.*#+}} xmm8 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; SSE-NEXT: movdqa %xmm10, %xmm11
+; SSE-NEXT: pandn %xmm8, %xmm11
+; SSE-NEXT: pcmpeqd %xmm9, %xmm9
+; SSE-NEXT: pavgb %xmm9, %xmm10
+; SSE-NEXT: pxor %xmm11, %xmm10
+; SSE-NEXT: pand %xmm4, %xmm0
+; SSE-NEXT: paddb %xmm10, %xmm0
+; SSE-NEXT: movdqa %xmm1, %xmm4
+; SSE-NEXT: pxor %xmm5, %xmm4
+; SSE-NEXT: movdqa %xmm4, %xmm10
+; SSE-NEXT: pandn %xmm8, %xmm10
+; SSE-NEXT: pavgb %xmm9, %xmm4
+; SSE-NEXT: pxor %xmm10, %xmm4
+; SSE-NEXT: pand %xmm5, %xmm1
+; SSE-NEXT: paddb %xmm4, %xmm1
+; SSE-NEXT: movdqa %xmm2, %xmm4
+; SSE-NEXT: pxor %xmm6, %xmm4
+; SSE-NEXT: movdqa %xmm4, %xmm5
+; SSE-NEXT: pandn %xmm8, %xmm5
+; SSE-NEXT: pavgb %xmm9, %xmm4
+; SSE-NEXT: pxor %xmm5, %xmm4
+; SSE-NEXT: pand %xmm6, %xmm2
+; SSE-NEXT: paddb %xmm4, %xmm2
+; SSE-NEXT: movdqa %xmm3, %xmm4
+; SSE-NEXT: pxor %xmm7, %xmm4
+; SSE-NEXT: movdqa %xmm4, %xmm5
+; SSE-NEXT: pandn %xmm8, %xmm5
+; SSE-NEXT: pavgb %xmm9, %xmm4
+; SSE-NEXT: pxor %xmm5, %xmm4
+; SSE-NEXT: pand %xmm7, %xmm3
+; SSE-NEXT: paddb %xmm4, %xmm3
; SSE-NEXT: retq
;
; AVX1-LABEL: test_fixed_v64i8:
; AVX1: # %bb.0:
; AVX1-NEXT: vxorps %ymm2, %ymm0, %ymm4
; AVX1-NEXT: vextractf128 $1, %ymm4, %xmm5
-; AVX1-NEXT: vpsrlw $1, %xmm5, %xmm5
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm5
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm7 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm6 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX1-NEXT: vpandn %xmm6, %xmm5, %xmm7
+; AVX1-NEXT: vpcmpeqd %xmm8, %xmm8, %xmm8
+; AVX1-NEXT: vpavgb %xmm5, %xmm8, %xmm5
; AVX1-NEXT: vpxor %xmm7, %xmm5, %xmm5
; AVX1-NEXT: vandps %ymm2, %ymm0, %ymm0
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT: vpaddb %xmm2, %xmm5, %xmm2
-; AVX1-NEXT: vpsubb %xmm7, %xmm2, %xmm2
-; AVX1-NEXT: vpsrlw $1, %xmm4, %xmm4
-; AVX1-NEXT: vpand %xmm6, %xmm4, %xmm4
-; AVX1-NEXT: vpxor %xmm7, %xmm4, %xmm4
-; AVX1-NEXT: vpaddb %xmm0, %xmm4, %xmm0
-; AVX1-NEXT: vpsubb %xmm7, %xmm0, %xmm0
+; AVX1-NEXT: vpaddb %xmm5, %xmm2, %xmm2
+; AVX1-NEXT: vpandn %xmm6, %xmm4, %xmm5
+; AVX1-NEXT: vpavgb %xmm4, %xmm8, %xmm4
+; AVX1-NEXT: vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpaddb %xmm4, %xmm0, %xmm0
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: vxorps %ymm3, %ymm1, %ymm2
; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT: vpsrlw $1, %xmm4, %xmm4
-; AVX1-NEXT: vpand %xmm6, %xmm4, %xmm4
-; AVX1-NEXT: vpxor %xmm7, %xmm4, %xmm4
+; AVX1-NEXT: vpandn %xmm6, %xmm4, %xmm5
+; AVX1-NEXT: vpavgb %xmm4, %xmm8, %xmm4
+; AVX1-NEXT: vpxor %xmm5, %xmm4, %xmm4
; AVX1-NEXT: vandps %ymm3, %ymm1, %ymm1
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT: vpaddb %xmm3, %xmm4, %xmm3
-; AVX1-NEXT: vpsubb %xmm7, %xmm3, %xmm3
-; AVX1-NEXT: vpsrlw $1, %xmm2, %xmm2
-; AVX1-NEXT: vpand %xmm6, %xmm2, %xmm2
-; AVX1-NEXT: vpxor %xmm7, %xmm2, %xmm2
-; AVX1-NEXT: vpaddb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT: vpsubb %xmm7, %xmm1, %xmm1
+; AVX1-NEXT: vpaddb %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpandn %xmm6, %xmm2, %xmm4
+; AVX1-NEXT: vpavgb %xmm2, %xmm8, %xmm2
+; AVX1-NEXT: vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT: vpaddb %xmm2, %xmm1, %xmm1
; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
; AVX1-NEXT: retq
;
; AVX2-LABEL: test_fixed_v64i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm4
-; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpsrlw $1, %ymm0, %ymm0
-; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm4
+; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm5 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX2-NEXT: vpandn %ymm5, %ymm4, %ymm6
+; AVX2-NEXT: vpcmpeqd %ymm7, %ymm7, %ymm7
+; AVX2-NEXT: vpavgb %ymm7, %ymm4, %ymm4
+; AVX2-NEXT: vpxor %ymm6, %ymm4, %ymm4
; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm5 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX2-NEXT: vpxor %ymm5, %ymm0, %ymm0
; AVX2-NEXT: vpaddb %ymm4, %ymm0, %ymm0
-; AVX2-NEXT: vpsubb %ymm5, %ymm0, %ymm0
-; AVX2-NEXT: vpand %ymm3, %ymm1, %ymm4
-; AVX2-NEXT: vpxor %ymm3, %ymm1, %ymm1
-; AVX2-NEXT: vpsrlw $1, %ymm1, %ymm1
-; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpxor %ymm5, %ymm1, %ymm1
-; AVX2-NEXT: vpaddb %ymm4, %ymm1, %ymm1
-; AVX2-NEXT: vpsubb %ymm5, %ymm1, %ymm1
+; AVX2-NEXT: vpxor %ymm3, %ymm1, %ymm2
+; AVX2-NEXT: vpandn %ymm5, %ymm2, %ymm4
+; AVX2-NEXT: vpavgb %ymm7, %ymm2, %ymm2
+; AVX2-NEXT: vpxor %ymm4, %ymm2, %ymm2
+; AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpaddb %ymm2, %ymm1, %ymm1
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_fixed_v64i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpandq %zmm1, %zmm0, %zmm2
-; AVX512-NEXT: vpxorq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpsrlw $1, %zmm0, %zmm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT: vpternlogd {{.*#+}} zmm0 = zmm1 ^ (zmm0 & m32bcst)
-; AVX512-NEXT: vpaddb %zmm2, %zmm0, %zmm0
-; AVX512-NEXT: vpsubb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxorq %zmm1, %zmm0, %zmm2
+; AVX512-NEXT: vpternlogd {{.*#+}} zmm3 = -1
+; AVX512-NEXT: vpavgb %zmm3, %zmm2, %zmm3
+; AVX512-NEXT: vpternlogd {{.*#+}} zmm3 = zmm3 ^ (m32bcst & ~zmm2)
+; AVX512-NEXT: vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddb %zmm3, %zmm0, %zmm0
; AVX512-NEXT: retq
%and = and <64 x i8> %a0, %a1
%xor = xor <64 x i8> %a0, %a1
@@ -980,107 +957,100 @@ define <64 x i8> @test_fixed_v64i8(<64 x i8> %a0, <64 x i8> %a1) nounwind {
define <64 x i8> @test_ext_v64i8(<64 x i8> %a0, <64 x i8> %a1) nounwind {
; SSE-LABEL: test_ext_v64i8:
; SSE: # %bb.0:
-; SSE-NEXT: movdqa %xmm0, %xmm9
-; SSE-NEXT: pand %xmm4, %xmm9
-; SSE-NEXT: pxor %xmm4, %xmm0
-; SSE-NEXT: psrlw $1, %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm8 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE-NEXT: pand %xmm8, %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm4 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; SSE-NEXT: pxor %xmm4, %xmm0
-; SSE-NEXT: paddb %xmm9, %xmm0
-; SSE-NEXT: psubb %xmm4, %xmm0
-; SSE-NEXT: movdqa %xmm1, %xmm9
-; SSE-NEXT: pand %xmm5, %xmm9
-; SSE-NEXT: pxor %xmm5, %xmm1
-; SSE-NEXT: psrlw $1, %xmm1
-; SSE-NEXT: pand %xmm8, %xmm1
-; SSE-NEXT: pxor %xmm4, %xmm1
-; SSE-NEXT: paddb %xmm9, %xmm1
-; SSE-NEXT: psubb %xmm4, %xmm1
-; SSE-NEXT: movdqa %xmm2, %xmm5
-; SSE-NEXT: pand %xmm6, %xmm5
-; SSE-NEXT: pxor %xmm6, %xmm2
-; SSE-NEXT: psrlw $1, %xmm2
-; SSE-NEXT: pand %xmm8, %xmm2
-; SSE-NEXT: pxor %xmm4, %xmm2
-; SSE-NEXT: paddb %xmm5, %xmm2
-; SSE-NEXT: psubb %xmm4, %xmm2
-; SSE-NEXT: movdqa %xmm3, %xmm5
-; SSE-NEXT: pand %xmm7, %xmm5
-; SSE-NEXT: pxor %xmm7, %xmm3
-; SSE-NEXT: psrlw $1, %xmm3
-; SSE-NEXT: pand %xmm8, %xmm3
-; SSE-NEXT: pxor %xmm4, %xmm3
-; SSE-NEXT: paddb %xmm5, %xmm3
-; SSE-NEXT: psubb %xmm4, %xmm3
+; SSE-NEXT: movdqa %xmm0, %xmm10
+; SSE-NEXT: pxor %xmm4, %xmm10
+; SSE-NEXT: movdqa {{.*#+}} xmm8 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; SSE-NEXT: movdqa %xmm10, %xmm11
+; SSE-NEXT: pandn %xmm8, %xmm11
+; SSE-NEXT: pcmpeqd %xmm9, %xmm9
+; SSE-NEXT: pavgb %xmm9, %xmm10
+; SSE-NEXT: pxor %xmm11, %xmm10
+; SSE-NEXT: pand %xmm4, %xmm0
+; SSE-NEXT: paddb %xmm10, %xmm0
+; SSE-NEXT: movdqa %xmm1, %xmm4
+; SSE-NEXT: pxor %xmm5, %xmm4
+; SSE-NEXT: movdqa %xmm4, %xmm10
+; SSE-NEXT: pandn %xmm8, %xmm10
+; SSE-NEXT: pavgb %xmm9, %xmm4
+; SSE-NEXT: pxor %xmm10, %xmm4
+; SSE-NEXT: pand %xmm5, %xmm1
+; SSE-NEXT: paddb %xmm4, %xmm1
+; SSE-NEXT: movdqa %xmm2, %xmm4
+; SSE-NEXT: pxor %xmm6, %xmm4
+; SSE-NEXT: movdqa %xmm4, %xmm5
+; SSE-NEXT: pandn %xmm8, %xmm5
+; SSE-NEXT: pavgb %xmm9, %xmm4
+; SSE-NEXT: pxor %xmm5, %xmm4
+; SSE-NEXT: pand %xmm6, %xmm2
+; SSE-NEXT: paddb %xmm4, %xmm2
+; SSE-NEXT: movdqa %xmm3, %xmm4
+; SSE-NEXT: pxor %xmm7, %xmm4
+; SSE-NEXT: movdqa %xmm4, %xmm5
+; SSE-NEXT: pandn %xmm8, %xmm5
+; SSE-NEXT: pavgb %xmm9, %xmm4
+; SSE-NEXT: pxor %xmm5, %xmm4
+; SSE-NEXT: pand %xmm7, %xmm3
+; SSE-NEXT: paddb %xmm4, %xmm3
; SSE-NEXT: retq
;
; AVX1-LABEL: test_ext_v64i8:
; AVX1: # %bb.0:
; AVX1-NEXT: vxorps %ymm2, %ymm0, %ymm4
; AVX1-NEXT: vextractf128 $1, %ymm4, %xmm5
-; AVX1-NEXT: vpsrlw $1, %xmm5, %xmm5
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm5
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm7 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm6 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX1-NEXT: vpandn %xmm6, %xmm5, %xmm7
+; AVX1-NEXT: vpcmpeqd %xmm8, %xmm8, %xmm8
+; AVX1-NEXT: vpavgb %xmm5, %xmm8, %xmm5
; AVX1-NEXT: vpxor %xmm7, %xmm5, %xmm5
; AVX1-NEXT: vandps %ymm2, %ymm0, %ymm0
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT: vpaddb %xmm2, %xmm5, %xmm2
-; AVX1-NEXT: vpsubb %xmm7, %xmm2, %xmm2
-; AVX1-NEXT: vpsrlw $1, %xmm4, %xmm4
-; AVX1-NEXT: vpand %xmm6, %xmm4, %xmm4
-; AVX1-NEXT: vpxor %xmm7, %xmm4, %xmm4
-; AVX1-NEXT: vpaddb %xmm0, %xmm4, %xmm0
-; AVX1-NEXT: vpsubb %xmm7, %xmm0, %xmm0
+; AVX1-NEXT: vpaddb %xmm5, %xmm2, %xmm2
+; AVX1-NEXT: vpandn %xmm6, %xmm4, %xmm5
+; AVX1-NEXT: vpavgb %xmm4, %xmm8, %xmm4
+; AVX1-NEXT: vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpaddb %xmm4, %xmm0, %xmm0
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: vxorps %ymm3, %ymm1, %ymm2
; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT: vpsrlw $1, %xmm4, %xmm4
-; AVX1-NEXT: vpand %xmm6, %xmm4, %xmm4
-; AVX1-NEXT: vpxor %xmm7, %xmm4, %xmm4
+; AVX1-NEXT: vpandn %xmm6, %xmm4, %xmm5
+; AVX1-NEXT: vpavgb %xmm4, %xmm8, %xmm4
+; AVX1-NEXT: vpxor %xmm5, %xmm4, %xmm4
; AVX1-NEXT: vandps %ymm3, %ymm1, %ymm1
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT: vpaddb %xmm3, %xmm4, %xmm3
-; AVX1-NEXT: vpsubb %xmm7, %xmm3, %xmm3
-; AVX1-NEXT: vpsrlw $1, %xmm2, %xmm2
-; AVX1-NEXT: vpand %xmm6, %xmm2, %xmm2
-; AVX1-NEXT: vpxor %xmm7, %xmm2, %xmm2
-; AVX1-NEXT: vpaddb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT: vpsubb %xmm7, %xmm1, %xmm1
+; AVX1-NEXT: vpaddb %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpandn %xmm6, %xmm2, %xmm4
+; AVX1-NEXT: vpavgb %xmm2, %xmm8, %xmm2
+; AVX1-NEXT: vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT: vpaddb %xmm2, %xmm1, %xmm1
; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
; AVX1-NEXT: retq
;
; AVX2-LABEL: test_ext_v64i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm4
-; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpsrlw $1, %ymm0, %ymm0
-; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm4
+; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm5 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX2-NEXT: vpandn %ymm5, %ymm4, %ymm6
+; AVX2-NEXT: vpcmpeqd %ymm7, %ymm7, %ymm7
+; AVX2-NEXT: vpavgb %ymm7, %ymm4, %ymm4
+; AVX2-NEXT: vpxor %ymm6, %ymm4, %ymm4
; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm5 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX2-NEXT: vpxor %ymm5, %ymm0, %ymm0
; AVX2-NEXT: vpaddb %ymm4, %ymm0, %ymm0
-; AVX2-NEXT: vpsubb %ymm5, %ymm0, %ymm0
-; AVX2-NEXT: vpand %ymm3, %ymm1, %ymm4
-; AVX2-NEXT: vpxor %ymm3, %ymm1, %ymm1
-; AVX2-NEXT: vpsrlw $1, %ymm1, %ymm1
-; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpxor %ymm5, %ymm1, %ymm1
-; AVX2-NEXT: vpaddb %ymm4, %ymm1, %ymm1
-; AVX2-NEXT: vpsubb %ymm5, %ymm1, %ymm1
+; AVX2-NEXT: vpxor %ymm3, %ymm1, %ymm2
+; AVX2-NEXT: vpandn %ymm5, %ymm2, %ymm4
+; AVX2-NEXT: vpavgb %ymm7, %ymm2, %ymm2
+; AVX2-NEXT: vpxor %ymm4, %ymm2, %ymm2
+; AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpaddb %ymm2, %ymm1, %ymm1
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_ext_v64i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpandq %zmm1, %zmm0, %zmm2
-; AVX512-NEXT: vpxorq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpsrlw $1, %zmm0, %zmm0
-; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT: vpternlogd {{.*#+}} zmm0 = zmm1 ^ (zmm0 & m32bcst)
-; AVX512-NEXT: vpaddb %zmm2, %zmm0, %zmm0
-; AVX512-NEXT: vpsubb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxorq %zmm1, %zmm0, %zmm2
+; AVX512-NEXT: vpternlogd {{.*#+}} zmm3 = -1
+; AVX512-NEXT: vpavgb %zmm3, %zmm2, %zmm3
+; AVX512-NEXT: vpternlogd {{.*#+}} zmm3 = zmm3 ^ (m32bcst & ~zmm2)
+; AVX512-NEXT: vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddb %zmm3, %zmm0, %zmm0
; AVX512-NEXT: retq
%x0 = sext <64 x i8> %a0 to <64 x i16>
%x1 = sext <64 x i8> %a1 to <64 x i16>
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
index 6d50e5cb3b9c0..609d662e9800b 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
@@ -2048,30 +2048,19 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind {
define <16 x i8> @splatconstant_shift_v16i8_by_one(<16 x i8> %a) nounwind {
; SSE-LABEL: splatconstant_shift_v16i8_by_one:
; SSE: # %bb.0:
-; SSE-NEXT: psrlw $1, %xmm0
-; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; SSE-NEXT: pcmpeqd %xmm1, %xmm1
+; SSE-NEXT: pavgb %xmm0, %xmm1
+; SSE-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE-NEXT: pxor %xmm1, %xmm0
-; SSE-NEXT: psubb %xmm1, %xmm0
; SSE-NEXT: retq
;
-; AVX1-LABEL: splatconstant_shift_v16i8_by_one:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: retq
-;
-; AVX2-LABEL: splatconstant_shift_v16i8_by_one:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: retq
+; AVX-LABEL: splatconstant_shift_v16i8_by_one:
+; AVX: # %bb.0:
+; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpavgb %xmm1, %xmm0, %xmm1
+; AVX-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm0, %xmm1, %xmm0
+; AVX-NEXT: retq
;
; XOP-LABEL: splatconstant_shift_v16i8_by_one:
; XOP: # %bb.0:
@@ -2081,36 +2070,25 @@ define <16 x i8> @splatconstant_shift_v16i8_by_one(<16 x i8> %a) nounwind {
;
; AVX512-LABEL: splatconstant_shift_v16i8_by_one:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpavgb %xmm1, %xmm0, %xmm1
+; AVX512-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm0, %xmm1, %xmm0
; AVX512-NEXT: retq
;
-; AVX512DQVL-LABEL: splatconstant_shift_v16i8_by_one:
-; AVX512DQVL: # %bb.0:
-; AVX512DQVL-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512DQVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
-; AVX512DQVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT: retq
-;
-; AVX512BWVL-LABEL: splatconstant_shift_v16i8_by_one:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512BWVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
-; AVX512BWVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: retq
+; AVX512VL-LABEL: splatconstant_shift_v16i8_by_one:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT: vpavgb %xmm1, %xmm0, %xmm1
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (m32bcst & ~xmm0)
+; AVX512VL-NEXT: retq
;
; X86-SSE-LABEL: splatconstant_shift_v16i8_by_one:
; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: psrlw $1, %xmm0
-; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE-NEXT: movdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; X86-SSE-NEXT: pcmpeqd %xmm1, %xmm1
+; X86-SSE-NEXT: pavgb %xmm0, %xmm1
+; X86-SSE-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
; X86-SSE-NEXT: pxor %xmm1, %xmm0
-; X86-SSE-NEXT: psubb %xmm1, %xmm0
; X86-SSE-NEXT: retl
%shift = ashr <16 x i8> %a, splat (i8 1)
ret <16 x i8> %shift
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
index e9454657d2599..63554180959a9 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
@@ -2339,26 +2339,20 @@ define <32 x i8> @splatconstant_shift_v32i8_by_one(<32 x i8> %a) nounwind {
; AVX1-LABEL: splatconstant_shift_v32i8_by_one:
; AVX1: # %bb.0:
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpsrlw $1, %xmm1, %xmm1
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX1-NEXT: vpxor %xmm3, %xmm1, %xmm1
-; AVX1-NEXT: vpsubb %xmm3, %xmm1, %xmm1
-; AVX1-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpxor %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vpsubb %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpavgb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpavgb %xmm2, %xmm0, %xmm2
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT: vandnps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT: vxorps %ymm0, %ymm1, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: splatconstant_shift_v32i8_by_one:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpsrlw $1, %ymm0, %ymm0
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; AVX2-NEXT: vpavgb %ymm1, %ymm0, %ymm1
+; AVX2-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT: vpxor %ymm0, %ymm1, %ymm0
; AVX2-NEXT: retq
;
; XOPAVX1-LABEL: splatconstant_shift_v32i8_by_one:
@@ -2372,61 +2366,44 @@ define <32 x i8> @splatconstant_shift_v32i8_by_one(<32 x i8> %a) nounwind {
;
; XOPAVX2-LABEL: splatconstant_shift_v32i8_by_one:
; XOPAVX2: # %bb.0:
-; XOPAVX2-NEXT: vpsrlw $1, %ymm0, %ymm0
-; XOPAVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; XOPAVX2-NEXT: vpbroadcastb {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; XOPAVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
-; XOPAVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0
+; XOPAVX2-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; XOPAVX2-NEXT: vpavgb %ymm1, %ymm0, %ymm1
+; XOPAVX2-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; XOPAVX2-NEXT: vpxor %ymm0, %ymm1, %ymm0
; XOPAVX2-NEXT: retq
;
; AVX512-LABEL: splatconstant_shift_v32i8_by_one:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsrlw $1, %ymm0, %ymm0
-; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT: vpbroadcastb {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT: vpxor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpsubb %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; AVX512-NEXT: vpavgb %ymm1, %ymm0, %ymm1
+; AVX512-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT: vpxor %ymm0, %ymm1, %ymm0
; AVX512-NEXT: retq
;
-; AVX512DQVL-LABEL: splatconstant_shift_v32i8_by_one:
-; AVX512DQVL: # %bb.0:
-; AVX512DQVL-NEXT: vpsrlw $1, %ymm0, %ymm0
-; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512DQVL-NEXT: vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & m32bcst)
-; AVX512DQVL-NEXT: vpsubb %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT: retq
-;
-; AVX512BWVL-LABEL: splatconstant_shift_v32i8_by_one:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpsrlw $1, %ymm0, %ymm0
-; AVX512BWVL-NEXT: vpbroadcastb {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512BWVL-NEXT: vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & m32bcst)
-; AVX512BWVL-NEXT: vpsubb %ymm1, %ymm0, %ymm0
-; AVX512BWVL-NEXT: retq
+; AVX512VL-LABEL: splatconstant_shift_v32i8_by_one:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; AVX512VL-NEXT: vpavgb %ymm1, %ymm0, %ymm1
+; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm0 = ymm1 ^ (m32bcst & ~ymm0)
+; AVX512VL-NEXT: retq
;
; X86-AVX1-LABEL: splatconstant_shift_v32i8_by_one:
; X86-AVX1: # %bb.0:
; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X86-AVX1-NEXT: vpsrlw $1, %xmm1, %xmm1
-; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X86-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
-; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; X86-AVX1-NEXT: vpxor %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsubb %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsrlw $1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpxor %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpsubb %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpavgb %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpavgb %xmm2, %xmm0, %xmm2
+; X86-AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; X86-AVX1-NEXT: vandnps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX1-NEXT: vxorps %ymm0, %ymm1, %ymm0
; X86-AVX1-NEXT: retl
;
; X86-AVX2-LABEL: splatconstant_shift_v32i8_by_one:
; X86-AVX2: # %bb.0:
-; X86-AVX2-NEXT: vpsrlw $1, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpbroadcastb {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; X86-AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpavgb %ymm1, %ymm0, %ymm1
+; X86-AVX2-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpxor %ymm0, %ymm1, %ymm0
; X86-AVX2-NEXT: retl
%shift = ashr <32 x i8> %a, splat (i8 1)
ret <32 x i8> %shift
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
index 888498150df85..6cfb2bd08dc0b 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll
@@ -531,25 +531,18 @@ define <64 x i8> @splatconstant_shift_v64i8_by_one(<64 x i8> %a) nounwind {
; AVX512DQ-LABEL: splatconstant_shift_v64i8_by_one:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512DQ-NEXT: vpsrlw $1, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpbroadcastb {{.*#+}} ymm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512DQ-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpbroadcastb {{.*#+}} ymm3 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512DQ-NEXT: vpxor %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpsubb %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpsrlw $1, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpand %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpxor %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpsubb %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX512DQ-NEXT: vpavgb %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT: vpavgb %ymm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512DQ-NEXT: vpternlogd {{.*#+}} zmm0 = zmm1 ^ (m32bcst & ~zmm0)
; AVX512DQ-NEXT: retq
;
; AVX512BW-LABEL: splatconstant_shift_v64i8_by_one:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpsrlw $1, %zmm0, %zmm0
-; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 = zmm1 ^ (zmm0 & m32bcst)
-; AVX512BW-NEXT: vpsubb %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1
+; AVX512BW-NEXT: vpavgb %zmm1, %zmm0, %zmm1
+; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 = zmm1 ^ (m32bcst & ~zmm0)
; AVX512BW-NEXT: retq
%shift = ashr <64 x i8> %a, splat (i8 1)
ret <64 x i8> %shift
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
index 9da157bb917d7..811cab366aef9 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
@@ -2357,30 +2357,19 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
define <8 x i8> @splatconstant_shift_v8i8_by_one(<8 x i8> %a) nounwind {
; SSE-LABEL: splatconstant_shift_v8i8_by_one:
; SSE: # %bb.0:
-; SSE-NEXT: psrlw $1, %xmm0
-; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; SSE-NEXT: pcmpeqd %xmm1, %xmm1
+; SSE-NEXT: pavgb %xmm0, %xmm1
+; SSE-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE-NEXT: pxor %xmm1, %xmm0
-; SSE-NEXT: psubb %xmm1, %xmm0
; SSE-NEXT: retq
;
-; AVX1-LABEL: splatconstant_shift_v8i8_by_one:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: retq
-;
-; AVX2-LABEL: splatconstant_shift_v8i8_by_one:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: retq
+; AVX-LABEL: splatconstant_shift_v8i8_by_one:
+; AVX: # %bb.0:
+; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpavgb %xmm1, %xmm0, %xmm1
+; AVX-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm0, %xmm1, %xmm0
+; AVX-NEXT: retq
;
; XOP-LABEL: splatconstant_shift_v8i8_by_one:
; XOP: # %bb.0:
@@ -2390,36 +2379,25 @@ define <8 x i8> @splatconstant_shift_v8i8_by_one(<8 x i8> %a) nounwind {
;
; AVX512-LABEL: splatconstant_shift_v8i8_by_one:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpavgb %xmm1, %xmm0, %xmm1
+; AVX512-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm0, %xmm1, %xmm0
; AVX512-NEXT: retq
;
-; AVX512DQVL-LABEL: splatconstant_shift_v8i8_by_one:
-; AVX512DQVL: # %bb.0:
-; AVX512DQVL-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512DQVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
-; AVX512DQVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT: retq
-;
-; AVX512BWVL-LABEL: splatconstant_shift_v8i8_by_one:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512BWVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
-; AVX512BWVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: retq
+; AVX512VL-LABEL: splatconstant_shift_v8i8_by_one:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT: vpavgb %xmm1, %xmm0, %xmm1
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (m32bcst & ~xmm0)
+; AVX512VL-NEXT: retq
;
; X86-SSE-LABEL: splatconstant_shift_v8i8_by_one:
; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: psrlw $1, %xmm0
-; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE-NEXT: movdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; X86-SSE-NEXT: pcmpeqd %xmm1, %xmm1
+; X86-SSE-NEXT: pavgb %xmm0, %xmm1
+; X86-SSE-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
; X86-SSE-NEXT: pxor %xmm1, %xmm0
-; X86-SSE-NEXT: psubb %xmm1, %xmm0
; X86-SSE-NEXT: retl
%shift = ashr <8 x i8> %a, splat (i8 1)
ret <8 x i8> %shift
@@ -2498,30 +2476,19 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
define <4 x i8> @splatconstant_shift_v4i8_by_one(<4 x i8> %a) nounwind {
; SSE-LABEL: splatconstant_shift_v4i8_by_one:
; SSE: # %bb.0:
-; SSE-NEXT: psrlw $1, %xmm0
-; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; SSE-NEXT: pcmpeqd %xmm1, %xmm1
+; SSE-NEXT: pavgb %xmm0, %xmm1
+; SSE-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE-NEXT: pxor %xmm1, %xmm0
-; SSE-NEXT: psubb %xmm1, %xmm0
; SSE-NEXT: retq
;
-; AVX1-LABEL: splatconstant_shift_v4i8_by_one:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: retq
-;
-; AVX2-LABEL: splatconstant_shift_v4i8_by_one:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: retq
+; AVX-LABEL: splatconstant_shift_v4i8_by_one:
+; AVX: # %bb.0:
+; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpavgb %xmm1, %xmm0, %xmm1
+; AVX-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm0, %xmm1, %xmm0
+; AVX-NEXT: retq
;
; XOP-LABEL: splatconstant_shift_v4i8_by_one:
; XOP: # %bb.0:
@@ -2531,36 +2498,25 @@ define <4 x i8> @splatconstant_shift_v4i8_by_one(<4 x i8> %a) nounwind {
;
; AVX512-LABEL: splatconstant_shift_v4i8_by_one:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpavgb %xmm1, %xmm0, %xmm1
+; AVX512-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm0, %xmm1, %xmm0
; AVX512-NEXT: retq
;
-; AVX512DQVL-LABEL: splatconstant_shift_v4i8_by_one:
-; AVX512DQVL: # %bb.0:
-; AVX512DQVL-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512DQVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
-; AVX512DQVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT: retq
-;
-; AVX512BWVL-LABEL: splatconstant_shift_v4i8_by_one:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512BWVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
-; AVX512BWVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: retq
+; AVX512VL-LABEL: splatconstant_shift_v4i8_by_one:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT: vpavgb %xmm1, %xmm0, %xmm1
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (m32bcst & ~xmm0)
+; AVX512VL-NEXT: retq
;
; X86-SSE-LABEL: splatconstant_shift_v4i8_by_one:
; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: psrlw $1, %xmm0
-; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE-NEXT: movdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; X86-SSE-NEXT: pcmpeqd %xmm1, %xmm1
+; X86-SSE-NEXT: pavgb %xmm0, %xmm1
+; X86-SSE-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
; X86-SSE-NEXT: pxor %xmm1, %xmm0
-; X86-SSE-NEXT: psubb %xmm1, %xmm0
; X86-SSE-NEXT: retl
%shift = ashr <4 x i8> %a, splat (i8 1)
ret <4 x i8> %shift
@@ -2639,30 +2595,19 @@ define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind {
define <2 x i8> @splatconstant_shift_v2i8_by_one(<2 x i8> %a) nounwind {
; SSE-LABEL: splatconstant_shift_v2i8_by_one:
; SSE: # %bb.0:
-; SSE-NEXT: psrlw $1, %xmm0
-; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; SSE-NEXT: pcmpeqd %xmm1, %xmm1
+; SSE-NEXT: pavgb %xmm0, %xmm1
+; SSE-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE-NEXT: pxor %xmm1, %xmm0
-; SSE-NEXT: psubb %xmm1, %xmm0
; SSE-NEXT: retq
;
-; AVX1-LABEL: splatconstant_shift_v2i8_by_one:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: retq
-;
-; AVX2-LABEL: splatconstant_shift_v2i8_by_one:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: retq
+; AVX-LABEL: splatconstant_shift_v2i8_by_one:
+; AVX: # %bb.0:
+; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpavgb %xmm1, %xmm0, %xmm1
+; AVX-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm0, %xmm1, %xmm0
+; AVX-NEXT: retq
;
; XOP-LABEL: splatconstant_shift_v2i8_by_one:
; XOP: # %bb.0:
@@ -2672,36 +2617,25 @@ define <2 x i8> @splatconstant_shift_v2i8_by_one(<2 x i8> %a) nounwind {
;
; AVX512-LABEL: splatconstant_shift_v2i8_by_one:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpavgb %xmm1, %xmm0, %xmm1
+; AVX512-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm0, %xmm1, %xmm0
; AVX512-NEXT: retq
;
-; AVX512DQVL-LABEL: splatconstant_shift_v2i8_by_one:
-; AVX512DQVL: # %bb.0:
-; AVX512DQVL-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512DQVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
-; AVX512DQVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT: retq
-;
-; AVX512BWVL-LABEL: splatconstant_shift_v2i8_by_one:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpsrlw $1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; AVX512BWVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & m32bcst)
-; AVX512BWVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: retq
+; AVX512VL-LABEL: splatconstant_shift_v2i8_by_one:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT: vpavgb %xmm1, %xmm0, %xmm1
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (m32bcst & ~xmm0)
+; AVX512VL-NEXT: retq
;
; X86-SSE-LABEL: splatconstant_shift_v2i8_by_one:
; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: psrlw $1, %xmm0
-; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE-NEXT: movdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; X86-SSE-NEXT: pcmpeqd %xmm1, %xmm1
+; X86-SSE-NEXT: pavgb %xmm0, %xmm1
+; X86-SSE-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
; X86-SSE-NEXT: pxor %xmm1, %xmm0
-; X86-SSE-NEXT: psubb %xmm1, %xmm0
; X86-SSE-NEXT: retl
%shift = ashr <2 x i8> %a, splat (i8 1)
ret <2 x i8> %shift
diff --git a/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll b/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll
index fb46f6205ab4e..2ae1bf7a5ff84 100644
--- a/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll
+++ b/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll
@@ -440,57 +440,41 @@ define <16 x i8> @test_128_i8_x_16_28_mask_ashr_4(<16 x i8> %a0) {
define <16 x i8> @test_128_i8_x_16_224_mask_ashr_1(<16 x i8> %a0) {
; X86-SSE2-LABEL: test_128_i8_x_16_224_mask_ashr_1:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: psrlw $1, %xmm0
-; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; X86-SSE2-NEXT: pxor %xmm1, %xmm0
-; X86-SSE2-NEXT: psubb %xmm1, %xmm0
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224]
+; X86-SSE2-NEXT: pand %xmm0, %xmm1
+; X86-SSE2-NEXT: pcmpeqd %xmm2, %xmm2
+; X86-SSE2-NEXT: pavgb %xmm1, %xmm2
+; X86-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT: pxor %xmm2, %xmm0
; X86-SSE2-NEXT: retl
;
-; X86-AVX1-LABEL: test_128_i8_x_16_224_mask_ashr_1:
-; X86-AVX1: # %bb.0:
-; X86-AVX1-NEXT: vpsrlw $1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; X86-AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: retl
-;
-; X86-AVX2-LABEL: test_128_i8_x_16_224_mask_ashr_1:
-; X86-AVX2: # %bb.0:
-; X86-AVX2-NEXT: vpsrlw $1, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; X86-AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT: retl
+; X86-AVX-LABEL: test_128_i8_x_16_224_mask_ashr_1:
+; X86-AVX: # %bb.0:
+; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1
+; X86-AVX-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; X86-AVX-NEXT: vpavgb %xmm2, %xmm1, %xmm1
+; X86-AVX-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX-NEXT: vpxor %xmm0, %xmm1, %xmm0
+; X86-AVX-NEXT: retl
;
; X64-SSE2-LABEL: test_128_i8_x_16_224_mask_ashr_1:
; X64-SSE2: # %bb.0:
-; X64-SSE2-NEXT: psrlw $1, %xmm0
-; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; X64-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; X64-SSE2-NEXT: pxor %xmm1, %xmm0
-; X64-SSE2-NEXT: psubb %xmm1, %xmm0
+; X64-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224]
+; X64-SSE2-NEXT: pand %xmm0, %xmm1
+; X64-SSE2-NEXT: pcmpeqd %xmm2, %xmm2
+; X64-SSE2-NEXT: pavgb %xmm1, %xmm2
+; X64-SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT: pxor %xmm2, %xmm0
; X64-SSE2-NEXT: retq
;
-; X64-AVX1-LABEL: test_128_i8_x_16_224_mask_ashr_1:
-; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vpsrlw $1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; X64-AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: retq
-;
-; X64-AVX2-LABEL: test_128_i8_x_16_224_mask_ashr_1:
-; X64-AVX2: # %bb.0:
-; X64-AVX2-NEXT: vpsrlw $1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; X64-AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: retq
+; X64-AVX-LABEL: test_128_i8_x_16_224_mask_ashr_1:
+; X64-AVX: # %bb.0:
+; X64-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; X64-AVX-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; X64-AVX-NEXT: vpavgb %xmm2, %xmm1, %xmm1
+; X64-AVX-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT: vpxor %xmm0, %xmm1, %xmm0
+; X64-AVX-NEXT: retq
%t0 = and <16 x i8> %a0, <i8 224, i8 224, i8 224, i8 224, i8 224, i8 224, i8 224, i8 224, i8 224, i8 224, i8 224, i8 224, i8 224, i8 224, i8 224, i8 224>
%t1 = ashr <16 x i8> %t0, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
ret <16 x i8> %t1
More information about the llvm-commits
mailing list