[llvm] [X86] Improve llvm.get.active.lane.mask codegen (PR #225832)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 9 01:27:33 PDT 2026
https://github.com/houngkoungting updated https://github.com/llvm/llvm-project/pull/225832
>From ee9343085718f3be84634632716b7306d3988c03 Mon Sep 17 00:00:00 2001
From: william <we3223 at gmail.com>
Date: Thu, 24 Sep 2026 00:07:36 +0800
Subject: [PATCH 1/5] [CodeGen] Clean up scalar bitmask type calculation in
TargetLowering
---
llvm/include/llvm/CodeGen/TargetLowering.h | 18 +
.../SelectionDAG/SelectionDAGBuilder.cpp | 8 +
.../CodeGen/SelectionDAG/TargetLowering.cpp | 45 +
llvm/lib/Target/X86/X86ISelLowering.cpp | 19 +
llvm/lib/Target/X86/X86ISelLowering.h | 3 +
llvm/test/CodeGen/X86/active_lane_mask.ll | 3241 +++--------------
llvm/test/CodeGen/X86/pr47299.ll | 70 +-
7 files changed, 635 insertions(+), 2769 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 46521fa8c544a6..1dfd2b63ca2fbf 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -5759,6 +5759,24 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
/// \returns The expansion if successful, SDValue() otherwise
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const;
+ /// Expand get_active_lane_mask into a scalar bitmask sequence.
+ /// \param DAG The SelectionDAG.
+ /// \param VT The target vector boolean type.
+ /// \param Index The loop iteration index.
+ /// \param TripCount The loop trip count.
+ /// \param DL The source location.
+ /// \returns The expanded SDValue, or an empty SDValue if expansion is not
+ /// supported.
+ SDValue expandGetActiveLaneMask(SelectionDAG &DAG, EVT VT, SDValue Index,
+ SDValue TripCount, const SDLoc &DL) const;
+
+ /// Return true if the target prefers to expand get_active_lane_mask
+ /// into a scalar bitmask sequence.
+ /// \param VT The vector mask type.
+ virtual bool shouldExpandGetActiveLaneMaskUsingScalar(EVT VT) const {
+ return false;
+ }
+
/// Expand shift-by-parts.
/// \param N Node to expand
/// \param Lo lower-output-part after conversion
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index f7eec8c5088d48..b19dff3a87f422 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -8541,6 +8541,14 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
SDValue Index = getValue(I.getOperand(0));
SDValue TripCount = getValue(I.getOperand(1));
EVT ElementVT = Index.getValueType();
+ // Attempt to expand into a scalar bitmask if preferred by target.
+ if (TLI.shouldExpandGetActiveLaneMaskUsingScalar(CCVT)) {
+ if (SDValue Res =
+ TLI.expandGetActiveLaneMask(DAG, CCVT, Index, TripCount, sdl)) {
+ setValue(&I, Res);
+ return;
+ }
+ }
if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
setValue(&I, DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, sdl, CCVT, Index,
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 1eeab43bd480e1..f00c371af10097 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -8768,6 +8768,51 @@ static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
/*AllowUndefs=*/true, /*AllowTruncation=*/true);
}
+SDValue TargetLowering::expandGetActiveLaneMask(SelectionDAG &DAG, EVT VT,
+ SDValue Index,
+ SDValue TripCount,
+ const SDLoc &DL) const {
+ if (!VT.isFixedLengthVector())
+ return SDValue();
+
+ unsigned NumElts = VT.getVectorNumElements();
+
+ if (NumElts > 64)
+ return SDValue();
+
+ EVT IndexVT = Index.getValueType();
+ // Scalar integer type wide enough to hold all lane bits (at least i8).
+ unsigned MaskBits = llvm::PowerOf2Ceil(std::max(8u, NumElts));
+ EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), MaskBits);
+
+ // Clamp remaining lanes: Diff = (TripCount <= Index) ? 0 : (TripCount -
+ // Index).
+ SDValue Sub = DAG.getNode(ISD::SUB, DL, IndexVT, TripCount, Index);
+ SDValue IsLe = DAG.getSetCC(
+ DL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), IndexVT),
+ TripCount, Index, ISD::SETULE);
+ SDValue Zero = DAG.getConstant(0, DL, IndexVT);
+ SDValue Diff = DAG.getSelect(DL, IndexVT, IsLe, Zero, Sub);
+ // Clamp to vector length: k = umin(Diff, NumElts).
+ SDValue MaxLanes = DAG.getConstant(NumElts, DL, IndexVT);
+ SDValue Clamped = DAG.getNode(ISD::UMIN, DL, IndexVT, Diff, MaxLanes);
+ // Build the scalar bitmask: (1 << k) - 1.
+ SDValue ShiftAmt = DAG.getZExtOrTrunc(Clamped, DL, MaskVT);
+ SDValue One = DAG.getConstant(1, DL, MaskVT);
+ SDValue Shl = DAG.getNode(ISD::SHL, DL, MaskVT, One, ShiftAmt);
+ SDValue ScalarMask = DAG.getNode(ISD::SUB, DL, MaskVT, Shl, One);
+ // Bitcast to a boolean vector matching the integer bit width.
+ EVT BoolVecVT =
+ EVT::getVectorVT(*DAG.getContext(), MVT::i1, MaskVT.getSizeInBits());
+ SDValue Bitcast = DAG.getNode(ISD::BITCAST, DL, BoolVecVT, ScalarMask);
+
+ if (BoolVecVT == VT)
+ return Bitcast;
+ // Extract the original subvector if padded (e.g. v4i1 from i8 -> v8i1).
+ return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Bitcast,
+ DAG.getVectorIdxConstant(0, DL));
+}
+
SDValue TargetLowering::expandFunnelShift(SDNode *Node,
SelectionDAG &DAG) const {
EVT VT = Node->getValueType(0);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7817c86155e5b0..30ff5844739bc8 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -32768,6 +32768,25 @@ X86TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
: AtomicExpansionKind::None;
}
+bool X86TargetLowering::shouldExpandGetActiveLaneMaskUsingScalar(EVT VT) const {
+ if (!VT.isFixedLengthVector())
+ return false;
+
+ unsigned NumElts = VT.getVectorNumElements();
+
+ // Only consider power-of-2 vector lengths up to 64.
+ if (NumElts > 64 || !isPowerOf2_32(NumElts))
+ return false;
+
+ // AVX-512 natively supports mask registers (kmov/bzhi) for >=16.
+ if (Subtarget.hasAVX512())
+ return NumElts >= 16;
+
+ // Without AVX-512, bitcasting to vXi1 requires costly unpacking (e.g. pinsrb)
+ // on smaller vectors. Only expand v64i1 to avoid its 80+ vector instructions.
+ return NumElts == 64;
+}
+
enum BitTestKind : unsigned {
UndefBit,
ConstantBit,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index ba04080573c421..caedf954ff068d 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -110,6 +110,9 @@ namespace llvm {
unsigned getJumpTableEncoding() const override;
bool useSoftFloat() const override;
+ /// Return true if X86 prefers to expand get_active_lane_mask into a scalar
+ /// bitmask sequence.
+ bool shouldExpandGetActiveLaneMaskUsingScalar(EVT VT) const override;
void markLibCallAttributes(MachineFunction *MF, unsigned CC,
ArgListTy &Args) const override;
diff --git a/llvm/test/CodeGen/X86/active_lane_mask.ll b/llvm/test/CodeGen/X86/active_lane_mask.ll
index be4264daffef03..74a37c8ba6b4a8 100644
--- a/llvm/test/CodeGen/X86/active_lane_mask.ll
+++ b/llvm/test/CodeGen/X86/active_lane_mask.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=SSE2
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE42
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE,SSE42
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=AVX2
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512
@@ -272,34 +272,34 @@ define <16 x i1> @lane_mask_v16i1_i32(i32 %index, i32 %TC) {
; SSE42-LABEL: lane_mask_v16i1_i32:
; SSE42: # %bb.0:
; SSE42-NEXT: movd %edi, %xmm0
-; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [4294967295,4294967294,4294967293,4294967292]
-; SSE42-NEXT: pminud %xmm2, %xmm0
+; SSE42-NEXT: pminud %xmm1, %xmm0
; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [0,1,2,3]
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [4294967291,4294967290,4294967289,4294967288]
-; SSE42-NEXT: pminud %xmm2, %xmm1
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [4,5,6,7]
+; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [4294967291,4294967290,4294967289,4294967288]
+; SSE42-NEXT: pminud %xmm1, %xmm2
+; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [4,5,6,7]
; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [4294967287,4294967286,4294967285,4294967284]
-; SSE42-NEXT: pminud %xmm2, %xmm3
+; SSE42-NEXT: pminud %xmm1, %xmm3
; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [8,9,10,11]
; SSE42-NEXT: movd %esi, %xmm4
-; SSE42-NEXT: pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE42-NEXT: pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; SSE42-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [12,13,14,15]
-; SSE42-NEXT: movdqa %xmm2, %xmm5
+; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [12,13,14,15]
+; SSE42-NEXT: movdqa %xmm1, %xmm5
; SSE42-NEXT: pmaxud %xmm4, %xmm5
-; SSE42-NEXT: pcmpeqd %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm3, %xmm2
-; SSE42-NEXT: pmaxud %xmm4, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm3, %xmm2
-; SSE42-NEXT: packssdw %xmm5, %xmm2
-; SSE42-NEXT: movdqa %xmm1, %xmm3
+; SSE42-NEXT: pcmpeqd %xmm1, %xmm5
+; SSE42-NEXT: movdqa %xmm3, %xmm1
+; SSE42-NEXT: pmaxud %xmm4, %xmm1
+; SSE42-NEXT: pcmpeqd %xmm3, %xmm1
+; SSE42-NEXT: packssdw %xmm5, %xmm1
+; SSE42-NEXT: movdqa %xmm2, %xmm3
; SSE42-NEXT: pmaxud %xmm4, %xmm3
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm3
+; SSE42-NEXT: pcmpeqd %xmm2, %xmm3
; SSE42-NEXT: pmaxud %xmm0, %xmm4
; SSE42-NEXT: pcmpeqd %xmm0, %xmm4
; SSE42-NEXT: packssdw %xmm3, %xmm4
-; SSE42-NEXT: packsswb %xmm2, %xmm4
+; SSE42-NEXT: packsswb %xmm1, %xmm4
; SSE42-NEXT: pcmpeqd %xmm0, %xmm0
; SSE42-NEXT: pxor %xmm4, %xmm0
; SSE42-NEXT: retq
@@ -329,13 +329,16 @@ define <16 x i1> @lane_mask_v16i1_i32(i32 %index, i32 %TC) {
;
; AVX512-LABEL: lane_mask_v16i1_i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastd %esi, %zmm0
-; AVX512-NEXT: vpbroadcastd %edi, %zmm1
-; AVX512-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512-NEXT: vpcmpltud %zmm0, %zmm1, %k0
+; AVX512-NEXT: xorl %eax, %eax
+; AVX512-NEXT: subl %edi, %esi
+; AVX512-NEXT: cmovbel %eax, %esi
+; AVX512-NEXT: cmpl $16, %esi
+; AVX512-NEXT: movl $16, %eax
+; AVX512-NEXT: cmovbl %esi, %eax
+; AVX512-NEXT: movl $-1, %ecx
+; AVX512-NEXT: bzhil %eax, %ecx, %eax
+; AVX512-NEXT: kmovd %eax, %k0
; AVX512-NEXT: vpmovm2b %k0, %xmm0
-; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %TC)
ret <16 x i1> %active.lane.mask
@@ -405,20 +408,9 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
; SSE2-NEXT: pcmpgtd %xmm5, %xmm6
; SSE2-NEXT: por %xmm4, %xmm6
; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [24,25,26,27]
-; SSE2-NEXT: paddd %xmm2, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm6
-; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm0, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: por %xmm4, %xmm7
-; SSE2-NEXT: pxor %xmm1, %xmm7
; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT: packssdw %xmm5, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [20,21,22,23]
+; SSE2-NEXT: pcmpgtd %xmm6, %xmm4
+; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [24,25,26,27]
; SSE2-NEXT: paddd %xmm2, %xmm5
; SSE2-NEXT: movdqa %xmm5, %xmm6
; SSE2-NEXT: pxor %xmm1, %xmm6
@@ -428,6 +420,17 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
; SSE2-NEXT: pxor %xmm1, %xmm7
; SSE2-NEXT: movdqa %xmm3, %xmm5
; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
+; SSE2-NEXT: packssdw %xmm4, %xmm5
+; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [20,21,22,23]
+; SSE2-NEXT: paddd %xmm2, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm6
+; SSE2-NEXT: pxor %xmm1, %xmm6
+; SSE2-NEXT: movdqa %xmm0, %xmm7
+; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
+; SSE2-NEXT: por %xmm4, %xmm7
+; SSE2-NEXT: pxor %xmm1, %xmm7
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: pcmpgtd %xmm7, %xmm4
; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [16,17,18,19]
; SSE2-NEXT: movdqa %xmm2, %xmm6
; SSE2-NEXT: pxor %xmm1, %xmm6
@@ -435,8 +438,8 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
; SSE2-NEXT: por %xmm2, %xmm0
; SSE2-NEXT: pxor %xmm1, %xmm0
; SSE2-NEXT: pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT: packssdw %xmm5, %xmm3
-; SSE2-NEXT: packsswb %xmm4, %xmm3
+; SSE2-NEXT: packssdw %xmm4, %xmm3
+; SSE2-NEXT: packsswb %xmm5, %xmm3
; SSE2-NEXT: pmovmskb %xmm3, %edx
; SSE2-NEXT: shll $16, %edx
; SSE2-NEXT: orl %ecx, %edx
@@ -456,28 +459,28 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
; SSE42-NEXT: movdqa %xmm2, %xmm3
; SSE42-NEXT: pmaxud %xmm0, %xmm3
; SSE42-NEXT: pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [4294967287,4294967286,4294967285,4294967284]
-; SSE42-NEXT: pminud %xmm1, %xmm4
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [8,9,10,11]
-; SSE42-NEXT: movdqa %xmm4, %xmm2
-; SSE42-NEXT: pmaxud %xmm0, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm4, %xmm2
-; SSE42-NEXT: packssdw %xmm3, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [4294967291,4294967290,4294967289,4294967288]
-; SSE42-NEXT: pminud %xmm1, %xmm3
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [4,5,6,7]
-; SSE42-NEXT: movdqa %xmm3, %xmm4
+; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [4294967287,4294967286,4294967285,4294967284]
+; SSE42-NEXT: pminud %xmm1, %xmm2
+; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [8,9,10,11]
+; SSE42-NEXT: movdqa %xmm2, %xmm4
; SSE42-NEXT: pmaxud %xmm0, %xmm4
+; SSE42-NEXT: pcmpeqd %xmm2, %xmm4
+; SSE42-NEXT: packssdw %xmm3, %xmm4
+; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [4294967291,4294967290,4294967289,4294967288]
+; SSE42-NEXT: pminud %xmm1, %xmm2
+; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [4,5,6,7]
+; SSE42-NEXT: movdqa %xmm2, %xmm3
+; SSE42-NEXT: pmaxud %xmm0, %xmm3
; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [4294967295,4294967294,4294967293,4294967292]
; SSE42-NEXT: pminud %xmm1, %xmm5
; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5 # [0,1,2,3]
-; SSE42-NEXT: pcmpeqd %xmm3, %xmm4
-; SSE42-NEXT: movdqa %xmm5, %xmm3
-; SSE42-NEXT: pmaxud %xmm0, %xmm3
-; SSE42-NEXT: pcmpeqd %xmm5, %xmm3
-; SSE42-NEXT: packssdw %xmm4, %xmm3
-; SSE42-NEXT: packsswb %xmm2, %xmm3
-; SSE42-NEXT: pmovmskb %xmm3, %ecx
+; SSE42-NEXT: pcmpeqd %xmm2, %xmm3
+; SSE42-NEXT: movdqa %xmm5, %xmm2
+; SSE42-NEXT: pmaxud %xmm0, %xmm2
+; SSE42-NEXT: pcmpeqd %xmm5, %xmm2
+; SSE42-NEXT: packssdw %xmm3, %xmm2
+; SSE42-NEXT: packsswb %xmm4, %xmm2
+; SSE42-NEXT: pmovmskb %xmm2, %ecx
; SSE42-NEXT: xorl $65535, %ecx # imm = 0xFFFF
; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [4294967267,4294967266,4294967265,4294967264]
; SSE42-NEXT: pminud %xmm1, %xmm2
@@ -546,15 +549,15 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
;
; AVX512-LABEL: lane_mask_v32i1_i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastd %esi, %zmm0
-; AVX512-NEXT: vpbroadcastd %edi, %zmm1
-; AVX512-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT: vpcmpltud %zmm0, %zmm2, %k0
-; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
-; AVX512-NEXT: vpcmpltud %zmm0, %zmm1, %k1
-; AVX512-NEXT: kunpckwd %k0, %k1, %k0
+; AVX512-NEXT: xorl %eax, %eax
+; AVX512-NEXT: subl %edi, %esi
+; AVX512-NEXT: cmovbel %eax, %esi
+; AVX512-NEXT: cmpl $32, %esi
+; AVX512-NEXT: movl $32, %eax
+; AVX512-NEXT: cmovbl %esi, %eax
+; AVX512-NEXT: movl $-1, %ecx
+; AVX512-NEXT: bzhil %eax, %ecx, %eax
+; AVX512-NEXT: kmovd %eax, %k0
; AVX512-NEXT: vpmovm2b %k0, %ymm0
; AVX512-NEXT: retq
%active.lane.mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i32(i32 %index, i32 %TC)
@@ -562,434 +565,69 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
}
define <64 x i1> @lane_mask_v64i1_i32(i32 %index, i32 %TC) {
-; SSE2-LABEL: lane_mask_v64i1_i32:
+; SSE-LABEL: lane_mask_v64i1_i32:
+; SSE: # %bb.0:
+; SSE-NEXT: xorl %eax, %eax
+; SSE-NEXT: subl %esi, %edx
+; SSE-NEXT: cmoval %edx, %eax
+; SSE-NEXT: cmpl $64, %eax
+; SSE-NEXT: movl $64, %ecx
+; SSE-NEXT: cmovbl %eax, %ecx
+; SSE-NEXT: movl $1, %edx
+; SSE-NEXT: # kill: def $cl killed $cl killed $ecx
+; SSE-NEXT: shlq %cl, %rdx
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: decq %rdx
+; SSE-NEXT: movq %rdx, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: lane_mask_v64i1_i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: subl %esi, %edx
+; AVX2-NEXT: cmovbel %eax, %edx
+; AVX2-NEXT: cmpl $64, %edx
+; AVX2-NEXT: movl $64, %ecx
+; AVX2-NEXT: cmovbl %edx, %ecx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq $-1, %rdx
+; AVX2-NEXT: bzhiq %rcx, %rdx, %rcx
+; AVX2-NEXT: movq %rcx, (%rdi)
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: lane_mask_v64i1_i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: xorl %eax, %eax
+; AVX512-NEXT: subl %edi, %esi
+; AVX512-NEXT: cmovbel %eax, %esi
+; AVX512-NEXT: cmpl $64, %esi
+; AVX512-NEXT: movl $64, %eax
+; AVX512-NEXT: cmovbl %esi, %eax
+; AVX512-NEXT: movq $-1, %rcx
+; AVX512-NEXT: bzhiq %rax, %rcx, %rax
+; AVX512-NEXT: kmovq %rax, %k0
+; AVX512-NEXT: vpmovm2b %k0, %zmm0
+; AVX512-NEXT: retq
+ %active.lane.mask = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 %index, i32 %TC)
+ ret <64 x i1> %active.lane.mask
+}
+
+;
+; i32 (zero index)
+;
+
+define <2 x i1> @lane_mask_v2i1_i32_zero(i32 %TC) {
+; SSE2-LABEL: lane_mask_v2i1_i32_zero:
; SSE2: # %bb.0:
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: movd %esi, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: pxor %xmm1, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [12,13,14,15]
-; SSE2-NEXT: paddd %xmm2, %xmm3
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pxor %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm0, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm4, %xmm5
-; SSE2-NEXT: por %xmm3, %xmm5
-; SSE2-NEXT: pxor %xmm1, %xmm5
-; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: pxor %xmm1, %xmm3
-; SSE2-NEXT: movdqa %xmm3, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [8,9,10,11]
-; SSE2-NEXT: paddd %xmm2, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pxor %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm0, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm5, %xmm7
-; SSE2-NEXT: por %xmm4, %xmm7
-; SSE2-NEXT: pxor %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT: packssdw %xmm6, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [4,5,6,7]
-; SSE2-NEXT: paddd %xmm2, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm0, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: por %xmm5, %xmm7
-; SSE2-NEXT: pxor %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [0,1,2,3]
-; SSE2-NEXT: paddd %xmm2, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm0, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: por %xmm6, %xmm8
-; SSE2-NEXT: pxor %xmm1, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT: packssdw %xmm5, %xmm6
-; SSE2-NEXT: packsswb %xmm4, %xmm6
-; SSE2-NEXT: pmovmskb %xmm6, %edx
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [28,29,30,31]
-; SSE2-NEXT: paddd %xmm2, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pxor %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm0, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT: por %xmm4, %xmm6
-; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [24,25,26,27]
-; SSE2-NEXT: paddd %xmm2, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm6
-; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm0, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: por %xmm4, %xmm7
-; SSE2-NEXT: pxor %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT: packssdw %xmm5, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [20,21,22,23]
-; SSE2-NEXT: paddd %xmm2, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm0, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: por %xmm5, %xmm7
-; SSE2-NEXT: pxor %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [16,17,18,19]
-; SSE2-NEXT: paddd %xmm2, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm0, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: por %xmm6, %xmm8
-; SSE2-NEXT: pxor %xmm1, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT: packssdw %xmm5, %xmm6
-; SSE2-NEXT: packsswb %xmm4, %xmm6
-; SSE2-NEXT: pmovmskb %xmm6, %ecx
-; SSE2-NEXT: shll $16, %ecx
-; SSE2-NEXT: orl %edx, %ecx
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [44,45,46,47]
-; SSE2-NEXT: paddd %xmm2, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pxor %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm0, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT: por %xmm4, %xmm6
-; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [40,41,42,43]
-; SSE2-NEXT: paddd %xmm2, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm6
-; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm0, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: por %xmm4, %xmm7
-; SSE2-NEXT: pxor %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT: packssdw %xmm5, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [36,37,38,39]
-; SSE2-NEXT: paddd %xmm2, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm0, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: por %xmm5, %xmm7
-; SSE2-NEXT: pxor %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [32,33,34,35]
-; SSE2-NEXT: paddd %xmm2, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm0, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: por %xmm6, %xmm8
-; SSE2-NEXT: pxor %xmm1, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT: packssdw %xmm5, %xmm6
-; SSE2-NEXT: packsswb %xmm4, %xmm6
-; SSE2-NEXT: pmovmskb %xmm6, %edx
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [60,61,62,63]
-; SSE2-NEXT: paddd %xmm2, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pxor %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm0, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT: por %xmm4, %xmm6
-; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [56,57,58,59]
-; SSE2-NEXT: paddd %xmm2, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm6
-; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm0, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: por %xmm4, %xmm7
-; SSE2-NEXT: pxor %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT: packssdw %xmm5, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [52,53,54,55]
-; SSE2-NEXT: paddd %xmm2, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm0, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: por %xmm5, %xmm7
-; SSE2-NEXT: pxor %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [48,49,50,51]
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: movdqa %xmm2, %xmm6
-; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm0
-; SSE2-NEXT: por %xmm2, %xmm0
-; SSE2-NEXT: pxor %xmm1, %xmm0
-; SSE2-NEXT: pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT: packssdw %xmm5, %xmm3
-; SSE2-NEXT: packsswb %xmm4, %xmm3
-; SSE2-NEXT: pmovmskb %xmm3, %esi
-; SSE2-NEXT: shll $16, %esi
-; SSE2-NEXT: orl %edx, %esi
-; SSE2-NEXT: shlq $32, %rsi
-; SSE2-NEXT: orq %rcx, %rsi
-; SSE2-NEXT: movq %rsi, (%rdi)
+; SSE2-NEXT: movd %edi, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE2-NEXT: retq
;
-; SSE42-LABEL: lane_mask_v64i1_i32:
+; SSE42-LABEL: lane_mask_v2i1_i32_zero:
; SSE42: # %bb.0:
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: movd %edx, %xmm1
-; SSE42-NEXT: movd %esi, %xmm0
-; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [4294967283,4294967282,4294967281,4294967280]
-; SSE42-NEXT: pminud %xmm0, %xmm2
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [12,13,14,15]
-; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE42-NEXT: movdqa %xmm2, %xmm3
-; SSE42-NEXT: pmaxud %xmm1, %xmm3
-; SSE42-NEXT: pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [4294967287,4294967286,4294967285,4294967284]
-; SSE42-NEXT: pminud %xmm0, %xmm4
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [8,9,10,11]
-; SSE42-NEXT: movdqa %xmm4, %xmm2
-; SSE42-NEXT: pmaxud %xmm1, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm4, %xmm2
-; SSE42-NEXT: packssdw %xmm3, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [4294967291,4294967290,4294967289,4294967288]
-; SSE42-NEXT: pminud %xmm0, %xmm3
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [4,5,6,7]
-; SSE42-NEXT: movdqa %xmm3, %xmm4
-; SSE42-NEXT: pmaxud %xmm1, %xmm4
-; SSE42-NEXT: pcmpeqd %xmm3, %xmm4
-; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [4294967295,4294967294,4294967293,4294967292]
-; SSE42-NEXT: pminud %xmm0, %xmm3
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [0,1,2,3]
-; SSE42-NEXT: movdqa %xmm3, %xmm5
-; SSE42-NEXT: pmaxud %xmm1, %xmm5
-; SSE42-NEXT: pcmpeqd %xmm3, %xmm5
-; SSE42-NEXT: packssdw %xmm4, %xmm5
-; SSE42-NEXT: packsswb %xmm2, %xmm5
-; SSE42-NEXT: pmovmskb %xmm5, %edx
-; SSE42-NEXT: xorl $65535, %edx # imm = 0xFFFF
-; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [4294967267,4294967266,4294967265,4294967264]
-; SSE42-NEXT: pminud %xmm0, %xmm2
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [28,29,30,31]
-; SSE42-NEXT: movdqa %xmm2, %xmm3
-; SSE42-NEXT: pmaxud %xmm1, %xmm3
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [4294967271,4294967270,4294967269,4294967268]
-; SSE42-NEXT: pminud %xmm0, %xmm4
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [24,25,26,27]
-; SSE42-NEXT: pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT: movdqa %xmm4, %xmm2
-; SSE42-NEXT: pmaxud %xmm1, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm4, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [4294967275,4294967274,4294967273,4294967272]
-; SSE42-NEXT: pminud %xmm0, %xmm4
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [20,21,22,23]
-; SSE42-NEXT: packssdw %xmm3, %xmm2
-; SSE42-NEXT: movdqa %xmm4, %xmm3
-; SSE42-NEXT: pmaxud %xmm1, %xmm3
-; SSE42-NEXT: pcmpeqd %xmm4, %xmm3
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [4294967279,4294967278,4294967277,4294967276]
-; SSE42-NEXT: pminud %xmm0, %xmm4
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [16,17,18,19]
-; SSE42-NEXT: movdqa %xmm4, %xmm5
-; SSE42-NEXT: pmaxud %xmm1, %xmm5
-; SSE42-NEXT: pcmpeqd %xmm4, %xmm5
-; SSE42-NEXT: packssdw %xmm3, %xmm5
-; SSE42-NEXT: packsswb %xmm2, %xmm5
-; SSE42-NEXT: pmovmskb %xmm5, %ecx
-; SSE42-NEXT: notl %ecx
-; SSE42-NEXT: shll $16, %ecx
-; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [4294967251,4294967250,4294967249,4294967248]
-; SSE42-NEXT: pminud %xmm0, %xmm2
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [44,45,46,47]
-; SSE42-NEXT: orl %edx, %ecx
-; SSE42-NEXT: movdqa %xmm2, %xmm3
-; SSE42-NEXT: pmaxud %xmm1, %xmm3
-; SSE42-NEXT: pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [4294967255,4294967254,4294967253,4294967252]
-; SSE42-NEXT: pminud %xmm0, %xmm4
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [40,41,42,43]
-; SSE42-NEXT: movdqa %xmm4, %xmm2
-; SSE42-NEXT: pmaxud %xmm1, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm4, %xmm2
-; SSE42-NEXT: packssdw %xmm3, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [4294967259,4294967258,4294967257,4294967256]
-; SSE42-NEXT: pminud %xmm0, %xmm3
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [36,37,38,39]
-; SSE42-NEXT: movdqa %xmm3, %xmm4
-; SSE42-NEXT: pmaxud %xmm1, %xmm4
-; SSE42-NEXT: pcmpeqd %xmm3, %xmm4
-; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [4294967263,4294967262,4294967261,4294967260]
-; SSE42-NEXT: pminud %xmm0, %xmm3
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [32,33,34,35]
-; SSE42-NEXT: movdqa %xmm3, %xmm5
-; SSE42-NEXT: pmaxud %xmm1, %xmm5
-; SSE42-NEXT: pcmpeqd %xmm3, %xmm5
-; SSE42-NEXT: packssdw %xmm4, %xmm5
-; SSE42-NEXT: packsswb %xmm2, %xmm5
-; SSE42-NEXT: pmovmskb %xmm5, %edx
-; SSE42-NEXT: xorl $65535, %edx # imm = 0xFFFF
-; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [4294967235,4294967234,4294967233,4294967232]
-; SSE42-NEXT: pminud %xmm0, %xmm2
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [60,61,62,63]
-; SSE42-NEXT: movdqa %xmm2, %xmm3
-; SSE42-NEXT: pmaxud %xmm1, %xmm3
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [4294967239,4294967238,4294967237,4294967236]
-; SSE42-NEXT: pminud %xmm0, %xmm4
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [56,57,58,59]
-; SSE42-NEXT: pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT: movdqa %xmm4, %xmm2
-; SSE42-NEXT: pmaxud %xmm1, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm4, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [4294967243,4294967242,4294967241,4294967240]
-; SSE42-NEXT: pminud %xmm0, %xmm4
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [52,53,54,55]
-; SSE42-NEXT: packssdw %xmm3, %xmm2
-; SSE42-NEXT: movdqa %xmm4, %xmm3
-; SSE42-NEXT: pmaxud %xmm1, %xmm3
-; SSE42-NEXT: pcmpeqd %xmm4, %xmm3
-; SSE42-NEXT: pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [48,49,50,51]
-; SSE42-NEXT: pmaxud %xmm0, %xmm1
-; SSE42-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE42-NEXT: packssdw %xmm3, %xmm1
-; SSE42-NEXT: packsswb %xmm2, %xmm1
-; SSE42-NEXT: pmovmskb %xmm1, %esi
-; SSE42-NEXT: notl %esi
-; SSE42-NEXT: shll $16, %esi
-; SSE42-NEXT: orl %edx, %esi
-; SSE42-NEXT: shlq $32, %rsi
-; SSE42-NEXT: orq %rcx, %rsi
-; SSE42-NEXT: movq %rsi, (%rdi)
-; SSE42-NEXT: retq
-;
-; AVX2-LABEL: lane_mask_v64i1_i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: vmovd %edx, %xmm0
-; AVX2-NEXT: vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT: vmovd %esi, %xmm1
-; AVX2-NEXT: vpbroadcastd %xmm1, %ymm1
-; AVX2-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
-; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [24,25,26,27,28,29,30,31]
-; AVX2-NEXT: vpmaxud %ymm0, %ymm2, %ymm3
-; AVX2-NEXT: vpcmpeqd %ymm3, %ymm2, %ymm2
-; AVX2-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3
-; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [16,17,18,19,20,21,22,23]
-; AVX2-NEXT: vpmaxud %ymm0, %ymm3, %ymm4
-; AVX2-NEXT: vpcmpeqd %ymm4, %ymm3, %ymm3
-; AVX2-NEXT: vpackssdw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm2[0,2,1,3]
-; AVX2-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX2-NEXT: vpxor %ymm2, %ymm3, %ymm3
-; AVX2-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm4
-; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4 # [8,9,10,11,12,13,14,15]
-; AVX2-NEXT: vpmaxud %ymm0, %ymm4, %ymm5
-; AVX2-NEXT: vpcmpeqd %ymm5, %ymm4, %ymm4
-; AVX2-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5
-; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm5 # [0,1,2,3,4,5,6,7]
-; AVX2-NEXT: vpmaxud %ymm0, %ymm5, %ymm6
-; AVX2-NEXT: vpcmpeqd %ymm6, %ymm5, %ymm5
-; AVX2-NEXT: vpackssdw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
-; AVX2-NEXT: vpxor %ymm2, %ymm4, %ymm4
-; AVX2-NEXT: vpacksswb %ymm3, %ymm4, %ymm3
-; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
-; AVX2-NEXT: vpmovmskb %ymm3, %ecx
-; AVX2-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3
-; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [56,57,58,59,60,61,62,63]
-; AVX2-NEXT: vpmaxud %ymm0, %ymm3, %ymm4
-; AVX2-NEXT: vpcmpeqd %ymm4, %ymm3, %ymm3
-; AVX2-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm4
-; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4 # [48,49,50,51,52,53,54,55]
-; AVX2-NEXT: vpmaxud %ymm0, %ymm4, %ymm5
-; AVX2-NEXT: vpcmpeqd %ymm5, %ymm4, %ymm4
-; AVX2-NEXT: vpackssdw %ymm3, %ymm4, %ymm3
-; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
-; AVX2-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm4
-; AVX2-NEXT: vpxor %ymm2, %ymm3, %ymm3
-; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4 # [40,41,42,43,44,45,46,47]
-; AVX2-NEXT: vpmaxud %ymm0, %ymm4, %ymm5
-; AVX2-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [32,33,34,35,36,37,38,39]
-; AVX2-NEXT: vpcmpeqd %ymm5, %ymm4, %ymm4
-; AVX2-NEXT: vpmaxud %ymm0, %ymm1, %ymm0
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm1, %ymm0
-; AVX2-NEXT: vpackssdw %ymm4, %ymm0, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpacksswb %ymm3, %ymm0, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT: vpmovmskb %ymm0, %edx
-; AVX2-NEXT: shlq $32, %rdx
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: movq %rdx, (%rdi)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: lane_mask_v64i1_i32:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastd %esi, %zmm0
-; AVX512-NEXT: vpbroadcastd %edi, %zmm1
-; AVX512-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm3
-; AVX512-NEXT: vpcmpltud %zmm0, %zmm2, %k0
-; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm2 # [16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
-; AVX512-NEXT: vpcmpltud %zmm0, %zmm2, %k1
-; AVX512-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT: kunpckwd %k0, %k1, %k0
-; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47]
-; AVX512-NEXT: vpcmpltud %zmm0, %zmm2, %k1
-; AVX512-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63]
-; AVX512-NEXT: vpcmpltud %zmm0, %zmm1, %k2
-; AVX512-NEXT: kunpckwd %k1, %k2, %k1
-; AVX512-NEXT: kunpckdq %k0, %k1, %k0
-; AVX512-NEXT: vpmovm2b %k0, %zmm0
-; AVX512-NEXT: retq
- %active.lane.mask = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 %index, i32 %TC)
- ret <64 x i1> %active.lane.mask
-}
-
-;
-; i32 (zero index)
-;
-
-define <2 x i1> @lane_mask_v2i1_i32_zero(i32 %TC) {
-; SSE2-LABEL: lane_mask_v2i1_i32_zero:
-; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: lane_mask_v2i1_i32_zero:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movd %edi, %xmm0
+; SSE42-NEXT: movd %edi, %xmm0
; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
; SSE42-NEXT: movq {{.*#+}} xmm1 = [0,1,0,0]
; SSE42-NEXT: pmaxud %xmm1, %xmm0
@@ -1180,10 +818,13 @@ define <16 x i1> @lane_mask_v16i1_i32_zero(i32 %TC) {
;
; AVX512-LABEL: lane_mask_v16i1_i32_zero:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastd %edi, %zmm0
-; AVX512-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
+; AVX512-NEXT: cmpl $16, %edi
+; AVX512-NEXT: movl $16, %eax
+; AVX512-NEXT: cmovbl %edi, %eax
+; AVX512-NEXT: movl $-1, %ecx
+; AVX512-NEXT: bzhil %eax, %ecx, %eax
+; AVX512-NEXT: kmovd %eax, %k0
; AVX512-NEXT: vpmovm2b %k0, %xmm0
-; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 0, i32 %TC)
ret <16 x i1> %active.lane.mask
@@ -1304,10 +945,12 @@ define <32 x i1> @lane_mask_v32i1_i32_zero(i32 %TC) {
;
; AVX512-LABEL: lane_mask_v32i1_i32_zero:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastd %edi, %zmm0
-; AVX512-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
-; AVX512-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT: kunpckwd %k0, %k1, %k0
+; AVX512-NEXT: cmpl $32, %edi
+; AVX512-NEXT: movl $32, %eax
+; AVX512-NEXT: cmovbl %edi, %eax
+; AVX512-NEXT: movl $-1, %ecx
+; AVX512-NEXT: bzhil %eax, %ecx, %eax
+; AVX512-NEXT: kmovd %eax, %k0
; AVX512-NEXT: vpmovm2b %k0, %ymm0
; AVX512-NEXT: retq
%active.lane.mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i32(i32 0, i32 %TC)
@@ -1315,229 +958,38 @@ define <32 x i1> @lane_mask_v32i1_i32_zero(i32 %TC) {
}
define <64 x i1> @lane_mask_v64i1_i32_zero(i32 %TC) {
-; SSE2-LABEL: lane_mask_v64i1_i32_zero:
-; SSE2: # %bb.0:
-; SSE2-NEXT: movd %esi, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: packssdw %xmm1, %xmm2
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: packssdw %xmm1, %xmm3
-; SSE2-NEXT: packsswb %xmm2, %xmm3
-; SSE2-NEXT: pmovmskb %xmm3, %edx
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: packssdw %xmm1, %xmm2
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: packssdw %xmm1, %xmm3
-; SSE2-NEXT: packsswb %xmm2, %xmm3
-; SSE2-NEXT: pmovmskb %xmm3, %ecx
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: shll $16, %ecx
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: orl %edx, %ecx
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: packssdw %xmm1, %xmm2
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: packssdw %xmm3, %xmm1
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: packsswb %xmm2, %xmm1
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pmovmskb %xmm1, %edx
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: packssdw %xmm3, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: packssdw %xmm1, %xmm0
-; SSE2-NEXT: packsswb %xmm2, %xmm0
-; SSE2-NEXT: pmovmskb %xmm0, %esi
-; SSE2-NEXT: shll $16, %esi
-; SSE2-NEXT: orl %edx, %esi
-; SSE2-NEXT: shlq $32, %rsi
-; SSE2-NEXT: orq %rcx, %rsi
-; SSE2-NEXT: movq %rsi, (%rdi)
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: lane_mask_v64i1_i32_zero:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: movd %esi, %xmm0
-; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [12,13,14,15]
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pmaxud %xmm1, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [8,9,10,11]
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pmaxud %xmm1, %xmm3
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm3
-; SSE42-NEXT: packssdw %xmm2, %xmm3
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [4,5,6,7]
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pmaxud %xmm1, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [0,1,2,3]
-; SSE42-NEXT: movdqa %xmm0, %xmm4
-; SSE42-NEXT: pmaxud %xmm1, %xmm4
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm4
-; SSE42-NEXT: packssdw %xmm2, %xmm4
-; SSE42-NEXT: packsswb %xmm3, %xmm4
-; SSE42-NEXT: pmovmskb %xmm4, %edx
-; SSE42-NEXT: xorl $65535, %edx # imm = 0xFFFF
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [28,29,30,31]
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pmaxud %xmm1, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [24,25,26,27]
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pmaxud %xmm1, %xmm3
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm3
-; SSE42-NEXT: packssdw %xmm2, %xmm3
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [20,21,22,23]
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pmaxud %xmm1, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [16,17,18,19]
-; SSE42-NEXT: movdqa %xmm0, %xmm4
-; SSE42-NEXT: pmaxud %xmm1, %xmm4
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm4
-; SSE42-NEXT: packssdw %xmm2, %xmm4
-; SSE42-NEXT: packsswb %xmm3, %xmm4
-; SSE42-NEXT: pmovmskb %xmm4, %ecx
-; SSE42-NEXT: notl %ecx
-; SSE42-NEXT: shll $16, %ecx
-; SSE42-NEXT: orl %edx, %ecx
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [44,45,46,47]
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pmaxud %xmm1, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [40,41,42,43]
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pmaxud %xmm1, %xmm3
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm3
-; SSE42-NEXT: packssdw %xmm2, %xmm3
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [36,37,38,39]
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pmaxud %xmm1, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [32,33,34,35]
-; SSE42-NEXT: movdqa %xmm0, %xmm4
-; SSE42-NEXT: pmaxud %xmm1, %xmm4
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm4
-; SSE42-NEXT: packssdw %xmm2, %xmm4
-; SSE42-NEXT: packsswb %xmm3, %xmm4
-; SSE42-NEXT: pmovmskb %xmm4, %edx
-; SSE42-NEXT: xorl $65535, %edx # imm = 0xFFFF
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [60,61,62,63]
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pmaxud %xmm1, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [56,57,58,59]
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pmaxud %xmm1, %xmm3
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm3
-; SSE42-NEXT: packssdw %xmm2, %xmm3
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [52,53,54,55]
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pmaxud %xmm1, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [48,49,50,51]
-; SSE42-NEXT: pmaxud %xmm1, %xmm0
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm0
-; SSE42-NEXT: packssdw %xmm2, %xmm0
-; SSE42-NEXT: packsswb %xmm3, %xmm0
-; SSE42-NEXT: pmovmskb %xmm0, %esi
-; SSE42-NEXT: notl %esi
-; SSE42-NEXT: shll $16, %esi
-; SSE42-NEXT: orl %edx, %esi
-; SSE42-NEXT: shlq $32, %rsi
-; SSE42-NEXT: orq %rcx, %rsi
-; SSE42-NEXT: movq %rsi, (%rdi)
-; SSE42-NEXT: retq
+; SSE-LABEL: lane_mask_v64i1_i32_zero:
+; SSE: # %bb.0:
+; SSE-NEXT: cmpl $64, %esi
+; SSE-NEXT: movl $64, %ecx
+; SSE-NEXT: cmovbl %esi, %ecx
+; SSE-NEXT: movl $1, %edx
+; SSE-NEXT: # kill: def $cl killed $cl killed $ecx
+; SSE-NEXT: shlq %cl, %rdx
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: decq %rdx
+; SSE-NEXT: movq %rdx, (%rdi)
+; SSE-NEXT: retq
;
; AVX2-LABEL: lane_mask_v64i1_i32_zero:
; AVX2: # %bb.0:
+; AVX2-NEXT: cmpl $64, %esi
+; AVX2-NEXT: movl $64, %ecx
+; AVX2-NEXT: cmovbl %esi, %ecx
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: vmovd %esi, %xmm0
-; AVX2-NEXT: vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [24,25,26,27,28,29,30,31]
-; AVX2-NEXT: vpmaxud %ymm1, %ymm0, %ymm2
-; AVX2-NEXT: vpcmpeqd %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [16,17,18,19,20,21,22,23]
-; AVX2-NEXT: vpmaxud %ymm2, %ymm0, %ymm3
-; AVX2-NEXT: vpcmpeqd %ymm2, %ymm3, %ymm2
-; AVX2-NEXT: vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [8,9,10,11,12,13,14,15]
-; AVX2-NEXT: vpmaxud %ymm3, %ymm0, %ymm4
-; AVX2-NEXT: vpcmpeqd %ymm3, %ymm4, %ymm3
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,2,3,4,5,6,7]
-; AVX2-NEXT: vpmaxud %ymm4, %ymm0, %ymm5
-; AVX2-NEXT: vpcmpeqd %ymm4, %ymm5, %ymm4
-; AVX2-NEXT: vpackssdw %ymm3, %ymm4, %ymm3
-; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
-; AVX2-NEXT: vpxor %ymm2, %ymm3, %ymm3
-; AVX2-NEXT: vpacksswb %ymm1, %ymm3, %ymm1
-; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT: vpmovmskb %ymm1, %ecx
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [56,57,58,59,60,61,62,63]
-; AVX2-NEXT: vpmaxud %ymm1, %ymm0, %ymm3
-; AVX2-NEXT: vpcmpeqd %ymm1, %ymm3, %ymm1
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [48,49,50,51,52,53,54,55]
-; AVX2-NEXT: vpmaxud %ymm3, %ymm0, %ymm4
-; AVX2-NEXT: vpcmpeqd %ymm3, %ymm4, %ymm3
-; AVX2-NEXT: vpackssdw %ymm1, %ymm3, %ymm1
-; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [40,41,42,43,44,45,46,47]
-; AVX2-NEXT: vpmaxud %ymm3, %ymm0, %ymm4
-; AVX2-NEXT: vpcmpeqd %ymm3, %ymm4, %ymm3
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [32,33,34,35,36,37,38,39]
-; AVX2-NEXT: vpmaxud %ymm4, %ymm0, %ymm0
-; AVX2-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0
-; AVX2-NEXT: vpackssdw %ymm3, %ymm0, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpacksswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT: vpmovmskb %ymm0, %edx
-; AVX2-NEXT: shlq $32, %rdx
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: movq %rdx, (%rdi)
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: movq $-1, %rdx
+; AVX2-NEXT: bzhiq %rcx, %rdx, %rcx
+; AVX2-NEXT: movq %rcx, (%rdi)
; AVX2-NEXT: retq
;
; AVX512-LABEL: lane_mask_v64i1_i32_zero:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastd %edi, %zmm0
-; AVX512-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
-; AVX512-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k2
-; AVX512-NEXT: kunpckwd %k0, %k1, %k0
-; AVX512-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT: kunpckwd %k2, %k1, %k1
-; AVX512-NEXT: kunpckdq %k0, %k1, %k0
+; AVX512-NEXT: cmpl $64, %edi
+; AVX512-NEXT: movl $64, %eax
+; AVX512-NEXT: cmovbl %edi, %eax
+; AVX512-NEXT: movq $-1, %rcx
+; AVX512-NEXT: bzhiq %rax, %rcx, %rax
+; AVX512-NEXT: kmovq %rax, %k0
; AVX512-NEXT: vpmovm2b %k0, %zmm0
; AVX512-NEXT: retq
%active.lane.mask = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 0, i32 %TC)
@@ -1948,195 +1400,195 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
; SSE2-LABEL: lane_mask_v16i1_i64:
; SSE2: # %bb.0:
; SSE2-NEXT: movq %rdi, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,1,0,1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,1,0,1]
; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pxor %xmm0, %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: pxor %xmm0, %xmm6
; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
-; SSE2-NEXT: paddq %xmm4, %xmm2
+; SSE2-NEXT: paddq %xmm5, %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm1
; SSE2-NEXT: pxor %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm5, %xmm3
+; SSE2-NEXT: movdqa %xmm6, %xmm3
; SSE2-NEXT: pcmpgtd %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm6, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm7
+; SSE2-NEXT: pand %xmm4, %xmm7
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
; SSE2-NEXT: por %xmm2, %xmm1
; SSE2-NEXT: por %xmm7, %xmm1
; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [2,3]
-; SSE2-NEXT: paddq %xmm4, %xmm3
+; SSE2-NEXT: paddq %xmm5, %xmm3
; SSE2-NEXT: movdqa %xmm3, %xmm2
; SSE2-NEXT: pxor %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT: movdqa %xmm6, %xmm4
+; SSE2-NEXT: pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm6, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm2[1,1,3,3]
; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
; SSE2-NEXT: por %xmm3, %xmm2
; SSE2-NEXT: por %xmm8, %xmm2
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [4,5]
-; SSE2-NEXT: paddq %xmm4, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm3
+; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [4,5]
+; SSE2-NEXT: paddq %xmm5, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm3
; SSE2-NEXT: pxor %xmm0, %xmm3
-; SSE2-NEXT: movdqa %xmm5, %xmm7
+; SSE2-NEXT: movdqa %xmm6, %xmm7
; SSE2-NEXT: pcmpgtd %xmm3, %xmm7
; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT: pcmpeqd %xmm6, %xmm3
; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm3[1,1,3,3]
; SSE2-NEXT: pand %xmm8, %xmm9
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm6, %xmm3
+; SSE2-NEXT: por %xmm4, %xmm3
; SSE2-NEXT: por %xmm9, %xmm3
; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [6,7]
-; SSE2-NEXT: paddq %xmm4, %xmm7
-; SSE2-NEXT: movdqa %xmm7, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm5, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm8
+; SSE2-NEXT: paddq %xmm5, %xmm7
+; SSE2-NEXT: movdqa %xmm7, %xmm4
+; SSE2-NEXT: pxor %xmm0, %xmm4
+; SSE2-NEXT: movdqa %xmm6, %xmm8
+; SSE2-NEXT: pcmpgtd %xmm4, %xmm8
; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]
+; SSE2-NEXT: pcmpeqd %xmm6, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm4[1,1,3,3]
; SSE2-NEXT: pand %xmm9, %xmm10
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm6
-; SSE2-NEXT: por %xmm10, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm8[1,1,3,3]
+; SSE2-NEXT: por %xmm7, %xmm4
+; SSE2-NEXT: por %xmm10, %xmm4
; SSE2-NEXT: movdqa {{.*#+}} xmm8 = [8,9]
-; SSE2-NEXT: paddq %xmm4, %xmm8
+; SSE2-NEXT: paddq %xmm5, %xmm8
; SSE2-NEXT: movdqa %xmm8, %xmm7
; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm5, %xmm9
+; SSE2-NEXT: movdqa %xmm6, %xmm9
; SSE2-NEXT: pcmpgtd %xmm7, %xmm9
; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm7
+; SSE2-NEXT: pcmpeqd %xmm6, %xmm7
; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm7[1,1,3,3]
; SSE2-NEXT: pand %xmm10, %xmm11
; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm9[1,1,3,3]
; SSE2-NEXT: por %xmm8, %xmm7
; SSE2-NEXT: por %xmm11, %xmm7
; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [10,11]
-; SSE2-NEXT: paddq %xmm4, %xmm9
+; SSE2-NEXT: paddq %xmm5, %xmm9
; SSE2-NEXT: movdqa %xmm9, %xmm8
; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm5, %xmm10
+; SSE2-NEXT: movdqa %xmm6, %xmm10
; SSE2-NEXT: pcmpgtd %xmm8, %xmm10
; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm8
+; SSE2-NEXT: pcmpeqd %xmm6, %xmm8
; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm8[1,1,3,3]
; SSE2-NEXT: pand %xmm11, %xmm12
; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm10[1,1,3,3]
; SSE2-NEXT: por %xmm9, %xmm8
; SSE2-NEXT: por %xmm12, %xmm8
; SSE2-NEXT: movdqa {{.*#+}} xmm10 = [12,13]
-; SSE2-NEXT: paddq %xmm4, %xmm10
+; SSE2-NEXT: paddq %xmm5, %xmm10
; SSE2-NEXT: movdqa %xmm10, %xmm9
; SSE2-NEXT: pxor %xmm0, %xmm9
-; SSE2-NEXT: movdqa %xmm5, %xmm11
+; SSE2-NEXT: movdqa %xmm6, %xmm11
; SSE2-NEXT: pcmpgtd %xmm9, %xmm11
; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm9
+; SSE2-NEXT: pcmpeqd %xmm6, %xmm9
; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm9[1,1,3,3]
; SSE2-NEXT: pand %xmm12, %xmm13
; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm11[1,1,3,3]
; SSE2-NEXT: por %xmm10, %xmm9
; SSE2-NEXT: por %xmm13, %xmm9
-; SSE2-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [14,15]
-; SSE2-NEXT: movdqa %xmm4, %xmm10
+; SSE2-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5 # [14,15]
+; SSE2-NEXT: movdqa %xmm5, %xmm10
; SSE2-NEXT: pxor %xmm0, %xmm10
-; SSE2-NEXT: movdqa %xmm5, %xmm11
+; SSE2-NEXT: movdqa %xmm6, %xmm11
; SSE2-NEXT: pcmpgtd %xmm10, %xmm11
; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm10
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm10[1,1,3,3]
-; SSE2-NEXT: pand %xmm12, %xmm5
+; SSE2-NEXT: pcmpeqd %xmm6, %xmm10
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm10[1,1,3,3]
+; SSE2-NEXT: pand %xmm12, %xmm6
; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm10
; SSE2-NEXT: por %xmm5, %xmm10
-; SSE2-NEXT: movq %rsi, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
+; SSE2-NEXT: por %xmm6, %xmm10
+; SSE2-NEXT: movq %rsi, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,0,1]
; SSE2-NEXT: pxor %xmm0, %xmm10
-; SSE2-NEXT: pxor %xmm0, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm10, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm10
+; SSE2-NEXT: pxor %xmm0, %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: pcmpgtd %xmm10, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm6[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm10
; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
; SSE2-NEXT: pand %xmm11, %xmm10
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm10, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT: por %xmm10, %xmm6
; SSE2-NEXT: pxor %xmm0, %xmm9
-; SSE2-NEXT: movdqa %xmm4, %xmm10
+; SSE2-NEXT: movdqa %xmm5, %xmm10
; SSE2-NEXT: pcmpgtd %xmm9, %xmm10
; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm9
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm9
; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm9[1,1,3,3]
; SSE2-NEXT: pand %xmm11, %xmm12
; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
; SSE2-NEXT: por %xmm12, %xmm9
-; SSE2-NEXT: packssdw %xmm5, %xmm9
+; SSE2-NEXT: packssdw %xmm6, %xmm9
; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm8
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: pcmpgtd %xmm8, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm8
; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
; SSE2-NEXT: pand %xmm10, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[1,1,3,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]
; SSE2-NEXT: por %xmm8, %xmm10
; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm7
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: pcmpgtd %xmm7, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm7
; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
; SSE2-NEXT: pand %xmm8, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm5
-; SSE2-NEXT: packssdw %xmm10, %xmm5
-; SSE2-NEXT: packssdw %xmm9, %xmm5
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm4, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm6
; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm6
+; SSE2-NEXT: por %xmm7, %xmm6
+; SSE2-NEXT: packssdw %xmm10, %xmm6
+; SSE2-NEXT: packssdw %xmm9, %xmm6
+; SSE2-NEXT: pxor %xmm0, %xmm4
+; SSE2-NEXT: movdqa %xmm5, %xmm7
+; SSE2-NEXT: pcmpgtd %xmm4, %xmm7
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT: pand %xmm8, %xmm4
; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm6, %xmm7
+; SSE2-NEXT: por %xmm4, %xmm7
; SSE2-NEXT: pxor %xmm0, %xmm3
-; SSE2-NEXT: movdqa %xmm4, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm3, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm3
+; SSE2-NEXT: movdqa %xmm5, %xmm4
+; SSE2-NEXT: pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm4[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm3
; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm3[1,1,3,3]
; SSE2-NEXT: pand %xmm8, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
; SSE2-NEXT: por %xmm9, %xmm3
; SSE2-NEXT: packssdw %xmm7, %xmm3
; SSE2-NEXT: pxor %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm4, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm2
+; SSE2-NEXT: movdqa %xmm5, %xmm4
+; SSE2-NEXT: pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
; SSE2-NEXT: pand %xmm7, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: por %xmm2, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT: por %xmm2, %xmm4
; SSE2-NEXT: pxor %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm4, %xmm0
+; SSE2-NEXT: movdqa %xmm5, %xmm0
; SSE2-NEXT: pcmpgtd %xmm1, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
; SSE2-NEXT: pand %xmm2, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: packssdw %xmm6, %xmm0
+; SSE2-NEXT: packssdw %xmm4, %xmm0
; SSE2-NEXT: packssdw %xmm3, %xmm0
-; SSE2-NEXT: packsswb %xmm5, %xmm0
+; SSE2-NEXT: packsswb %xmm6, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: lane_mask_v16i1_i64:
@@ -2144,72 +1596,72 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
; SSE42-NEXT: movq %rdi, %xmm0
; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
-; SSE42-NEXT: movdqa %xmm0, %xmm4
-; SSE42-NEXT: pxor %xmm1, %xmm4
+; SSE42-NEXT: movdqa %xmm0, %xmm5
+; SSE42-NEXT: pxor %xmm1, %xmm5
; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
; SSE42-NEXT: paddq %xmm0, %xmm3
-; SSE42-NEXT: movdqa %xmm3, %xmm5
-; SSE42-NEXT: pxor %xmm1, %xmm5
-; SSE42-NEXT: movdqa %xmm4, %xmm2
-; SSE42-NEXT: pcmpgtq %xmm5, %xmm2
+; SSE42-NEXT: movdqa %xmm3, %xmm4
+; SSE42-NEXT: pxor %xmm1, %xmm4
+; SSE42-NEXT: movdqa %xmm5, %xmm2
+; SSE42-NEXT: pcmpgtq %xmm4, %xmm2
; SSE42-NEXT: por %xmm3, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [2,3]
-; SSE42-NEXT: paddq %xmm0, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm6
+; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [2,3]
+; SSE42-NEXT: paddq %xmm0, %xmm4
+; SSE42-NEXT: movdqa %xmm4, %xmm6
; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm4, %xmm3
+; SSE42-NEXT: movdqa %xmm5, %xmm3
; SSE42-NEXT: pcmpgtq %xmm6, %xmm3
-; SSE42-NEXT: por %xmm5, %xmm3
+; SSE42-NEXT: por %xmm4, %xmm3
; SSE42-NEXT: movdqa {{.*#+}} xmm6 = [4,5]
; SSE42-NEXT: paddq %xmm0, %xmm6
; SSE42-NEXT: movdqa %xmm6, %xmm7
; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm4, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm5
-; SSE42-NEXT: por %xmm6, %xmm5
+; SSE42-NEXT: movdqa %xmm5, %xmm4
+; SSE42-NEXT: pcmpgtq %xmm7, %xmm4
+; SSE42-NEXT: por %xmm6, %xmm4
; SSE42-NEXT: movdqa {{.*#+}} xmm7 = [6,7]
; SSE42-NEXT: paddq %xmm0, %xmm7
; SSE42-NEXT: movdqa %xmm7, %xmm8
; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm4, %xmm6
+; SSE42-NEXT: movdqa %xmm5, %xmm6
; SSE42-NEXT: pcmpgtq %xmm8, %xmm6
; SSE42-NEXT: por %xmm7, %xmm6
; SSE42-NEXT: movdqa {{.*#+}} xmm8 = [8,9]
; SSE42-NEXT: paddq %xmm0, %xmm8
; SSE42-NEXT: movdqa %xmm8, %xmm9
; SSE42-NEXT: pxor %xmm1, %xmm9
-; SSE42-NEXT: movdqa %xmm4, %xmm7
+; SSE42-NEXT: movdqa %xmm5, %xmm7
; SSE42-NEXT: pcmpgtq %xmm9, %xmm7
; SSE42-NEXT: por %xmm8, %xmm7
; SSE42-NEXT: movdqa {{.*#+}} xmm9 = [10,11]
; SSE42-NEXT: paddq %xmm0, %xmm9
; SSE42-NEXT: movdqa %xmm9, %xmm10
; SSE42-NEXT: pxor %xmm1, %xmm10
-; SSE42-NEXT: movdqa %xmm4, %xmm8
+; SSE42-NEXT: movdqa %xmm5, %xmm8
; SSE42-NEXT: pcmpgtq %xmm10, %xmm8
; SSE42-NEXT: por %xmm9, %xmm8
; SSE42-NEXT: movdqa {{.*#+}} xmm9 = [12,13]
; SSE42-NEXT: paddq %xmm0, %xmm9
; SSE42-NEXT: movdqa %xmm9, %xmm10
; SSE42-NEXT: pxor %xmm1, %xmm10
-; SSE42-NEXT: movdqa %xmm4, %xmm11
+; SSE42-NEXT: movdqa %xmm5, %xmm11
; SSE42-NEXT: pcmpgtq %xmm10, %xmm11
; SSE42-NEXT: por %xmm9, %xmm11
; SSE42-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [14,15]
; SSE42-NEXT: movdqa %xmm0, %xmm9
; SSE42-NEXT: pxor %xmm1, %xmm9
-; SSE42-NEXT: pcmpgtq %xmm9, %xmm4
-; SSE42-NEXT: por %xmm0, %xmm4
+; SSE42-NEXT: pcmpgtq %xmm9, %xmm5
+; SSE42-NEXT: por %xmm0, %xmm5
; SSE42-NEXT: movq %rsi, %xmm0
; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE42-NEXT: pxor %xmm1, %xmm4
+; SSE42-NEXT: pxor %xmm1, %xmm5
; SSE42-NEXT: pxor %xmm1, %xmm0
; SSE42-NEXT: movdqa %xmm0, %xmm9
-; SSE42-NEXT: pcmpgtq %xmm4, %xmm9
+; SSE42-NEXT: pcmpgtq %xmm5, %xmm9
; SSE42-NEXT: pxor %xmm1, %xmm11
-; SSE42-NEXT: movdqa %xmm0, %xmm4
-; SSE42-NEXT: pcmpgtq %xmm11, %xmm4
-; SSE42-NEXT: packssdw %xmm9, %xmm4
+; SSE42-NEXT: movdqa %xmm0, %xmm5
+; SSE42-NEXT: pcmpgtq %xmm11, %xmm5
+; SSE42-NEXT: packssdw %xmm9, %xmm5
; SSE42-NEXT: pxor %xmm1, %xmm8
; SSE42-NEXT: movdqa %xmm0, %xmm9
; SSE42-NEXT: pcmpgtq %xmm8, %xmm9
@@ -2217,14 +1669,14 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
; SSE42-NEXT: movdqa %xmm0, %xmm8
; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
; SSE42-NEXT: packssdw %xmm9, %xmm8
-; SSE42-NEXT: packssdw %xmm4, %xmm8
+; SSE42-NEXT: packssdw %xmm5, %xmm8
; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm4
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm4
-; SSE42-NEXT: pxor %xmm1, %xmm5
+; SSE42-NEXT: movdqa %xmm0, %xmm5
+; SSE42-NEXT: pcmpgtq %xmm6, %xmm5
+; SSE42-NEXT: pxor %xmm1, %xmm4
; SSE42-NEXT: movdqa %xmm0, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm5, %xmm6
-; SSE42-NEXT: packssdw %xmm4, %xmm6
+; SSE42-NEXT: pcmpgtq %xmm4, %xmm6
+; SSE42-NEXT: packssdw %xmm5, %xmm6
; SSE42-NEXT: pxor %xmm1, %xmm3
; SSE42-NEXT: movdqa %xmm0, %xmm4
; SSE42-NEXT: pcmpgtq %xmm3, %xmm4
@@ -2281,17 +1733,16 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
;
; AVX512-LABEL: lane_mask_v16i1_i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq %rsi, %zmm0
-; AVX512-NEXT: vpbroadcastq %rdi, %zmm1
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [0,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm2, %k0
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [8,9,10,11,12,13,14,15]
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm1, %k1
-; AVX512-NEXT: kunpckbw %k0, %k1, %k0
+; AVX512-NEXT: xorl %eax, %eax
+; AVX512-NEXT: subq %rdi, %rsi
+; AVX512-NEXT: cmovaq %rsi, %rax
+; AVX512-NEXT: cmpq $16, %rax
+; AVX512-NEXT: movl $16, %ecx
+; AVX512-NEXT: cmovbq %rax, %rcx
+; AVX512-NEXT: movl $-1, %eax
+; AVX512-NEXT: bzhil %ecx, %eax, %eax
+; AVX512-NEXT: kmovd %eax, %k0
; AVX512-NEXT: vpmovm2b %k0, %xmm0
-; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 %index, i64 %TC)
ret <16 x i1> %active.lane.mask
@@ -2831,20 +2282,9 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
; SSE42-NEXT: por %xmm5, %xmm7
; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [20,21]
-; SSE42-NEXT: paddq %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm5, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
; SSE42-NEXT: movdqa %xmm3, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm5
-; SSE42-NEXT: packssdw %xmm6, %xmm5
-; SSE42-NEXT: movdqa {{.*#+}} xmm6 = [18,19]
+; SSE42-NEXT: pcmpgtq %xmm7, %xmm5
+; SSE42-NEXT: movdqa {{.*#+}} xmm6 = [20,21]
; SSE42-NEXT: paddq %xmm2, %xmm6
; SSE42-NEXT: movdqa %xmm6, %xmm7
; SSE42-NEXT: pxor %xmm1, %xmm7
@@ -2854,6 +2294,17 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
; SSE42-NEXT: pxor %xmm1, %xmm8
; SSE42-NEXT: movdqa %xmm3, %xmm6
; SSE42-NEXT: pcmpgtq %xmm8, %xmm6
+; SSE42-NEXT: packssdw %xmm5, %xmm6
+; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [18,19]
+; SSE42-NEXT: paddq %xmm2, %xmm5
+; SSE42-NEXT: movdqa %xmm5, %xmm7
+; SSE42-NEXT: pxor %xmm1, %xmm7
+; SSE42-NEXT: movdqa %xmm0, %xmm8
+; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
+; SSE42-NEXT: por %xmm5, %xmm8
+; SSE42-NEXT: pxor %xmm1, %xmm8
+; SSE42-NEXT: movdqa %xmm3, %xmm5
+; SSE42-NEXT: pcmpgtq %xmm8, %xmm5
; SSE42-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [16,17]
; SSE42-NEXT: movdqa %xmm2, %xmm7
; SSE42-NEXT: pxor %xmm1, %xmm7
@@ -2861,8 +2312,8 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
; SSE42-NEXT: por %xmm2, %xmm0
; SSE42-NEXT: pxor %xmm1, %xmm0
; SSE42-NEXT: pcmpgtq %xmm0, %xmm3
-; SSE42-NEXT: packssdw %xmm6, %xmm3
; SSE42-NEXT: packssdw %xmm5, %xmm3
+; SSE42-NEXT: packssdw %xmm6, %xmm3
; SSE42-NEXT: packsswb %xmm4, %xmm3
; SSE42-NEXT: pmovmskb %xmm3, %edx
; SSE42-NEXT: shll $16, %edx
@@ -2945,23 +2396,15 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
;
; AVX512-LABEL: lane_mask_v32i1_i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq %rsi, %zmm0
-; AVX512-NEXT: vpbroadcastq %rdi, %zmm1
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [0,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm3
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm2, %k0
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm2 # [8,9,10,11,12,13,14,15]
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm2, %k1
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT: kunpckbw %k0, %k1, %k0
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [16,17,18,19,20,21,22,23]
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm2, %k1
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [24,25,26,27,28,29,30,31]
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm1, %k2
-; AVX512-NEXT: kunpckbw %k1, %k2, %k1
-; AVX512-NEXT: kunpckwd %k0, %k1, %k0
+; AVX512-NEXT: xorl %eax, %eax
+; AVX512-NEXT: subq %rdi, %rsi
+; AVX512-NEXT: cmovaq %rsi, %rax
+; AVX512-NEXT: cmpq $32, %rax
+; AVX512-NEXT: movl $32, %ecx
+; AVX512-NEXT: cmovbq %rax, %rcx
+; AVX512-NEXT: movl $-1, %eax
+; AVX512-NEXT: bzhil %ecx, %eax, %eax
+; AVX512-NEXT: kmovd %eax, %k0
; AVX512-NEXT: vpmovm2b %k0, %ymm0
; AVX512-NEXT: retq
%active.lane.mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 %index, i64 %TC)
@@ -2969,1367 +2412,105 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
}
define <64 x i1> @lane_mask_v64i1_i64(i64 %index, i64 %TC) {
-; SSE2-LABEL: lane_mask_v64i1_i64:
+; SSE-LABEL: lane_mask_v64i1_i64:
+; SSE: # %bb.0:
+; SSE-NEXT: xorl %eax, %eax
+; SSE-NEXT: subq %rsi, %rdx
+; SSE-NEXT: cmovaq %rdx, %rax
+; SSE-NEXT: cmpq $64, %rax
+; SSE-NEXT: movl $64, %ecx
+; SSE-NEXT: cmovbq %rax, %rcx
+; SSE-NEXT: movl $1, %edx
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shlq %cl, %rdx
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: decq %rdx
+; SSE-NEXT: movq %rdx, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: lane_mask_v64i1_i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: subq %rsi, %rdx
+; AVX2-NEXT: cmovaq %rdx, %rax
+; AVX2-NEXT: cmpq $64, %rax
+; AVX2-NEXT: movl $64, %ecx
+; AVX2-NEXT: cmovbq %rax, %rcx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq $-1, %rdx
+; AVX2-NEXT: bzhiq %rcx, %rdx, %rcx
+; AVX2-NEXT: movq %rcx, (%rdi)
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: lane_mask_v64i1_i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: xorl %eax, %eax
+; AVX512-NEXT: subq %rdi, %rsi
+; AVX512-NEXT: cmovaq %rsi, %rax
+; AVX512-NEXT: cmpq $64, %rax
+; AVX512-NEXT: movl $64, %ecx
+; AVX512-NEXT: cmovbq %rax, %rcx
+; AVX512-NEXT: movq $-1, %rax
+; AVX512-NEXT: bzhiq %rcx, %rax, %rax
+; AVX512-NEXT: kmovq %rax, %k0
+; AVX512-NEXT: vpmovm2b %k0, %zmm0
+; AVX512-NEXT: retq
+ %active.lane.mask = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 %index, i64 %TC)
+ ret <64 x i1> %active.lane.mask
+}
+
+;
+; i64 (zero index)
+;
+
+define <2 x i1> @lane_mask_v2i1_i64_zero(i64 %TC) {
+; SSE2-LABEL: lane_mask_v2i1_i64_zero:
; SSE2: # %bb.0:
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: movq %rsi, %xmm0
+; SSE2-NEXT: movq %rdi, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE2-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE2-NEXT: por %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: lane_mask_v2i1_i64_zero:
+; SSE42: # %bb.0:
+; SSE42-NEXT: movq %rdi, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; SSE42-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE42-NEXT: retq
+;
+; AVX2-LABEL: lane_mask_v2i1_i64_zero:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovq %rdi, %xmm0
+; AVX2-NEXT: vpbroadcastq %xmm0, %xmm0
+; AVX2-NEXT: vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: lane_mask_v2i1_i64_zero:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpbroadcastq %rdi, %xmm0
+; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %k0
+; AVX512-NEXT: vpmovm2q %k0, %xmm0
+; AVX512-NEXT: retq
+ %active.lane.mask = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 %TC)
+ ret <2 x i1> %active.lane.mask
+}
+
+define <4 x i1> @lane_mask_v4i1_i64_zero(i64 %TC) {
+; SSE2-LABEL: lane_mask_v4i1_i64_zero:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movq %rdi, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,1,0,1]
-; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT: movdqa %xmm1, %xmm2
-; SSE2-NEXT: pxor %xmm0, %xmm2
-; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [14,15]
-; SSE2-NEXT: paddq %xmm1, %xmm3
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pxor %xmm0, %xmm4
-; SSE2-NEXT: movdqa %xmm2, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm4, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm3, %xmm5
-; SSE2-NEXT: por %xmm4, %xmm5
-; SSE2-NEXT: pxor %xmm0, %xmm5
-; SSE2-NEXT: movq %rdx, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,1]
-; SSE2-NEXT: pxor %xmm0, %xmm3
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm5, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [12,13]
-; SSE2-NEXT: paddq %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm2, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm7
-; SSE2-NEXT: por %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm5
-; SSE2-NEXT: packssdw %xmm4, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [10,11]
-; SSE2-NEXT: paddq %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm2, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm7
-; SSE2-NEXT: por %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [8,9]
-; SSE2-NEXT: paddq %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm8
-; SSE2-NEXT: por %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm8, %xmm4
-; SSE2-NEXT: packssdw %xmm6, %xmm4
-; SSE2-NEXT: packssdw %xmm5, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [6,7]
-; SSE2-NEXT: paddq %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm2, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm7
-; SSE2-NEXT: por %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [4,5]
-; SSE2-NEXT: paddq %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm8
-; SSE2-NEXT: por %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm8, %xmm5
-; SSE2-NEXT: packssdw %xmm6, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [2,3]
-; SSE2-NEXT: paddq %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm6, %xmm8
-; SSE2-NEXT: por %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: por %xmm8, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
-; SSE2-NEXT: paddq %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm2, %xmm9
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm10, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm9
-; SSE2-NEXT: por %xmm8, %xmm9
-; SSE2-NEXT: pxor %xmm0, %xmm9
-; SSE2-NEXT: movdqa %xmm3, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm9, %xmm7
-; SSE2-NEXT: packssdw %xmm6, %xmm7
-; SSE2-NEXT: packssdw %xmm5, %xmm7
-; SSE2-NEXT: packsswb %xmm4, %xmm7
-; SSE2-NEXT: pmovmskb %xmm7, %edx
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [30,31]
-; SSE2-NEXT: paddq %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pxor %xmm0, %xmm5
-; SSE2-NEXT: movdqa %xmm2, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm6
-; SSE2-NEXT: por %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm5, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm6, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [28,29]
-; SSE2-NEXT: paddq %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm2, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm7
-; SSE2-NEXT: por %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm5
-; SSE2-NEXT: packssdw %xmm4, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [26,27]
-; SSE2-NEXT: paddq %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm2, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm7
-; SSE2-NEXT: por %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [24,25]
-; SSE2-NEXT: paddq %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm8
-; SSE2-NEXT: por %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm8, %xmm4
-; SSE2-NEXT: packssdw %xmm6, %xmm4
-; SSE2-NEXT: packssdw %xmm5, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [22,23]
-; SSE2-NEXT: paddq %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm2, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm7
-; SSE2-NEXT: por %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [20,21]
-; SSE2-NEXT: paddq %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm8
-; SSE2-NEXT: por %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm8, %xmm5
-; SSE2-NEXT: packssdw %xmm6, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [18,19]
-; SSE2-NEXT: paddq %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm6, %xmm8
-; SSE2-NEXT: por %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: por %xmm8, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [16,17]
-; SSE2-NEXT: paddq %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm2, %xmm9
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm10, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm9
-; SSE2-NEXT: por %xmm8, %xmm9
-; SSE2-NEXT: pxor %xmm0, %xmm9
-; SSE2-NEXT: movdqa %xmm3, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm9, %xmm7
-; SSE2-NEXT: packssdw %xmm6, %xmm7
-; SSE2-NEXT: packssdw %xmm5, %xmm7
-; SSE2-NEXT: packsswb %xmm4, %xmm7
-; SSE2-NEXT: pmovmskb %xmm7, %ecx
-; SSE2-NEXT: shll $16, %ecx
-; SSE2-NEXT: orl %edx, %ecx
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [46,47]
-; SSE2-NEXT: paddq %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pxor %xmm0, %xmm5
-; SSE2-NEXT: movdqa %xmm2, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm6
-; SSE2-NEXT: por %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm5, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm6, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [44,45]
-; SSE2-NEXT: paddq %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm2, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm7
-; SSE2-NEXT: por %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm5
-; SSE2-NEXT: packssdw %xmm4, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [42,43]
-; SSE2-NEXT: paddq %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm2, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm7
-; SSE2-NEXT: por %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [40,41]
-; SSE2-NEXT: paddq %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm8
-; SSE2-NEXT: por %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm8, %xmm4
-; SSE2-NEXT: packssdw %xmm6, %xmm4
-; SSE2-NEXT: packssdw %xmm5, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [38,39]
-; SSE2-NEXT: paddq %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm2, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm7
-; SSE2-NEXT: por %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [36,37]
-; SSE2-NEXT: paddq %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm8
-; SSE2-NEXT: por %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm8, %xmm5
-; SSE2-NEXT: packssdw %xmm6, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [34,35]
-; SSE2-NEXT: paddq %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm6, %xmm8
-; SSE2-NEXT: por %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: por %xmm8, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [32,33]
-; SSE2-NEXT: paddq %xmm1, %xmm7
-; SSE2-NEXT: movdqa %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm2, %xmm9
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm10, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm9
-; SSE2-NEXT: por %xmm8, %xmm9
-; SSE2-NEXT: pxor %xmm0, %xmm9
-; SSE2-NEXT: movdqa %xmm3, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm9, %xmm7
-; SSE2-NEXT: packssdw %xmm6, %xmm7
-; SSE2-NEXT: packssdw %xmm5, %xmm7
-; SSE2-NEXT: packsswb %xmm4, %xmm7
-; SSE2-NEXT: pmovmskb %xmm7, %edx
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [62,63]
-; SSE2-NEXT: paddq %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pxor %xmm0, %xmm5
-; SSE2-NEXT: movdqa %xmm2, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm6
-; SSE2-NEXT: por %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm5, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm6, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [60,61]
-; SSE2-NEXT: paddq %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm2, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm7
-; SSE2-NEXT: por %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm5
-; SSE2-NEXT: packssdw %xmm4, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [58,59]
-; SSE2-NEXT: paddq %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm2, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm7
-; SSE2-NEXT: por %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [56,57]
-; SSE2-NEXT: paddq %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm8
-; SSE2-NEXT: por %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm8, %xmm4
-; SSE2-NEXT: packssdw %xmm6, %xmm4
-; SSE2-NEXT: packssdw %xmm5, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [54,55]
-; SSE2-NEXT: paddq %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
-; SSE2-NEXT: movdqa %xmm2, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm7
-; SSE2-NEXT: por %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm6, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm6
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [52,53]
-; SSE2-NEXT: paddq %xmm1, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm8
-; SSE2-NEXT: por %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm8, %xmm5
-; SSE2-NEXT: packssdw %xmm6, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [50,51]
-; SSE2-NEXT: paddq %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm9, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm6, %xmm8
-; SSE2-NEXT: por %xmm7, %xmm8
-; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm3, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: por %xmm8, %xmm6
-; SSE2-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [48,49]
-; SSE2-NEXT: movdqa %xmm1, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm9, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm1, %xmm7
-; SSE2-NEXT: por %xmm2, %xmm7
-; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm3, %xmm0
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm7[1,1,3,3]
-; SSE2-NEXT: pand %xmm1, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT: por %xmm2, %xmm0
-; SSE2-NEXT: packssdw %xmm6, %xmm0
-; SSE2-NEXT: packssdw %xmm5, %xmm0
-; SSE2-NEXT: packsswb %xmm4, %xmm0
-; SSE2-NEXT: pmovmskb %xmm0, %esi
-; SSE2-NEXT: shll $16, %esi
-; SSE2-NEXT: orl %edx, %esi
-; SSE2-NEXT: shlq $32, %rsi
-; SSE2-NEXT: orq %rcx, %rsi
-; SSE2-NEXT: movq %rsi, (%rdi)
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: lane_mask_v64i1_i64:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: movq %rsi, %xmm0
-; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,1,0,1]
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
-; SSE42-NEXT: movdqa %xmm2, %xmm0
-; SSE42-NEXT: pxor %xmm1, %xmm0
-; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [14,15]
-; SSE42-NEXT: paddq %xmm2, %xmm3
-; SSE42-NEXT: movdqa %xmm3, %xmm4
-; SSE42-NEXT: pxor %xmm1, %xmm4
-; SSE42-NEXT: movdqa %xmm0, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm4, %xmm5
-; SSE42-NEXT: por %xmm3, %xmm5
-; SSE42-NEXT: pxor %xmm1, %xmm5
-; SSE42-NEXT: movq %rdx, %xmm3
-; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,1]
-; SSE42-NEXT: pxor %xmm1, %xmm3
-; SSE42-NEXT: movdqa %xmm3, %xmm4
-; SSE42-NEXT: pcmpgtq %xmm5, %xmm4
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [12,13]
-; SSE42-NEXT: paddq %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT: por %xmm5, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm5
-; SSE42-NEXT: packssdw %xmm4, %xmm5
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [10,11]
-; SSE42-NEXT: paddq %xmm2, %xmm4
-; SSE42-NEXT: movdqa %xmm4, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT: por %xmm4, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [8,9]
-; SSE42-NEXT: paddq %xmm2, %xmm4
-; SSE42-NEXT: movdqa %xmm4, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm4, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm3, %xmm4
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm4
-; SSE42-NEXT: packssdw %xmm6, %xmm4
-; SSE42-NEXT: packssdw %xmm5, %xmm4
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [6,7]
-; SSE42-NEXT: paddq %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT: por %xmm5, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [4,5]
-; SSE42-NEXT: paddq %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm5, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm3, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm5
-; SSE42-NEXT: packssdw %xmm6, %xmm5
-; SSE42-NEXT: movdqa {{.*#+}} xmm6 = [2,3]
-; SSE42-NEXT: paddq %xmm2, %xmm6
-; SSE42-NEXT: movdqa %xmm6, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm6, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm6
-; SSE42-NEXT: movdqa {{.*#+}} xmm7 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
-; SSE42-NEXT: paddq %xmm2, %xmm7
-; SSE42-NEXT: movdqa %xmm7, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm0, %xmm9
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm9
-; SSE42-NEXT: por %xmm7, %xmm9
-; SSE42-NEXT: pxor %xmm1, %xmm9
-; SSE42-NEXT: movdqa %xmm3, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm9, %xmm7
-; SSE42-NEXT: packssdw %xmm6, %xmm7
-; SSE42-NEXT: packssdw %xmm5, %xmm7
-; SSE42-NEXT: packsswb %xmm4, %xmm7
-; SSE42-NEXT: pmovmskb %xmm7, %edx
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [30,31]
-; SSE42-NEXT: paddq %xmm2, %xmm4
-; SSE42-NEXT: movdqa %xmm4, %xmm5
-; SSE42-NEXT: pxor %xmm1, %xmm5
-; SSE42-NEXT: movdqa %xmm0, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm5, %xmm6
-; SSE42-NEXT: por %xmm4, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm3, %xmm4
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm4
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [28,29]
-; SSE42-NEXT: paddq %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT: por %xmm5, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm5
-; SSE42-NEXT: packssdw %xmm4, %xmm5
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [26,27]
-; SSE42-NEXT: paddq %xmm2, %xmm4
-; SSE42-NEXT: movdqa %xmm4, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT: por %xmm4, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [24,25]
-; SSE42-NEXT: paddq %xmm2, %xmm4
-; SSE42-NEXT: movdqa %xmm4, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm4, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm3, %xmm4
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm4
-; SSE42-NEXT: packssdw %xmm6, %xmm4
-; SSE42-NEXT: packssdw %xmm5, %xmm4
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [22,23]
-; SSE42-NEXT: paddq %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT: por %xmm5, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [20,21]
-; SSE42-NEXT: paddq %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm5, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm3, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm5
-; SSE42-NEXT: packssdw %xmm6, %xmm5
-; SSE42-NEXT: movdqa {{.*#+}} xmm6 = [18,19]
-; SSE42-NEXT: paddq %xmm2, %xmm6
-; SSE42-NEXT: movdqa %xmm6, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm6, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm6
-; SSE42-NEXT: movdqa {{.*#+}} xmm7 = [16,17]
-; SSE42-NEXT: paddq %xmm2, %xmm7
-; SSE42-NEXT: movdqa %xmm7, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm0, %xmm9
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm9
-; SSE42-NEXT: por %xmm7, %xmm9
-; SSE42-NEXT: pxor %xmm1, %xmm9
-; SSE42-NEXT: movdqa %xmm3, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm9, %xmm7
-; SSE42-NEXT: packssdw %xmm6, %xmm7
-; SSE42-NEXT: packssdw %xmm5, %xmm7
-; SSE42-NEXT: packsswb %xmm4, %xmm7
-; SSE42-NEXT: pmovmskb %xmm7, %ecx
-; SSE42-NEXT: shll $16, %ecx
-; SSE42-NEXT: orl %edx, %ecx
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [46,47]
-; SSE42-NEXT: paddq %xmm2, %xmm4
-; SSE42-NEXT: movdqa %xmm4, %xmm5
-; SSE42-NEXT: pxor %xmm1, %xmm5
-; SSE42-NEXT: movdqa %xmm0, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm5, %xmm6
-; SSE42-NEXT: por %xmm4, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm3, %xmm4
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm4
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [44,45]
-; SSE42-NEXT: paddq %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT: por %xmm5, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm5
-; SSE42-NEXT: packssdw %xmm4, %xmm5
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [42,43]
-; SSE42-NEXT: paddq %xmm2, %xmm4
-; SSE42-NEXT: movdqa %xmm4, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT: por %xmm4, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [40,41]
-; SSE42-NEXT: paddq %xmm2, %xmm4
-; SSE42-NEXT: movdqa %xmm4, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm4, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm3, %xmm4
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm4
-; SSE42-NEXT: packssdw %xmm6, %xmm4
-; SSE42-NEXT: packssdw %xmm5, %xmm4
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [38,39]
-; SSE42-NEXT: paddq %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT: por %xmm5, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [36,37]
-; SSE42-NEXT: paddq %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm5, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm3, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm5
-; SSE42-NEXT: packssdw %xmm6, %xmm5
-; SSE42-NEXT: movdqa {{.*#+}} xmm6 = [34,35]
-; SSE42-NEXT: paddq %xmm2, %xmm6
-; SSE42-NEXT: movdqa %xmm6, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm6, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm6
-; SSE42-NEXT: movdqa {{.*#+}} xmm7 = [32,33]
-; SSE42-NEXT: paddq %xmm2, %xmm7
-; SSE42-NEXT: movdqa %xmm7, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm0, %xmm9
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm9
-; SSE42-NEXT: por %xmm7, %xmm9
-; SSE42-NEXT: pxor %xmm1, %xmm9
-; SSE42-NEXT: movdqa %xmm3, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm9, %xmm7
-; SSE42-NEXT: packssdw %xmm6, %xmm7
-; SSE42-NEXT: packssdw %xmm5, %xmm7
-; SSE42-NEXT: packsswb %xmm4, %xmm7
-; SSE42-NEXT: pmovmskb %xmm7, %edx
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [62,63]
-; SSE42-NEXT: paddq %xmm2, %xmm4
-; SSE42-NEXT: movdqa %xmm4, %xmm5
-; SSE42-NEXT: pxor %xmm1, %xmm5
-; SSE42-NEXT: movdqa %xmm0, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm5, %xmm6
-; SSE42-NEXT: por %xmm4, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm3, %xmm4
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm4
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [60,61]
-; SSE42-NEXT: paddq %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT: por %xmm5, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm5
-; SSE42-NEXT: packssdw %xmm4, %xmm5
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [58,59]
-; SSE42-NEXT: paddq %xmm2, %xmm4
-; SSE42-NEXT: movdqa %xmm4, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT: por %xmm4, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [56,57]
-; SSE42-NEXT: paddq %xmm2, %xmm4
-; SSE42-NEXT: movdqa %xmm4, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm4, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm3, %xmm4
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm4
-; SSE42-NEXT: packssdw %xmm6, %xmm4
-; SSE42-NEXT: packssdw %xmm5, %xmm4
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [54,55]
-; SSE42-NEXT: paddq %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm6
-; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT: por %xmm5, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [52,53]
-; SSE42-NEXT: paddq %xmm2, %xmm5
-; SSE42-NEXT: movdqa %xmm5, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm5, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm3, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm5
-; SSE42-NEXT: packssdw %xmm6, %xmm5
-; SSE42-NEXT: movdqa {{.*#+}} xmm6 = [50,51]
-; SSE42-NEXT: paddq %xmm2, %xmm6
-; SSE42-NEXT: movdqa %xmm6, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm6, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm6
-; SSE42-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [48,49]
-; SSE42-NEXT: movdqa %xmm2, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm0
-; SSE42-NEXT: por %xmm2, %xmm0
-; SSE42-NEXT: pxor %xmm1, %xmm0
-; SSE42-NEXT: pcmpgtq %xmm0, %xmm3
-; SSE42-NEXT: packssdw %xmm6, %xmm3
-; SSE42-NEXT: packssdw %xmm5, %xmm3
-; SSE42-NEXT: packsswb %xmm4, %xmm3
-; SSE42-NEXT: pmovmskb %xmm3, %esi
-; SSE42-NEXT: shll $16, %esi
-; SSE42-NEXT: orl %edx, %esi
-; SSE42-NEXT: shlq $32, %rsi
-; SSE42-NEXT: orq %rcx, %rsi
-; SSE42-NEXT: movq %rsi, (%rdi)
-; SSE42-NEXT: retq
-;
-; AVX2-LABEL: lane_mask_v64i1_i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: vmovq %rsi, %xmm0
-; AVX2-NEXT: vpbroadcastq %xmm0, %ymm1
-; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm0 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT: vpxor %ymm0, %ymm1, %ymm2
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3 # [28,29,30,31]
-; AVX2-NEXT: vpxor %ymm0, %ymm3, %ymm4
-; AVX2-NEXT: vpcmpgtq %ymm4, %ymm2, %ymm4
-; AVX2-NEXT: vpor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT: vpxor %ymm0, %ymm3, %ymm4
-; AVX2-NEXT: vmovq %rdx, %xmm3
-; AVX2-NEXT: vpbroadcastq %xmm3, %ymm3
-; AVX2-NEXT: vpxor %ymm0, %ymm3, %ymm3
-; AVX2-NEXT: vpcmpgtq %ymm4, %ymm3, %ymm4
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5 # [24,25,26,27]
-; AVX2-NEXT: vpxor %ymm0, %ymm5, %ymm6
-; AVX2-NEXT: vpcmpgtq %ymm6, %ymm2, %ymm6
-; AVX2-NEXT: vpor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT: vpxor %ymm0, %ymm5, %ymm5
-; AVX2-NEXT: vpcmpgtq %ymm5, %ymm3, %ymm5
-; AVX2-NEXT: vpackssdw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5 # [20,21,22,23]
-; AVX2-NEXT: vpxor %ymm0, %ymm5, %ymm6
-; AVX2-NEXT: vpcmpgtq %ymm6, %ymm2, %ymm6
-; AVX2-NEXT: vpor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT: vpxor %ymm0, %ymm5, %ymm5
-; AVX2-NEXT: vpcmpgtq %ymm5, %ymm3, %ymm5
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm6 # [16,17,18,19]
-; AVX2-NEXT: vpxor %ymm0, %ymm6, %ymm7
-; AVX2-NEXT: vpcmpgtq %ymm7, %ymm2, %ymm7
-; AVX2-NEXT: vpor %ymm7, %ymm6, %ymm6
-; AVX2-NEXT: vpxor %ymm0, %ymm6, %ymm6
-; AVX2-NEXT: vpcmpgtq %ymm6, %ymm3, %ymm6
-; AVX2-NEXT: vpackssdw %ymm5, %ymm6, %ymm5
-; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,2,1,3]
-; AVX2-NEXT: vpackssdw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5 # [12,13,14,15]
-; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
-; AVX2-NEXT: vpxor %ymm0, %ymm5, %ymm6
-; AVX2-NEXT: vpcmpgtq %ymm6, %ymm2, %ymm6
-; AVX2-NEXT: vpor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT: vpxor %ymm0, %ymm5, %ymm5
-; AVX2-NEXT: vpcmpgtq %ymm5, %ymm3, %ymm5
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm6 # [8,9,10,11]
-; AVX2-NEXT: vpxor %ymm0, %ymm6, %ymm7
-; AVX2-NEXT: vpcmpgtq %ymm7, %ymm2, %ymm7
-; AVX2-NEXT: vpor %ymm7, %ymm6, %ymm6
-; AVX2-NEXT: vpxor %ymm0, %ymm6, %ymm6
-; AVX2-NEXT: vpcmpgtq %ymm6, %ymm3, %ymm6
-; AVX2-NEXT: vpackssdw %ymm5, %ymm6, %ymm5
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm6 # [4,5,6,7]
-; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,2,1,3]
-; AVX2-NEXT: vpxor %ymm0, %ymm6, %ymm7
-; AVX2-NEXT: vpcmpgtq %ymm7, %ymm2, %ymm7
-; AVX2-NEXT: vpor %ymm7, %ymm6, %ymm6
-; AVX2-NEXT: vpxor %ymm0, %ymm6, %ymm6
-; AVX2-NEXT: vpcmpgtq %ymm6, %ymm3, %ymm6
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm7 # [0,1,2,3]
-; AVX2-NEXT: vpxor %ymm0, %ymm7, %ymm8
-; AVX2-NEXT: vpcmpgtq %ymm8, %ymm2, %ymm8
-; AVX2-NEXT: vpor %ymm7, %ymm8, %ymm7
-; AVX2-NEXT: vpxor %ymm0, %ymm7, %ymm7
-; AVX2-NEXT: vpcmpgtq %ymm7, %ymm3, %ymm7
-; AVX2-NEXT: vpackssdw %ymm6, %ymm7, %ymm6
-; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,2,1,3]
-; AVX2-NEXT: vpackssdw %ymm5, %ymm6, %ymm5
-; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,2,1,3]
-; AVX2-NEXT: vpacksswb %ymm4, %ymm5, %ymm4
-; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
-; AVX2-NEXT: vpsllw $7, %ymm4, %ymm4
-; AVX2-NEXT: vpmovmskb %ymm4, %ecx
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm4 # [60,61,62,63]
-; AVX2-NEXT: vpxor %ymm0, %ymm4, %ymm5
-; AVX2-NEXT: vpcmpgtq %ymm5, %ymm2, %ymm5
-; AVX2-NEXT: vpor %ymm5, %ymm4, %ymm4
-; AVX2-NEXT: vpxor %ymm0, %ymm4, %ymm4
-; AVX2-NEXT: vpcmpgtq %ymm4, %ymm3, %ymm4
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5 # [56,57,58,59]
-; AVX2-NEXT: vpxor %ymm0, %ymm5, %ymm6
-; AVX2-NEXT: vpcmpgtq %ymm6, %ymm2, %ymm6
-; AVX2-NEXT: vpor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT: vpxor %ymm0, %ymm5, %ymm5
-; AVX2-NEXT: vpcmpgtq %ymm5, %ymm3, %ymm5
-; AVX2-NEXT: vpackssdw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5 # [52,53,54,55]
-; AVX2-NEXT: vpxor %ymm0, %ymm5, %ymm6
-; AVX2-NEXT: vpcmpgtq %ymm6, %ymm2, %ymm6
-; AVX2-NEXT: vpor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT: vpxor %ymm0, %ymm5, %ymm5
-; AVX2-NEXT: vpcmpgtq %ymm5, %ymm3, %ymm5
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm6 # [48,49,50,51]
-; AVX2-NEXT: vpxor %ymm0, %ymm6, %ymm7
-; AVX2-NEXT: vpcmpgtq %ymm7, %ymm2, %ymm7
-; AVX2-NEXT: vpor %ymm7, %ymm6, %ymm6
-; AVX2-NEXT: vpxor %ymm0, %ymm6, %ymm6
-; AVX2-NEXT: vpcmpgtq %ymm6, %ymm3, %ymm6
-; AVX2-NEXT: vpackssdw %ymm5, %ymm6, %ymm5
-; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,2,1,3]
-; AVX2-NEXT: vpackssdw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5 # [44,45,46,47]
-; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
-; AVX2-NEXT: vpxor %ymm0, %ymm5, %ymm6
-; AVX2-NEXT: vpcmpgtq %ymm6, %ymm2, %ymm6
-; AVX2-NEXT: vpor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT: vpxor %ymm0, %ymm5, %ymm5
-; AVX2-NEXT: vpcmpgtq %ymm5, %ymm3, %ymm5
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm6 # [40,41,42,43]
-; AVX2-NEXT: vpxor %ymm0, %ymm6, %ymm7
-; AVX2-NEXT: vpcmpgtq %ymm7, %ymm2, %ymm7
-; AVX2-NEXT: vpor %ymm7, %ymm6, %ymm6
-; AVX2-NEXT: vpxor %ymm0, %ymm6, %ymm6
-; AVX2-NEXT: vpcmpgtq %ymm6, %ymm3, %ymm6
-; AVX2-NEXT: vpackssdw %ymm5, %ymm6, %ymm5
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm6 # [36,37,38,39]
-; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,2,1,3]
-; AVX2-NEXT: vpxor %ymm0, %ymm6, %ymm7
-; AVX2-NEXT: vpcmpgtq %ymm7, %ymm2, %ymm7
-; AVX2-NEXT: vpor %ymm7, %ymm6, %ymm6
-; AVX2-NEXT: vpxor %ymm0, %ymm6, %ymm6
-; AVX2-NEXT: vpcmpgtq %ymm6, %ymm3, %ymm6
-; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [32,33,34,35]
-; AVX2-NEXT: vpxor %ymm0, %ymm1, %ymm7
-; AVX2-NEXT: vpcmpgtq %ymm7, %ymm2, %ymm2
-; AVX2-NEXT: vpor %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpxor %ymm0, %ymm1, %ymm0
-; AVX2-NEXT: vpcmpgtq %ymm0, %ymm3, %ymm0
-; AVX2-NEXT: vpackssdw %ymm6, %ymm0, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT: vpackssdw %ymm5, %ymm0, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT: vpacksswb %ymm4, %ymm0, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT: vpsllw $7, %ymm0, %ymm0
-; AVX2-NEXT: vpmovmskb %ymm0, %edx
-; AVX2-NEXT: shlq $32, %rdx
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: movq %rdx, (%rdi)
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: lane_mask_v64i1_i64:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq %rsi, %zmm0
-; AVX512-NEXT: vpbroadcastq %rdi, %zmm1
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [0,1,2,3,4,5,6,7]
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm3
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm2, %k0
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm2 # [8,9,10,11,12,13,14,15]
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm2, %k1
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT: kunpckbw %k0, %k1, %k0
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [16,17,18,19,20,21,22,23]
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm2, %k1
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [24,25,26,27,28,29,30,31]
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm2, %k2
-; AVX512-NEXT: kunpckbw %k1, %k2, %k1
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT: kunpckwd %k0, %k1, %k0
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [32,33,34,35,36,37,38,39]
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm2, %k1
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [40,41,42,43,44,45,46,47]
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm2, %k2
-; AVX512-NEXT: kunpckbw %k1, %k2, %k1
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [48,49,50,51,52,53,54,55]
-; AVX512-NEXT: vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [56,57,58,59,60,61,62,63]
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm2, %k2
-; AVX512-NEXT: vpcmpltuq %zmm0, %zmm1, %k3
-; AVX512-NEXT: kunpckbw %k2, %k3, %k2
-; AVX512-NEXT: kunpckwd %k1, %k2, %k1
-; AVX512-NEXT: kunpckdq %k0, %k1, %k0
-; AVX512-NEXT: vpmovm2b %k0, %zmm0
-; AVX512-NEXT: retq
- %active.lane.mask = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 %index, i64 %TC)
- ret <64 x i1> %active.lane.mask
-}
-
-;
-; i64 (zero index)
-;
-
-define <2 x i1> @lane_mask_v2i1_i64_zero(i64 %TC) {
-; SSE2-LABEL: lane_mask_v2i1_i64_zero:
-; SSE2: # %bb.0:
-; SSE2-NEXT: movq %rdi, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
-; SSE2-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT: por %xmm2, %xmm0
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: lane_mask_v2i1_i64_zero:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movq %rdi, %xmm0
-; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE42-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: retq
-;
-; AVX2-LABEL: lane_mask_v2i1_i64_zero:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vmovq %rdi, %xmm0
-; AVX2-NEXT: vpbroadcastq %xmm0, %xmm0
-; AVX2-NEXT: vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: lane_mask_v2i1_i64_zero:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq %rdi, %xmm0
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %k0
-; AVX512-NEXT: vpmovm2q %k0, %xmm0
-; AVX512-NEXT: retq
- %active.lane.mask = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 %TC)
- ret <2 x i1> %active.lane.mask
-}
-
-define <4 x i1> @lane_mask_v4i1_i64_zero(i64 %TC) {
-; SSE2-LABEL: lane_mask_v4i1_i64_zero:
-; SSE2: # %bb.0:
-; SSE2-NEXT: movq %rdi, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,1,0,1]
-; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; SSE2-NEXT: movdqa %xmm1, %xmm2
; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,3,1,3]
@@ -4520,223 +2701,10 @@ define <16 x i1> @lane_mask_v16i1_i64_zero(i64 %TC) {
; SSE2-NEXT: retq
;
; SSE42-LABEL: lane_mask_v16i1_i64_zero:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movq %rdi, %xmm0
-; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE42-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: movdqa %xmm0, %xmm1
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT: packssdw %xmm1, %xmm2
-; SSE42-NEXT: movdqa %xmm0, %xmm1
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT: packssdw %xmm1, %xmm3
-; SSE42-NEXT: packssdw %xmm2, %xmm3
-; SSE42-NEXT: movdqa %xmm0, %xmm1
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT: packssdw %xmm1, %xmm2
-; SSE42-NEXT: movdqa %xmm0, %xmm1
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: packssdw %xmm1, %xmm0
-; SSE42-NEXT: packssdw %xmm2, %xmm0
-; SSE42-NEXT: packsswb %xmm3, %xmm0
-; SSE42-NEXT: retq
-;
-; AVX2-LABEL: lane_mask_v16i1_i64_zero:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vmovq %rdi, %xmm0
-; AVX2-NEXT: vpbroadcastq %xmm0, %ymm0
-; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT: vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2-NEXT: vpackssdw %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT: vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpacksswb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: lane_mask_v16i1_i64_zero:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq %rdi, %zmm0
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT: kunpckbw %k0, %k1, %k0
-; AVX512-NEXT: vpmovm2b %k0, %xmm0
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: retq
- %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 %TC)
- ret <16 x i1> %active.lane.mask
-}
-
-define <32 x i1> @lane_mask_v32i1_i64_zero(i64 %TC) {
-; SSE2-LABEL: lane_mask_v32i1_i64_zero:
-; SSE2: # %bb.0:
-; SSE2-NEXT: movq %rsi, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE2-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT: por %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm3
-; SSE2-NEXT: packssdw %xmm2, %xmm3
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm5
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm2
-; SSE2-NEXT: packssdw %xmm5, %xmm2
-; SSE2-NEXT: packssdw %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm3
-; SSE2-NEXT: movdqa %xmm0, %xmm4
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm4
-; SSE2-NEXT: packssdw %xmm3, %xmm4
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT: movdqa %xmm0, %xmm6
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6
-; SSE2-NEXT: por %xmm5, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm6
-; SSE2-NEXT: packssdw %xmm3, %xmm6
-; SSE2-NEXT: packssdw %xmm4, %xmm6
-; SSE2-NEXT: packsswb %xmm2, %xmm6
-; SSE2-NEXT: pmovmskb %xmm6, %ecx
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT: por %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm3
-; SSE2-NEXT: packssdw %xmm2, %xmm3
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm5
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm2
-; SSE2-NEXT: packssdw %xmm5, %xmm2
-; SSE2-NEXT: packssdw %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm3
-; SSE2-NEXT: movdqa %xmm0, %xmm4
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm4
-; SSE2-NEXT: packssdw %xmm3, %xmm4
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm0
-; SSE2-NEXT: packssdw %xmm3, %xmm0
-; SSE2-NEXT: packssdw %xmm4, %xmm0
-; SSE2-NEXT: packsswb %xmm2, %xmm0
-; SSE2-NEXT: pmovmskb %xmm0, %edx
-; SSE2-NEXT: shll $16, %edx
-; SSE2-NEXT: orl %ecx, %edx
-; SSE2-NEXT: movl %edx, (%rdi)
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: lane_mask_v32i1_i64_zero:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movq %rsi, %xmm0
-; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE42-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: movdqa %xmm0, %xmm1
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT: movdqa %xmm0, %xmm4
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: movdqa %xmm0, %xmm5
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE42-NEXT: packssdw %xmm1, %xmm2
-; SSE42-NEXT: movdqa %xmm0, %xmm1
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT: packssdw %xmm3, %xmm4
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT: packssdw %xmm2, %xmm4
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT: packssdw %xmm5, %xmm1
-; SSE42-NEXT: packssdw %xmm3, %xmm2
-; SSE42-NEXT: packssdw %xmm1, %xmm2
-; SSE42-NEXT: packsswb %xmm4, %xmm2
-; SSE42-NEXT: pmovmskb %xmm2, %ecx
+; SSE42: # %bb.0:
+; SSE42-NEXT: movq %rdi, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; SSE42-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE42-NEXT: movdqa %xmm0, %xmm1
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; SSE42-NEXT: movdqa %xmm0, %xmm2
@@ -4759,13 +2727,9 @@ define <32 x i1> @lane_mask_v32i1_i64_zero(i64 %TC) {
; SSE42-NEXT: packssdw %xmm1, %xmm0
; SSE42-NEXT: packssdw %xmm2, %xmm0
; SSE42-NEXT: packsswb %xmm3, %xmm0
-; SSE42-NEXT: pmovmskb %xmm0, %edx
-; SSE42-NEXT: shll $16, %edx
-; SSE42-NEXT: orl %ecx, %edx
-; SSE42-NEXT: movl %edx, (%rdi)
; SSE42-NEXT: retq
;
-; AVX2-LABEL: lane_mask_v32i1_i64_zero:
+; AVX2-LABEL: lane_mask_v16i1_i64_zero:
; AVX2: # %bb.0:
; AVX2-NEXT: vmovq %rdi, %xmm0
; AVX2-NEXT: vpbroadcastq %xmm0, %ymm0
@@ -4776,43 +2740,32 @@ define <32 x i1> @lane_mask_v32i1_i64_zero(i64 %TC) {
; AVX2-NEXT: vpackssdw %ymm1, %ymm2, %ymm1
; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
-; AVX2-NEXT: vpackssdw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT: vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
-; AVX2-NEXT: vpackssdw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2-NEXT: vpackssdw %ymm3, %ymm0, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT: vpackssdw %ymm2, %ymm0, %ymm0
; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT: vpacksswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpackssdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512-LABEL: lane_mask_v32i1_i64_zero:
+; AVX512-LABEL: lane_mask_v16i1_i64_zero:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq %rdi, %zmm0
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k2
-; AVX512-NEXT: kunpckbw %k0, %k1, %k0
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT: kunpckbw %k2, %k1, %k1
-; AVX512-NEXT: kunpckwd %k0, %k1, %k0
-; AVX512-NEXT: vpmovm2b %k0, %ymm0
+; AVX512-NEXT: cmpq $16, %rdi
+; AVX512-NEXT: movl $16, %eax
+; AVX512-NEXT: cmovbq %rdi, %rax
+; AVX512-NEXT: movl $-1, %ecx
+; AVX512-NEXT: bzhil %eax, %ecx, %eax
+; AVX512-NEXT: kmovd %eax, %k0
+; AVX512-NEXT: vpmovm2b %k0, %xmm0
; AVX512-NEXT: retq
- %active.lane.mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 %TC)
- ret <32 x i1> %active.lane.mask
+ %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 %TC)
+ ret <16 x i1> %active.lane.mask
}
-define <64 x i1> @lane_mask_v64i1_i64_zero(i64 %TC) {
-; SSE2-LABEL: lane_mask_v64i1_i64_zero:
+define <32 x i1> @lane_mask_v32i1_i64_zero(i64 %TC) {
+; SSE2-LABEL: lane_mask_v32i1_i64_zero:
; SSE2: # %bb.0:
; SSE2-NEXT: movq %rsi, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
@@ -4875,7 +2828,7 @@ define <64 x i1> @lane_mask_v64i1_i64_zero(i64 %TC) {
; SSE2-NEXT: packssdw %xmm3, %xmm6
; SSE2-NEXT: packssdw %xmm4, %xmm6
; SSE2-NEXT: packsswb %xmm2, %xmm6
-; SSE2-NEXT: pmovmskb %xmm6, %edx
+; SSE2-NEXT: pmovmskb %xmm6, %ecx
; SSE2-NEXT: movdqa %xmm0, %xmm2
; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
@@ -4921,120 +2874,6 @@ define <64 x i1> @lane_mask_v64i1_i64_zero(i64 %TC) {
; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
; SSE2-NEXT: pand %xmm1, %xmm5
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT: movdqa %xmm0, %xmm6
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6
-; SSE2-NEXT: por %xmm5, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm6
-; SSE2-NEXT: packssdw %xmm3, %xmm6
-; SSE2-NEXT: packssdw %xmm4, %xmm6
-; SSE2-NEXT: packsswb %xmm2, %xmm6
-; SSE2-NEXT: pmovmskb %xmm6, %ecx
-; SSE2-NEXT: shll $16, %ecx
-; SSE2-NEXT: orl %edx, %ecx
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT: movdqa %xmm0, %xmm4
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT: por %xmm3, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm3, %xmm4
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: packssdw %xmm2, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT: por %xmm2, %xmm3
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm2
-; SSE2-NEXT: packssdw %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: packssdw %xmm4, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm3
-; SSE2-NEXT: movdqa %xmm0, %xmm4
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm4
-; SSE2-NEXT: packssdw %xmm3, %xmm4
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm3
-; SSE2-NEXT: movdqa %xmm0, %xmm5
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT: por %xmm6, %xmm5
-; SSE2-NEXT: packssdw %xmm3, %xmm5
-; SSE2-NEXT: packssdw %xmm4, %xmm5
-; SSE2-NEXT: packsswb %xmm2, %xmm5
-; SSE2-NEXT: pmovmskb %xmm5, %edx
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT: movdqa %xmm0, %xmm4
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT: por %xmm3, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm3, %xmm4
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: packssdw %xmm2, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT: por %xmm2, %xmm3
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm2
-; SSE2-NEXT: packssdw %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: packssdw %xmm4, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm3
-; SSE2-NEXT: movdqa %xmm0, %xmm4
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm5, %xmm4
-; SSE2-NEXT: packssdw %xmm3, %xmm4
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT: pand %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
; SSE2-NEXT: por %xmm5, %xmm3
; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
@@ -5044,203 +2883,147 @@ define <64 x i1> @lane_mask_v64i1_i64_zero(i64 %TC) {
; SSE2-NEXT: packssdw %xmm3, %xmm0
; SSE2-NEXT: packssdw %xmm4, %xmm0
; SSE2-NEXT: packsswb %xmm2, %xmm0
-; SSE2-NEXT: pmovmskb %xmm0, %esi
-; SSE2-NEXT: shll $16, %esi
-; SSE2-NEXT: orl %edx, %esi
-; SSE2-NEXT: shlq $32, %rsi
-; SSE2-NEXT: orq %rcx, %rsi
-; SSE2-NEXT: movq %rsi, (%rdi)
+; SSE2-NEXT: pmovmskb %xmm0, %edx
+; SSE2-NEXT: shll $16, %edx
+; SSE2-NEXT: orl %ecx, %edx
+; SSE2-NEXT: movl %edx, (%rdi)
; SSE2-NEXT: retq
;
-; SSE42-LABEL: lane_mask_v64i1_i64_zero:
+; SSE42-LABEL: lane_mask_v32i1_i64_zero:
; SSE42: # %bb.0:
; SSE42-NEXT: movq %rsi, %xmm0
; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
; SSE42-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE42-NEXT: movdqa %xmm0, %xmm1
+; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; SSE42-NEXT: movdqa %xmm0, %xmm2
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE42-NEXT: movdqa %xmm0, %xmm3
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
; SSE42-NEXT: movdqa %xmm0, %xmm4
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE42-NEXT: movdqa %xmm0, %xmm1
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; SSE42-NEXT: movq %rdi, %rax
; SSE42-NEXT: movdqa %xmm0, %xmm5
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE42-NEXT: packssdw %xmm2, %xmm3
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT: packssdw %xmm4, %xmm1
-; SSE42-NEXT: movdqa %xmm0, %xmm4
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE42-NEXT: packssdw %xmm3, %xmm1
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT: packssdw %xmm5, %xmm2
-; SSE42-NEXT: movdqa %xmm0, %xmm5
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE42-NEXT: packssdw %xmm4, %xmm3
-; SSE42-NEXT: movdqa %xmm0, %xmm4
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE42-NEXT: packssdw %xmm2, %xmm3
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT: packsswb %xmm1, %xmm3
+; SSE42-NEXT: packssdw %xmm1, %xmm2
; SSE42-NEXT: movdqa %xmm0, %xmm1
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT: pmovmskb %xmm3, %edx
+; SSE42-NEXT: packssdw %xmm3, %xmm4
; SSE42-NEXT: movdqa %xmm0, %xmm3
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT: packssdw %xmm5, %xmm4
-; SSE42-NEXT: movdqa %xmm0, %xmm5
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE42-NEXT: packssdw %xmm2, %xmm1
+; SSE42-NEXT: packssdw %xmm2, %xmm4
; SSE42-NEXT: movdqa %xmm0, %xmm2
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT: packssdw %xmm4, %xmm1
-; SSE42-NEXT: movdqa %xmm0, %xmm4
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE42-NEXT: packssdw %xmm3, %xmm5
-; SSE42-NEXT: packssdw %xmm2, %xmm4
-; SSE42-NEXT: packssdw %xmm5, %xmm4
-; SSE42-NEXT: packsswb %xmm1, %xmm4
-; SSE42-NEXT: pmovmskb %xmm4, %ecx
+; SSE42-NEXT: packssdw %xmm5, %xmm1
+; SSE42-NEXT: packssdw %xmm3, %xmm2
+; SSE42-NEXT: packssdw %xmm1, %xmm2
+; SSE42-NEXT: packsswb %xmm4, %xmm2
+; SSE42-NEXT: pmovmskb %xmm2, %ecx
; SSE42-NEXT: movdqa %xmm0, %xmm1
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT: shll $16, %ecx
; SSE42-NEXT: movdqa %xmm0, %xmm2
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT: orl %edx, %ecx
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
; SSE42-NEXT: packssdw %xmm1, %xmm2
; SSE42-NEXT: movdqa %xmm0, %xmm1
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT: packssdw %xmm3, %xmm1
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT: packssdw %xmm2, %xmm1
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT: packssdw %xmm3, %xmm2
; SSE42-NEXT: movdqa %xmm0, %xmm3
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT: movdqa %xmm0, %xmm4
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE42-NEXT: packssdw %xmm3, %xmm4
-; SSE42-NEXT: packssdw %xmm2, %xmm4
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT: packsswb %xmm1, %xmm4
+; SSE42-NEXT: packssdw %xmm1, %xmm3
+; SSE42-NEXT: packssdw %xmm2, %xmm3
; SSE42-NEXT: movdqa %xmm0, %xmm1
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT: pmovmskb %xmm4, %edx
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT: packssdw %xmm2, %xmm1
; SSE42-NEXT: movdqa %xmm0, %xmm2
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT: packssdw %xmm3, %xmm2
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
; SSE42-NEXT: packssdw %xmm1, %xmm2
; SSE42-NEXT: movdqa %xmm0, %xmm1
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT: packssdw %xmm3, %xmm1
-; SSE42-NEXT: movdqa %xmm0, %xmm3
-; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
; SSE42-NEXT: pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT: packssdw %xmm3, %xmm0
; SSE42-NEXT: packssdw %xmm1, %xmm0
-; SSE42-NEXT: packsswb %xmm2, %xmm0
-; SSE42-NEXT: pmovmskb %xmm0, %esi
-; SSE42-NEXT: shll $16, %esi
-; SSE42-NEXT: orl %edx, %esi
-; SSE42-NEXT: shlq $32, %rsi
-; SSE42-NEXT: orq %rcx, %rsi
-; SSE42-NEXT: movq %rsi, (%rdi)
+; SSE42-NEXT: packssdw %xmm2, %xmm0
+; SSE42-NEXT: packsswb %xmm3, %xmm0
+; SSE42-NEXT: pmovmskb %xmm0, %edx
+; SSE42-NEXT: shll $16, %edx
+; SSE42-NEXT: orl %ecx, %edx
+; SSE42-NEXT: movl %edx, (%rdi)
; SSE42-NEXT: retq
;
-; AVX2-LABEL: lane_mask_v64i1_i64_zero:
+; AVX2-LABEL: lane_mask_v32i1_i64_zero:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovq %rsi, %xmm0
+; AVX2-NEXT: vmovq %rdi, %xmm0
; AVX2-NEXT: vpbroadcastq %xmm0, %ymm0
; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1
; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
-; AVX2-NEXT: vpackssdw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT: vpackssdw %ymm1, %ymm2, %ymm1
; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
; AVX2-NEXT: vpackssdw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm4
-; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT: vpackssdw %ymm3, %ymm4, %ymm3
-; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
-; AVX2-NEXT: vpackssdw %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT: vpacksswb %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpackssdw %ymm1, %ymm2, %ymm1
; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT: vpsllw $7, %ymm1, %ymm1
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
-; AVX2-NEXT: vpmovmskb %ymm1, %ecx
-; AVX2-NEXT: vpackssdw %ymm2, %ymm3, %ymm1
; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
; AVX2-NEXT: vpackssdw %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT: vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
-; AVX2-NEXT: vpackssdw %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
; AVX2-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
; AVX2-NEXT: vpackssdw %ymm3, %ymm0, %ymm0
; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT: vpackssdw %ymm2, %ymm0, %ymm0
; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT: vpacksswb %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT: vpsllw $7, %ymm0, %ymm0
-; AVX2-NEXT: vpmovmskb %ymm0, %edx
-; AVX2-NEXT: shlq $32, %rdx
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: movq %rdx, (%rdi)
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: lane_mask_v32i1_i64_zero:
+; AVX512: # %bb.0:
+; AVX512-NEXT: cmpq $32, %rdi
+; AVX512-NEXT: movl $32, %eax
+; AVX512-NEXT: cmovbq %rdi, %rax
+; AVX512-NEXT: movl $-1, %ecx
+; AVX512-NEXT: bzhil %eax, %ecx, %eax
+; AVX512-NEXT: kmovd %eax, %k0
+; AVX512-NEXT: vpmovm2b %k0, %ymm0
+; AVX512-NEXT: retq
+ %active.lane.mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 %TC)
+ ret <32 x i1> %active.lane.mask
+}
+
+define <64 x i1> @lane_mask_v64i1_i64_zero(i64 %TC) {
+; SSE-LABEL: lane_mask_v64i1_i64_zero:
+; SSE: # %bb.0:
+; SSE-NEXT: cmpq $64, %rsi
+; SSE-NEXT: movl $64, %ecx
+; SSE-NEXT: cmovbq %rsi, %rcx
+; SSE-NEXT: movl $1, %edx
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shlq %cl, %rdx
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: decq %rdx
+; SSE-NEXT: movq %rdx, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: lane_mask_v64i1_i64_zero:
+; AVX2: # %bb.0:
+; AVX2-NEXT: cmpq $64, %rsi
+; AVX2-NEXT: movl $64, %ecx
+; AVX2-NEXT: cmovbq %rsi, %rcx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq $-1, %rdx
+; AVX2-NEXT: bzhiq %rcx, %rdx, %rcx
+; AVX2-NEXT: movq %rcx, (%rdi)
; AVX2-NEXT: retq
;
; AVX512-LABEL: lane_mask_v64i1_i64_zero:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq %rdi, %zmm0
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k2
-; AVX512-NEXT: kunpckbw %k0, %k1, %k0
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT: kunpckbw %k2, %k1, %k1
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k2
-; AVX512-NEXT: kunpckwd %k0, %k1, %k0
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT: kunpckbw %k2, %k1, %k1
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k2
-; AVX512-NEXT: vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k3
-; AVX512-NEXT: kunpckbw %k2, %k3, %k2
-; AVX512-NEXT: kunpckwd %k1, %k2, %k1
-; AVX512-NEXT: kunpckdq %k0, %k1, %k0
+; AVX512-NEXT: cmpq $64, %rdi
+; AVX512-NEXT: movl $64, %eax
+; AVX512-NEXT: cmovbq %rdi, %rax
+; AVX512-NEXT: movq $-1, %rcx
+; AVX512-NEXT: bzhiq %rax, %rcx, %rax
+; AVX512-NEXT: kmovq %rax, %k0
; AVX512-NEXT: vpmovm2b %k0, %zmm0
; AVX512-NEXT: retq
%active.lane.mask = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 0, i64 %TC)
diff --git a/llvm/test/CodeGen/X86/pr47299.ll b/llvm/test/CodeGen/X86/pr47299.ll
index 7cb1112402ebe5..1e502fd52268b5 100644
--- a/llvm/test/CodeGen/X86/pr47299.ll
+++ b/llvm/test/CodeGen/X86/pr47299.ll
@@ -56,12 +56,13 @@ define <7 x i1> @create_mask7(i64 %0) {
define <16 x i1> @create_mask16(i64 %0) {
; CHECK-LABEL: create_mask16:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpbroadcastq zmm0, rdi
-; CHECK-NEXT: vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: kunpckbw k0, k1, k0
+; CHECK-NEXT: cmp rdi, 16
+; CHECK-NEXT: mov eax, 16
+; CHECK-NEXT: cmovb rax, rdi
+; CHECK-NEXT: mov ecx, -1
+; CHECK-NEXT: bzhi eax, ecx, eax
+; CHECK-NEXT: kmovd k0, eax
; CHECK-NEXT: vpmovm2b xmm0, k0
-; CHECK-NEXT: vzeroupper
; CHECK-NEXT: ret
%2 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 %0)
ret <16 x i1> %2
@@ -70,14 +71,12 @@ define <16 x i1> @create_mask16(i64 %0) {
define <32 x i1> @create_mask32(i64 %0) {
; CHECK-LABEL: create_mask32:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpbroadcastq zmm0, rdi
-; CHECK-NEXT: vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: kunpckbw k0, k1, k0
-; CHECK-NEXT: vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: kunpckbw k1, k1, k2
-; CHECK-NEXT: kunpckwd k0, k1, k0
+; CHECK-NEXT: cmp rdi, 32
+; CHECK-NEXT: mov eax, 32
+; CHECK-NEXT: cmovb rax, rdi
+; CHECK-NEXT: mov ecx, -1
+; CHECK-NEXT: bzhi eax, ecx, eax
+; CHECK-NEXT: kmovd k0, eax
; CHECK-NEXT: vpmovm2b ymm0, k0
; CHECK-NEXT: ret
%2 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 %0)
@@ -87,22 +86,12 @@ define <32 x i1> @create_mask32(i64 %0) {
define <64 x i1> @create_mask64(i64 %0) {
; CHECK-LABEL: create_mask64:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpbroadcastq zmm0, rdi
-; CHECK-NEXT: vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: kunpckbw k0, k1, k0
-; CHECK-NEXT: vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: kunpckbw k1, k1, k2
-; CHECK-NEXT: vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: kunpckwd k0, k1, k0
-; CHECK-NEXT: vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: kunpckbw k1, k1, k2
-; CHECK-NEXT: vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: vpcmpnleuq k3, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: kunpckbw k2, k3, k2
-; CHECK-NEXT: kunpckwd k1, k2, k1
-; CHECK-NEXT: kunpckdq k0, k1, k0
+; CHECK-NEXT: cmp rdi, 64
+; CHECK-NEXT: mov eax, 64
+; CHECK-NEXT: cmovb rax, rdi
+; CHECK-NEXT: mov rcx, -1
+; CHECK-NEXT: bzhi rax, rcx, rax
+; CHECK-NEXT: kmovq k0, rax
; CHECK-NEXT: vpmovm2b zmm0, k0
; CHECK-NEXT: ret
%2 = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 0, i64 %0)
@@ -112,10 +101,13 @@ define <64 x i1> @create_mask64(i64 %0) {
define <16 x i1> @create_mask16_i32(i32 %0) {
; CHECK-LABEL: create_mask16_i32:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpbroadcastd zmm0, edi
-; CHECK-NEXT: vpcmpnleud k0, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
+; CHECK-NEXT: cmp edi, 16
+; CHECK-NEXT: mov eax, 16
+; CHECK-NEXT: cmovb eax, edi
+; CHECK-NEXT: mov ecx, -1
+; CHECK-NEXT: bzhi eax, ecx, eax
+; CHECK-NEXT: kmovd k0, eax
; CHECK-NEXT: vpmovm2b xmm0, k0
-; CHECK-NEXT: vzeroupper
; CHECK-NEXT: ret
%2 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 0, i32 %0)
ret <16 x i1> %2
@@ -124,14 +116,12 @@ define <16 x i1> @create_mask16_i32(i32 %0) {
define <64 x i1> @create_mask64_i32(i32 %0) {
; CHECK-LABEL: create_mask64_i32:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpbroadcastd zmm0, edi
-; CHECK-NEXT: vpcmpnleud k0, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: vpcmpnleud k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: vpcmpnleud k2, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: kunpckwd k0, k1, k0
-; CHECK-NEXT: vpcmpnleud k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT: kunpckwd k1, k1, k2
-; CHECK-NEXT: kunpckdq k0, k1, k0
+; CHECK-NEXT: cmp edi, 64
+; CHECK-NEXT: mov eax, 64
+; CHECK-NEXT: cmovb eax, edi
+; CHECK-NEXT: mov rcx, -1
+; CHECK-NEXT: bzhi rax, rcx, rax
+; CHECK-NEXT: kmovq k0, rax
; CHECK-NEXT: vpmovm2b zmm0, k0
; CHECK-NEXT: ret
%2 = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 0, i32 %0)
>From 2c7897d59ccc46133c91f6a1137d80541ecf0b54 Mon Sep 17 00:00:00 2001
From: william <we3223 at gmail.com>
Date: Wed, 30 Sep 2026 23:54:13 +0800
Subject: [PATCH 2/5] [X86][test] Add common SSE check prefix in
active_lane_mask.ll-1
---
llvm/test/CodeGen/X86/active_lane_mask.ll | 376 +++++++++++-----------
1 file changed, 188 insertions(+), 188 deletions(-)
diff --git a/llvm/test/CodeGen/X86/active_lane_mask.ll b/llvm/test/CodeGen/X86/active_lane_mask.ll
index 74a37c8ba6b4a8..d316db74246d9e 100644
--- a/llvm/test/CodeGen/X86/active_lane_mask.ll
+++ b/llvm/test/CodeGen/X86/active_lane_mask.ll
@@ -272,34 +272,34 @@ define <16 x i1> @lane_mask_v16i1_i32(i32 %index, i32 %TC) {
; SSE42-LABEL: lane_mask_v16i1_i32:
; SSE42: # %bb.0:
; SSE42-NEXT: movd %edi, %xmm0
-; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [4294967295,4294967294,4294967293,4294967292]
-; SSE42-NEXT: pminud %xmm1, %xmm0
+; SSE42-NEXT: pminud %xmm2, %xmm0
; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [0,1,2,3]
-; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [4294967291,4294967290,4294967289,4294967288]
-; SSE42-NEXT: pminud %xmm1, %xmm2
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [4,5,6,7]
+; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [4294967291,4294967290,4294967289,4294967288]
+; SSE42-NEXT: pminud %xmm2, %xmm1
+; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [4,5,6,7]
; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [4294967287,4294967286,4294967285,4294967284]
-; SSE42-NEXT: pminud %xmm1, %xmm3
+; SSE42-NEXT: pminud %xmm2, %xmm3
; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [8,9,10,11]
; SSE42-NEXT: movd %esi, %xmm4
-; SSE42-NEXT: pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE42-NEXT: pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE42-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [12,13,14,15]
-; SSE42-NEXT: movdqa %xmm1, %xmm5
+; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [12,13,14,15]
+; SSE42-NEXT: movdqa %xmm2, %xmm5
; SSE42-NEXT: pmaxud %xmm4, %xmm5
-; SSE42-NEXT: pcmpeqd %xmm1, %xmm5
-; SSE42-NEXT: movdqa %xmm3, %xmm1
-; SSE42-NEXT: pmaxud %xmm4, %xmm1
-; SSE42-NEXT: pcmpeqd %xmm3, %xmm1
-; SSE42-NEXT: packssdw %xmm5, %xmm1
-; SSE42-NEXT: movdqa %xmm2, %xmm3
+; SSE42-NEXT: pcmpeqd %xmm2, %xmm5
+; SSE42-NEXT: movdqa %xmm3, %xmm2
+; SSE42-NEXT: pmaxud %xmm4, %xmm2
+; SSE42-NEXT: pcmpeqd %xmm3, %xmm2
+; SSE42-NEXT: packssdw %xmm5, %xmm2
+; SSE42-NEXT: movdqa %xmm1, %xmm3
; SSE42-NEXT: pmaxud %xmm4, %xmm3
-; SSE42-NEXT: pcmpeqd %xmm2, %xmm3
+; SSE42-NEXT: pcmpeqd %xmm1, %xmm3
; SSE42-NEXT: pmaxud %xmm0, %xmm4
; SSE42-NEXT: pcmpeqd %xmm0, %xmm4
; SSE42-NEXT: packssdw %xmm3, %xmm4
-; SSE42-NEXT: packsswb %xmm1, %xmm4
+; SSE42-NEXT: packsswb %xmm2, %xmm4
; SSE42-NEXT: pcmpeqd %xmm0, %xmm0
; SSE42-NEXT: pxor %xmm4, %xmm0
; SSE42-NEXT: retq
@@ -408,20 +408,9 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
; SSE2-NEXT: pcmpgtd %xmm5, %xmm6
; SSE2-NEXT: por %xmm4, %xmm6
; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm4
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [24,25,26,27]
-; SSE2-NEXT: paddd %xmm2, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm1, %xmm6
-; SSE2-NEXT: movdqa %xmm0, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT: por %xmm5, %xmm7
-; SSE2-NEXT: pxor %xmm1, %xmm7
; SSE2-NEXT: movdqa %xmm3, %xmm5
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT: packssdw %xmm4, %xmm5
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [20,21,22,23]
+; SSE2-NEXT: pcmpgtd %xmm6, %xmm5
+; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [24,25,26,27]
; SSE2-NEXT: paddd %xmm2, %xmm4
; SSE2-NEXT: movdqa %xmm4, %xmm6
; SSE2-NEXT: pxor %xmm1, %xmm6
@@ -431,6 +420,17 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
; SSE2-NEXT: pxor %xmm1, %xmm7
; SSE2-NEXT: movdqa %xmm3, %xmm4
; SSE2-NEXT: pcmpgtd %xmm7, %xmm4
+; SSE2-NEXT: packssdw %xmm5, %xmm4
+; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [20,21,22,23]
+; SSE2-NEXT: paddd %xmm2, %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: pxor %xmm1, %xmm6
+; SSE2-NEXT: movdqa %xmm0, %xmm7
+; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
+; SSE2-NEXT: por %xmm5, %xmm7
+; SSE2-NEXT: pxor %xmm1, %xmm7
+; SSE2-NEXT: movdqa %xmm3, %xmm5
+; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [16,17,18,19]
; SSE2-NEXT: movdqa %xmm2, %xmm6
; SSE2-NEXT: pxor %xmm1, %xmm6
@@ -438,8 +438,8 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
; SSE2-NEXT: por %xmm2, %xmm0
; SSE2-NEXT: pxor %xmm1, %xmm0
; SSE2-NEXT: pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT: packssdw %xmm4, %xmm3
-; SSE2-NEXT: packsswb %xmm5, %xmm3
+; SSE2-NEXT: packssdw %xmm5, %xmm3
+; SSE2-NEXT: packsswb %xmm4, %xmm3
; SSE2-NEXT: pmovmskb %xmm3, %edx
; SSE2-NEXT: shll $16, %edx
; SSE2-NEXT: orl %ecx, %edx
@@ -459,28 +459,28 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
; SSE42-NEXT: movdqa %xmm2, %xmm3
; SSE42-NEXT: pmaxud %xmm0, %xmm3
; SSE42-NEXT: pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [4294967287,4294967286,4294967285,4294967284]
-; SSE42-NEXT: pminud %xmm1, %xmm2
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [8,9,10,11]
-; SSE42-NEXT: movdqa %xmm2, %xmm4
+; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [4294967287,4294967286,4294967285,4294967284]
+; SSE42-NEXT: pminud %xmm1, %xmm4
+; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [8,9,10,11]
+; SSE42-NEXT: movdqa %xmm4, %xmm2
+; SSE42-NEXT: pmaxud %xmm0, %xmm2
+; SSE42-NEXT: pcmpeqd %xmm4, %xmm2
+; SSE42-NEXT: packssdw %xmm3, %xmm2
+; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [4294967291,4294967290,4294967289,4294967288]
+; SSE42-NEXT: pminud %xmm1, %xmm3
+; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [4,5,6,7]
+; SSE42-NEXT: movdqa %xmm3, %xmm4
; SSE42-NEXT: pmaxud %xmm0, %xmm4
-; SSE42-NEXT: pcmpeqd %xmm2, %xmm4
-; SSE42-NEXT: packssdw %xmm3, %xmm4
-; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [4294967291,4294967290,4294967289,4294967288]
-; SSE42-NEXT: pminud %xmm1, %xmm2
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [4,5,6,7]
-; SSE42-NEXT: movdqa %xmm2, %xmm3
-; SSE42-NEXT: pmaxud %xmm0, %xmm3
; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [4294967295,4294967294,4294967293,4294967292]
; SSE42-NEXT: pminud %xmm1, %xmm5
; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5 # [0,1,2,3]
-; SSE42-NEXT: pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT: movdqa %xmm5, %xmm2
-; SSE42-NEXT: pmaxud %xmm0, %xmm2
-; SSE42-NEXT: pcmpeqd %xmm5, %xmm2
-; SSE42-NEXT: packssdw %xmm3, %xmm2
-; SSE42-NEXT: packsswb %xmm4, %xmm2
-; SSE42-NEXT: pmovmskb %xmm2, %ecx
+; SSE42-NEXT: pcmpeqd %xmm3, %xmm4
+; SSE42-NEXT: movdqa %xmm5, %xmm3
+; SSE42-NEXT: pmaxud %xmm0, %xmm3
+; SSE42-NEXT: pcmpeqd %xmm5, %xmm3
+; SSE42-NEXT: packssdw %xmm4, %xmm3
+; SSE42-NEXT: packsswb %xmm2, %xmm3
+; SSE42-NEXT: pmovmskb %xmm3, %ecx
; SSE42-NEXT: xorl $65535, %ecx # imm = 0xFFFF
; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [4294967267,4294967266,4294967265,4294967264]
; SSE42-NEXT: pminud %xmm1, %xmm2
@@ -1400,195 +1400,195 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
; SSE2-LABEL: lane_mask_v16i1_i64:
; SSE2: # %bb.0:
; SSE2-NEXT: movq %rdi, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,1,0,1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,1,0,1]
; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm6
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pxor %xmm0, %xmm5
; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
-; SSE2-NEXT: paddq %xmm5, %xmm2
+; SSE2-NEXT: paddq %xmm4, %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm1
; SSE2-NEXT: pxor %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm6, %xmm3
+; SSE2-NEXT: movdqa %xmm5, %xmm3
; SSE2-NEXT: pcmpgtd %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm6, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
-; SSE2-NEXT: pand %xmm4, %xmm7
+; SSE2-NEXT: pand %xmm6, %xmm7
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
; SSE2-NEXT: por %xmm2, %xmm1
; SSE2-NEXT: por %xmm7, %xmm1
; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [2,3]
-; SSE2-NEXT: paddq %xmm5, %xmm3
+; SSE2-NEXT: paddq %xmm4, %xmm3
; SSE2-NEXT: movdqa %xmm3, %xmm2
; SSE2-NEXT: pxor %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm6, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm6, %xmm2
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm2[1,1,3,3]
; SSE2-NEXT: pand %xmm7, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
; SSE2-NEXT: por %xmm3, %xmm2
; SSE2-NEXT: por %xmm8, %xmm2
-; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [4,5]
-; SSE2-NEXT: paddq %xmm5, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm3
+; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [4,5]
+; SSE2-NEXT: paddq %xmm4, %xmm6
+; SSE2-NEXT: movdqa %xmm6, %xmm3
; SSE2-NEXT: pxor %xmm0, %xmm3
-; SSE2-NEXT: movdqa %xmm6, %xmm7
+; SSE2-NEXT: movdqa %xmm5, %xmm7
; SSE2-NEXT: pcmpgtd %xmm3, %xmm7
; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm6, %xmm3
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm3
; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm3[1,1,3,3]
; SSE2-NEXT: pand %xmm8, %xmm9
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm3
+; SSE2-NEXT: por %xmm6, %xmm3
; SSE2-NEXT: por %xmm9, %xmm3
; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [6,7]
-; SSE2-NEXT: paddq %xmm5, %xmm7
-; SSE2-NEXT: movdqa %xmm7, %xmm4
-; SSE2-NEXT: pxor %xmm0, %xmm4
-; SSE2-NEXT: movdqa %xmm6, %xmm8
-; SSE2-NEXT: pcmpgtd %xmm4, %xmm8
+; SSE2-NEXT: paddq %xmm4, %xmm7
+; SSE2-NEXT: movdqa %xmm7, %xmm6
+; SSE2-NEXT: pxor %xmm0, %xmm6
+; SSE2-NEXT: movdqa %xmm5, %xmm8
+; SSE2-NEXT: pcmpgtd %xmm6, %xmm8
; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm6, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm4[1,1,3,3]
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]
; SSE2-NEXT: pand %xmm9, %xmm10
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm8[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm4
-; SSE2-NEXT: por %xmm10, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm8[1,1,3,3]
+; SSE2-NEXT: por %xmm7, %xmm6
+; SSE2-NEXT: por %xmm10, %xmm6
; SSE2-NEXT: movdqa {{.*#+}} xmm8 = [8,9]
-; SSE2-NEXT: paddq %xmm5, %xmm8
+; SSE2-NEXT: paddq %xmm4, %xmm8
; SSE2-NEXT: movdqa %xmm8, %xmm7
; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm6, %xmm9
+; SSE2-NEXT: movdqa %xmm5, %xmm9
; SSE2-NEXT: pcmpgtd %xmm7, %xmm9
; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm6, %xmm7
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm7
; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm7[1,1,3,3]
; SSE2-NEXT: pand %xmm10, %xmm11
; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm9[1,1,3,3]
; SSE2-NEXT: por %xmm8, %xmm7
; SSE2-NEXT: por %xmm11, %xmm7
; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [10,11]
-; SSE2-NEXT: paddq %xmm5, %xmm9
+; SSE2-NEXT: paddq %xmm4, %xmm9
; SSE2-NEXT: movdqa %xmm9, %xmm8
; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm6, %xmm10
+; SSE2-NEXT: movdqa %xmm5, %xmm10
; SSE2-NEXT: pcmpgtd %xmm8, %xmm10
; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm6, %xmm8
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm8
; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm8[1,1,3,3]
; SSE2-NEXT: pand %xmm11, %xmm12
; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm10[1,1,3,3]
; SSE2-NEXT: por %xmm9, %xmm8
; SSE2-NEXT: por %xmm12, %xmm8
; SSE2-NEXT: movdqa {{.*#+}} xmm10 = [12,13]
-; SSE2-NEXT: paddq %xmm5, %xmm10
+; SSE2-NEXT: paddq %xmm4, %xmm10
; SSE2-NEXT: movdqa %xmm10, %xmm9
; SSE2-NEXT: pxor %xmm0, %xmm9
-; SSE2-NEXT: movdqa %xmm6, %xmm11
+; SSE2-NEXT: movdqa %xmm5, %xmm11
; SSE2-NEXT: pcmpgtd %xmm9, %xmm11
; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm6, %xmm9
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm9
; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm9[1,1,3,3]
; SSE2-NEXT: pand %xmm12, %xmm13
; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm11[1,1,3,3]
; SSE2-NEXT: por %xmm10, %xmm9
; SSE2-NEXT: por %xmm13, %xmm9
-; SSE2-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5 # [14,15]
-; SSE2-NEXT: movdqa %xmm5, %xmm10
+; SSE2-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [14,15]
+; SSE2-NEXT: movdqa %xmm4, %xmm10
; SSE2-NEXT: pxor %xmm0, %xmm10
-; SSE2-NEXT: movdqa %xmm6, %xmm11
+; SSE2-NEXT: movdqa %xmm5, %xmm11
; SSE2-NEXT: pcmpgtd %xmm10, %xmm11
; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm6, %xmm10
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm10[1,1,3,3]
-; SSE2-NEXT: pand %xmm12, %xmm6
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm10
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm10[1,1,3,3]
+; SSE2-NEXT: pand %xmm12, %xmm5
; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
+; SSE2-NEXT: por %xmm4, %xmm10
; SSE2-NEXT: por %xmm5, %xmm10
-; SSE2-NEXT: por %xmm6, %xmm10
-; SSE2-NEXT: movq %rsi, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,0,1]
+; SSE2-NEXT: movq %rsi, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
; SSE2-NEXT: pxor %xmm0, %xmm10
-; SSE2-NEXT: pxor %xmm0, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm10, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm6[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm10
+; SSE2-NEXT: pxor %xmm0, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pcmpgtd %xmm10, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm5[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm10
; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
; SSE2-NEXT: pand %xmm11, %xmm10
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: por %xmm10, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT: por %xmm10, %xmm5
; SSE2-NEXT: pxor %xmm0, %xmm9
-; SSE2-NEXT: movdqa %xmm5, %xmm10
+; SSE2-NEXT: movdqa %xmm4, %xmm10
; SSE2-NEXT: pcmpgtd %xmm9, %xmm10
; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm9
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm9
; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm9[1,1,3,3]
; SSE2-NEXT: pand %xmm11, %xmm12
; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
; SSE2-NEXT: por %xmm12, %xmm9
-; SSE2-NEXT: packssdw %xmm6, %xmm9
+; SSE2-NEXT: packssdw %xmm5, %xmm9
; SSE2-NEXT: pxor %xmm0, %xmm8
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm8
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pcmpgtd %xmm8, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm8
; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
; SSE2-NEXT: pand %xmm10, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm5[1,1,3,3]
; SSE2-NEXT: por %xmm8, %xmm10
; SSE2-NEXT: pxor %xmm0, %xmm7
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: pcmpgtd %xmm7, %xmm6
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm7
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pcmpgtd %xmm7, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm7
; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
; SSE2-NEXT: pand %xmm8, %xmm7
-; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT: por %xmm7, %xmm6
-; SSE2-NEXT: packssdw %xmm10, %xmm6
-; SSE2-NEXT: packssdw %xmm9, %xmm6
-; SSE2-NEXT: pxor %xmm0, %xmm4
-; SSE2-NEXT: movdqa %xmm5, %xmm7
-; SSE2-NEXT: pcmpgtd %xmm4, %xmm7
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT: por %xmm7, %xmm5
+; SSE2-NEXT: packssdw %xmm10, %xmm5
+; SSE2-NEXT: packssdw %xmm9, %xmm5
+; SSE2-NEXT: pxor %xmm0, %xmm6
+; SSE2-NEXT: movdqa %xmm4, %xmm7
+; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: pand %xmm8, %xmm4
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT: pand %xmm8, %xmm6
; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT: por %xmm4, %xmm7
+; SSE2-NEXT: por %xmm6, %xmm7
; SSE2-NEXT: pxor %xmm0, %xmm3
-; SSE2-NEXT: movdqa %xmm5, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT: movdqa %xmm4, %xmm6
+; SSE2-NEXT: pcmpgtd %xmm3, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm3
; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm3[1,1,3,3]
; SSE2-NEXT: pand %xmm8, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
; SSE2-NEXT: por %xmm9, %xmm3
; SSE2-NEXT: packssdw %xmm7, %xmm3
; SSE2-NEXT: pxor %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm5, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT: movdqa %xmm4, %xmm6
+; SSE2-NEXT: pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
; SSE2-NEXT: pand %xmm7, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm2, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT: por %xmm2, %xmm6
; SSE2-NEXT: pxor %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm5, %xmm0
+; SSE2-NEXT: movdqa %xmm4, %xmm0
; SSE2-NEXT: pcmpgtd %xmm1, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
; SSE2-NEXT: pand %xmm2, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: packssdw %xmm4, %xmm0
+; SSE2-NEXT: packssdw %xmm6, %xmm0
; SSE2-NEXT: packssdw %xmm3, %xmm0
-; SSE2-NEXT: packsswb %xmm6, %xmm0
+; SSE2-NEXT: packsswb %xmm5, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: lane_mask_v16i1_i64:
@@ -1596,72 +1596,72 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
; SSE42-NEXT: movq %rdi, %xmm0
; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
-; SSE42-NEXT: movdqa %xmm0, %xmm5
-; SSE42-NEXT: pxor %xmm1, %xmm5
+; SSE42-NEXT: movdqa %xmm0, %xmm4
+; SSE42-NEXT: pxor %xmm1, %xmm4
; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
; SSE42-NEXT: paddq %xmm0, %xmm3
-; SSE42-NEXT: movdqa %xmm3, %xmm4
-; SSE42-NEXT: pxor %xmm1, %xmm4
-; SSE42-NEXT: movdqa %xmm5, %xmm2
-; SSE42-NEXT: pcmpgtq %xmm4, %xmm2
+; SSE42-NEXT: movdqa %xmm3, %xmm5
+; SSE42-NEXT: pxor %xmm1, %xmm5
+; SSE42-NEXT: movdqa %xmm4, %xmm2
+; SSE42-NEXT: pcmpgtq %xmm5, %xmm2
; SSE42-NEXT: por %xmm3, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [2,3]
-; SSE42-NEXT: paddq %xmm0, %xmm4
-; SSE42-NEXT: movdqa %xmm4, %xmm6
+; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [2,3]
+; SSE42-NEXT: paddq %xmm0, %xmm5
+; SSE42-NEXT: movdqa %xmm5, %xmm6
; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm5, %xmm3
+; SSE42-NEXT: movdqa %xmm4, %xmm3
; SSE42-NEXT: pcmpgtq %xmm6, %xmm3
-; SSE42-NEXT: por %xmm4, %xmm3
+; SSE42-NEXT: por %xmm5, %xmm3
; SSE42-NEXT: movdqa {{.*#+}} xmm6 = [4,5]
; SSE42-NEXT: paddq %xmm0, %xmm6
; SSE42-NEXT: movdqa %xmm6, %xmm7
; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm5, %xmm4
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm4
-; SSE42-NEXT: por %xmm6, %xmm4
+; SSE42-NEXT: movdqa %xmm4, %xmm5
+; SSE42-NEXT: pcmpgtq %xmm7, %xmm5
+; SSE42-NEXT: por %xmm6, %xmm5
; SSE42-NEXT: movdqa {{.*#+}} xmm7 = [6,7]
; SSE42-NEXT: paddq %xmm0, %xmm7
; SSE42-NEXT: movdqa %xmm7, %xmm8
; SSE42-NEXT: pxor %xmm1, %xmm8
-; SSE42-NEXT: movdqa %xmm5, %xmm6
+; SSE42-NEXT: movdqa %xmm4, %xmm6
; SSE42-NEXT: pcmpgtq %xmm8, %xmm6
; SSE42-NEXT: por %xmm7, %xmm6
; SSE42-NEXT: movdqa {{.*#+}} xmm8 = [8,9]
; SSE42-NEXT: paddq %xmm0, %xmm8
; SSE42-NEXT: movdqa %xmm8, %xmm9
; SSE42-NEXT: pxor %xmm1, %xmm9
-; SSE42-NEXT: movdqa %xmm5, %xmm7
+; SSE42-NEXT: movdqa %xmm4, %xmm7
; SSE42-NEXT: pcmpgtq %xmm9, %xmm7
; SSE42-NEXT: por %xmm8, %xmm7
; SSE42-NEXT: movdqa {{.*#+}} xmm9 = [10,11]
; SSE42-NEXT: paddq %xmm0, %xmm9
; SSE42-NEXT: movdqa %xmm9, %xmm10
; SSE42-NEXT: pxor %xmm1, %xmm10
-; SSE42-NEXT: movdqa %xmm5, %xmm8
+; SSE42-NEXT: movdqa %xmm4, %xmm8
; SSE42-NEXT: pcmpgtq %xmm10, %xmm8
; SSE42-NEXT: por %xmm9, %xmm8
; SSE42-NEXT: movdqa {{.*#+}} xmm9 = [12,13]
; SSE42-NEXT: paddq %xmm0, %xmm9
; SSE42-NEXT: movdqa %xmm9, %xmm10
; SSE42-NEXT: pxor %xmm1, %xmm10
-; SSE42-NEXT: movdqa %xmm5, %xmm11
+; SSE42-NEXT: movdqa %xmm4, %xmm11
; SSE42-NEXT: pcmpgtq %xmm10, %xmm11
; SSE42-NEXT: por %xmm9, %xmm11
; SSE42-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [14,15]
; SSE42-NEXT: movdqa %xmm0, %xmm9
; SSE42-NEXT: pxor %xmm1, %xmm9
-; SSE42-NEXT: pcmpgtq %xmm9, %xmm5
-; SSE42-NEXT: por %xmm0, %xmm5
+; SSE42-NEXT: pcmpgtq %xmm9, %xmm4
+; SSE42-NEXT: por %xmm0, %xmm4
; SSE42-NEXT: movq %rsi, %xmm0
; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE42-NEXT: pxor %xmm1, %xmm5
+; SSE42-NEXT: pxor %xmm1, %xmm4
; SSE42-NEXT: pxor %xmm1, %xmm0
; SSE42-NEXT: movdqa %xmm0, %xmm9
-; SSE42-NEXT: pcmpgtq %xmm5, %xmm9
+; SSE42-NEXT: pcmpgtq %xmm4, %xmm9
; SSE42-NEXT: pxor %xmm1, %xmm11
-; SSE42-NEXT: movdqa %xmm0, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm11, %xmm5
-; SSE42-NEXT: packssdw %xmm9, %xmm5
+; SSE42-NEXT: movdqa %xmm0, %xmm4
+; SSE42-NEXT: pcmpgtq %xmm11, %xmm4
+; SSE42-NEXT: packssdw %xmm9, %xmm4
; SSE42-NEXT: pxor %xmm1, %xmm8
; SSE42-NEXT: movdqa %xmm0, %xmm9
; SSE42-NEXT: pcmpgtq %xmm8, %xmm9
@@ -1669,14 +1669,14 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
; SSE42-NEXT: movdqa %xmm0, %xmm8
; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
; SSE42-NEXT: packssdw %xmm9, %xmm8
-; SSE42-NEXT: packssdw %xmm5, %xmm8
+; SSE42-NEXT: packssdw %xmm4, %xmm8
; SSE42-NEXT: pxor %xmm1, %xmm6
-; SSE42-NEXT: movdqa %xmm0, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm6, %xmm5
-; SSE42-NEXT: pxor %xmm1, %xmm4
+; SSE42-NEXT: movdqa %xmm0, %xmm4
+; SSE42-NEXT: pcmpgtq %xmm6, %xmm4
+; SSE42-NEXT: pxor %xmm1, %xmm5
; SSE42-NEXT: movdqa %xmm0, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm4, %xmm6
-; SSE42-NEXT: packssdw %xmm5, %xmm6
+; SSE42-NEXT: pcmpgtq %xmm5, %xmm6
+; SSE42-NEXT: packssdw %xmm4, %xmm6
; SSE42-NEXT: pxor %xmm1, %xmm3
; SSE42-NEXT: movdqa %xmm0, %xmm4
; SSE42-NEXT: pcmpgtq %xmm3, %xmm4
@@ -2282,20 +2282,9 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
; SSE42-NEXT: pcmpgtq %xmm6, %xmm7
; SSE42-NEXT: por %xmm5, %xmm7
; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm3, %xmm5
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm5
-; SSE42-NEXT: movdqa {{.*#+}} xmm6 = [20,21]
-; SSE42-NEXT: paddq %xmm2, %xmm6
-; SSE42-NEXT: movdqa %xmm6, %xmm7
-; SSE42-NEXT: pxor %xmm1, %xmm7
-; SSE42-NEXT: movdqa %xmm0, %xmm8
-; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT: por %xmm6, %xmm8
-; SSE42-NEXT: pxor %xmm1, %xmm8
; SSE42-NEXT: movdqa %xmm3, %xmm6
-; SSE42-NEXT: pcmpgtq %xmm8, %xmm6
-; SSE42-NEXT: packssdw %xmm5, %xmm6
-; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [18,19]
+; SSE42-NEXT: pcmpgtq %xmm7, %xmm6
+; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [20,21]
; SSE42-NEXT: paddq %xmm2, %xmm5
; SSE42-NEXT: movdqa %xmm5, %xmm7
; SSE42-NEXT: pxor %xmm1, %xmm7
@@ -2305,6 +2294,17 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
; SSE42-NEXT: pxor %xmm1, %xmm8
; SSE42-NEXT: movdqa %xmm3, %xmm5
; SSE42-NEXT: pcmpgtq %xmm8, %xmm5
+; SSE42-NEXT: packssdw %xmm6, %xmm5
+; SSE42-NEXT: movdqa {{.*#+}} xmm6 = [18,19]
+; SSE42-NEXT: paddq %xmm2, %xmm6
+; SSE42-NEXT: movdqa %xmm6, %xmm7
+; SSE42-NEXT: pxor %xmm1, %xmm7
+; SSE42-NEXT: movdqa %xmm0, %xmm8
+; SSE42-NEXT: pcmpgtq %xmm7, %xmm8
+; SSE42-NEXT: por %xmm6, %xmm8
+; SSE42-NEXT: pxor %xmm1, %xmm8
+; SSE42-NEXT: movdqa %xmm3, %xmm6
+; SSE42-NEXT: pcmpgtq %xmm8, %xmm6
; SSE42-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [16,17]
; SSE42-NEXT: movdqa %xmm2, %xmm7
; SSE42-NEXT: pxor %xmm1, %xmm7
@@ -2312,8 +2312,8 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
; SSE42-NEXT: por %xmm2, %xmm0
; SSE42-NEXT: pxor %xmm1, %xmm0
; SSE42-NEXT: pcmpgtq %xmm0, %xmm3
-; SSE42-NEXT: packssdw %xmm5, %xmm3
; SSE42-NEXT: packssdw %xmm6, %xmm3
+; SSE42-NEXT: packssdw %xmm5, %xmm3
; SSE42-NEXT: packsswb %xmm4, %xmm3
; SSE42-NEXT: pmovmskb %xmm3, %edx
; SSE42-NEXT: shll $16, %edx
>From 81f07d0929f89c354deca8f056ec731793c686af Mon Sep 17 00:00:00 2001
From: william <we3223 at gmail.com>
Date: Mon, 5 Oct 2026 22:59:13 +0800
Subject: [PATCH 3/5] [X86][TTI] Enable DataAndControlFlow tail folding style
on AVX-512
---
llvm/lib/Target/X86/X86TargetTransformInfo.h | 7 +-
.../LoopVectorize/X86/cost-model.ll | 8 +-
.../X86/divs-with-tail-folding.ll | 47 ++---
.../X86/fold-tail-low-trip-count.ll | 34 ++--
.../Transforms/LoopVectorize/X86/optsize.ll | 96 ++++------
.../X86/scev-checks-unprofitable.ll | 5 +-
.../X86/vectorize-interleaved-accesses-gap.ll | 33 ++--
.../widen-canonical-iv-register-pressure.ll | 59 +++---
.../x86-interleaved-accesses-masked-group.ll | 178 +++++++-----------
...86-interleaved-store-accesses-with-gaps.ll | 60 +++---
10 files changed, 228 insertions(+), 299 deletions(-)
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 41fbcc77304fb2..f87cd2fee49271 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -57,7 +57,12 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
/// \name Vector TTI Implementations
/// @{
-
+
+ TailFoldingStyle
+ getPreferredTailFoldingStyle() const {
+ return ST->hasAVX512() ? TailFoldingStyle::DataAndControlFlow
+ : TailFoldingStyle::DataWithoutLaneMask;
+ }
unsigned getNumberOfRegisters(unsigned ClassID) const override;
unsigned getRegisterClassForType(bool Vector, Type *Ty) const override;
bool hasConditionalLoadStoreForType(Type *Ty, bool IsStore) const override;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 42c8fe3011f7d2..26dc78ff94b76b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -981,20 +981,22 @@ define void @replicating_sdiv_operand_profitable_to_scalarize(i32 %x) #3 {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 0, i32 13)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> splat (i32 2), <16 x i32> [[BROADCAST_SPLAT]], <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>)
+; CHECK-NEXT: [[TMP0:%.*]] = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> splat (i32 2), <16 x i32> [[BROADCAST_SPLAT]], <16 x i1> [[ACTIVE_LANE_MASK_ENTRY]])
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> zeroinitializer, <16 x i32> [[TMP0]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
; CHECK-NEXT: [[TMP2:%.*]] = sub <16 x i32> [[BROADCAST_SPLAT]], [[TMP0]]
; CHECK-NEXT: [[TMP3:%.*]] = or <16 x i32> [[TMP2]], [[TMP1]]
; CHECK-NEXT: [[TMP4:%.*]] = or <16 x i32> [[TMP0]], splat (i32 3)
-; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> [[TMP0]], <16 x i32> [[TMP4]], <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>)
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> [[TMP0]], <16 x i32> [[TMP4]], <16 x i1> [[ACTIVE_LANE_MASK_ENTRY]])
; CHECK-NEXT: [[TMP6:%.*]] = or <16 x i32> [[TMP3]], [[TMP5]]
-; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> <i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true>, i1 false)
+; CHECK-NEXT: [[TMP8:%.*]] = xor <16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], splat (i1 true)
+; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> [[TMP8]], i1 false)
; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <16 x i32> [[TMP6]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT1:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
index fada3253c14717..0d7a3dc5163ce4 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
@@ -16,17 +16,11 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
; CHECK-NEXT: [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[N]])
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0
; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP16]], i64 [[CONV6]], i64 1
; CHECK-NEXT: [[TMP8:%.*]] = sdiv i64 [[M]], [[TMP9]]
@@ -40,9 +34,10 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP25]]
; CHECK-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> zeroinitializer, ptr align 8 [[TMP26]], <4 x i1> [[TMP6]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IV]], splat (i64 4)
-; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP17:%.*]] = xor i1 [[TMP18]], true
+; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -103,19 +98,14 @@ define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i
; CHECK-NEXT: [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[N]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP8]], i64 0
@@ -131,9 +121,11 @@ define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP27]]
; CHECK-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> zeroinitializer, ptr align 8 [[TMP28]], <4 x i1> [[TMP8]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[TMP5]], 4
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP19:%.*]] = xor i1 [[TMP18]], true
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -207,19 +199,14 @@ define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
; CHECK-NEXT: [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; CHECK-NEXT: [[TMP5:%.*]] = and i64 [[N_RND_UP]], 3
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP5]]
-; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP0]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[MUL_2_I]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP7:%.*]] = udiv <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP7]], i64 0
; CHECK-NEXT: [[TMP9:%.*]] = urem i64 [[INDEX]], [[MUL_2_I]]
@@ -238,9 +225,11 @@ define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP21]]
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP7]], ptr align 4 [[TMP22]], <4 x i1> [[TMP6]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP24:%.*]] = xor i1 [[TMP23]], true
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll
index b92a591ae5147c..6636b358c3a8e0 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll
@@ -50,27 +50,22 @@ define ptr @low_trip_count_via_profile_info_with_iv_live_out(ptr align 16 %start
; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP8]], [[TMP7]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP1]], 63
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 63
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP1]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <64 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <64 x i64> [[BROADCAST_SPLATINSERT]], <64 x i64> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 0, i64 [[TMP1]])
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IV:%.*]] = phi <64 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31, i64 32, i64 33, i64 34, i64 35, i64 36, i64 37, i64 38, i64 39, i64 40, i64 41, i64 42, i64 43, i64 44, i64 45, i64 46, i64 47, i64 48, i64 49, i64 50, i64 51, i64 52, i64 53, i64 54, i64 55, i64 56, i64 57, i64 58, i64 59, i64 60, i64 61, i64 62, i64 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <64 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = phi <64 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP4]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[TMP5]], <64 x i1> [[TMP3]], <64 x i8> poison)
; CHECK-NEXT: call void @llvm.masked.store.v64i8.p0(<64 x i8> [[WIDE_MASKED_LOAD]], ptr align 1 [[NEXT_GEP]], <64 x i1> [[TMP3]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 64
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <64 x i64> [[VEC_IV]], splat (i64 64)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP1]])
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -104,26 +99,21 @@ define void @low_trip_count_via_profile_info(ptr align 16 %start, ptr align 16 %
; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP7]], [[TMP6]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP1]], 63
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 63
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <64 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <64 x i64> [[BROADCAST_SPLATINSERT]], <64 x i64> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 0, i64 [[TMP1]])
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IV:%.*]] = phi <64 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31, i64 32, i64 33, i64 34, i64 35, i64 36, i64 37, i64 38, i64 39, i64 40, i64 41, i64 42, i64 43, i64 44, i64 45, i64 46, i64 47, i64 48, i64 49, i64 50, i64 51, i64 52, i64 53, i64 54, i64 55, i64 56, i64 57, i64 58, i64 59, i64 60, i64 61, i64 62, i64 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <64 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <64 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[TMP4]], <64 x i1> [[TMP2]], <64 x i8> poison)
; CHECK-NEXT: call void @llvm.masked.store.v64i8.p0(<64 x i8> [[WIDE_MASKED_LOAD]], ptr align 1 [[NEXT_GEP]], <64 x i1> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 64
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <64 x i64> [[VEC_IV]], splat (i64 64)
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP1]])
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/optsize.ll b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
index c78688c348b840..0dc598e7b4e7bb 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
@@ -15,19 +15,20 @@ define i32 @foo_optsize() #0 {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 0, i32 203)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <64 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 39, i8 40, i8 41, i8 42, i8 43, i8 44, i8 45, i8 46, i8 47, i8 48, i8 49, i8 50, i8 51, i8 52, i8 53, i8 54, i8 55, i8 56, i8 57, i8 58, i8 59, i8 60, i8 61, i8 62, i8 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <64 x i8> [[VEC_IND]], splat (i8 -54)
+; CHECK-NEXT: [[TMP1:%.*]] = phi <64 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[TMP2]], <64 x i1> [[TMP1]], <64 x i8> poison)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <64 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = select <64 x i1> [[TMP4]], <64 x i8> splat (i8 2), <64 x i8> splat (i8 1)
; CHECK-NEXT: call void @llvm.masked.store.v64i8.p0(<64 x i8> [[TMP5]], ptr align 1 [[TMP2]], <64 x i1> [[TMP1]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 64
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <64 x i8> [[VEC_IND]], splat (i8 64)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 64
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 [[INDEX_NEXT]], i32 203)
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = xor i1 [[TMP3]], true
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[FOR_END:.*]]
@@ -39,19 +40,20 @@ define i32 @foo_optsize() #0 {
; AUTOVF-NEXT: [[ENTRY:.*:]]
; AUTOVF-NEXT: br label %[[VECTOR_PH:.*]]
; AUTOVF: [[VECTOR_PH]]:
+; AUTOVF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i32(i32 0, i32 203)
; AUTOVF-NEXT: br label %[[VECTOR_BODY:.*]]
; AUTOVF: [[VECTOR_BODY]]:
; AUTOVF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AUTOVF-NEXT: [[VEC_IND:%.*]] = phi <32 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AUTOVF-NEXT: [[TMP1:%.*]] = icmp ule <32 x i8> [[VEC_IND]], splat (i8 -54)
+; AUTOVF-NEXT: [[TMP1:%.*]] = phi <32 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; AUTOVF-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[INDEX]]
; AUTOVF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP2]], <32 x i1> [[TMP1]], <32 x i8> poison)
; AUTOVF-NEXT: [[TMP4:%.*]] = icmp eq <32 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
; AUTOVF-NEXT: [[TMP5:%.*]] = select <32 x i1> [[TMP4]], <32 x i8> splat (i8 2), <32 x i8> splat (i8 1)
; AUTOVF-NEXT: call void @llvm.masked.store.v32i8.p0(<32 x i8> [[TMP5]], ptr align 1 [[TMP2]], <32 x i1> [[TMP1]])
-; AUTOVF-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
-; AUTOVF-NEXT: [[VEC_IND_NEXT]] = add nuw <32 x i8> [[VEC_IND]], splat (i8 32)
-; AUTOVF-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 224
+; AUTOVF-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 32
+; AUTOVF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i32(i32 [[INDEX_NEXT]], i32 203)
+; AUTOVF-NEXT: [[TMP3:%.*]] = extractelement <32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; AUTOVF-NEXT: [[TMP6:%.*]] = xor i1 [[TMP3]], true
; AUTOVF-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; AUTOVF: [[MIDDLE_BLOCK]]:
; AUTOVF-NEXT: br label %[[FOR_END:.*]]
@@ -85,19 +87,20 @@ define i32 @foo_minsize() #1 {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 0, i32 203)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <64 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 39, i8 40, i8 41, i8 42, i8 43, i8 44, i8 45, i8 46, i8 47, i8 48, i8 49, i8 50, i8 51, i8 52, i8 53, i8 54, i8 55, i8 56, i8 57, i8 58, i8 59, i8 60, i8 61, i8 62, i8 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <64 x i8> [[VEC_IND]], splat (i8 -54)
+; CHECK-NEXT: [[TMP1:%.*]] = phi <64 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[TMP2]], <64 x i1> [[TMP1]], <64 x i8> poison)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <64 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = select <64 x i1> [[TMP4]], <64 x i8> splat (i8 2), <64 x i8> splat (i8 1)
; CHECK-NEXT: call void @llvm.masked.store.v64i8.p0(<64 x i8> [[TMP5]], ptr align 1 [[TMP2]], <64 x i1> [[TMP1]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 64
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <64 x i8> [[VEC_IND]], splat (i8 64)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 64
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 [[INDEX_NEXT]], i32 203)
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = xor i1 [[TMP3]], true
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[FOR_END:.*]]
@@ -106,25 +109,18 @@ define i32 @foo_minsize() #1 {
;
; AUTOVF-LABEL: define i32 @foo_minsize(
; AUTOVF-SAME: ) #[[ATTR1:[0-9]+]] {
-; AUTOVF-NEXT: [[ENTRY:.*:]]
-; AUTOVF-NEXT: br label %[[VECTOR_PH:.*]]
-; AUTOVF: [[VECTOR_PH]]:
+; AUTOVF-NEXT: [[VECTOR_PH:.*]]:
; AUTOVF-NEXT: br label %[[VECTOR_BODY:.*]]
; AUTOVF: [[VECTOR_BODY]]:
-; AUTOVF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AUTOVF-NEXT: [[VEC_IND:%.*]] = phi <16 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AUTOVF-NEXT: [[TMP0:%.*]] = icmp ule <16 x i8> [[VEC_IND]], splat (i8 -54)
+; AUTOVF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INC:%.*]], %[[VECTOR_BODY]] ]
; AUTOVF-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[INDEX]]
-; AUTOVF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[TMP0]], <16 x i8> poison)
-; AUTOVF-NEXT: [[TMP5:%.*]] = icmp eq <16 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
-; AUTOVF-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP5]], <16 x i8> splat (i8 2), <16 x i8> splat (i8 1)
-; AUTOVF-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP3]], ptr align 1 [[TMP2]], <16 x i1> [[TMP0]])
-; AUTOVF-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; AUTOVF-NEXT: [[VEC_IND_NEXT]] = add nuw <16 x i8> [[VEC_IND]], splat (i8 16)
-; AUTOVF-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 208
-; AUTOVF-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; AUTOVF: [[MIDDLE_BLOCK]]:
-; AUTOVF-NEXT: br label %[[FOR_END:.*]]
+; AUTOVF-NEXT: [[TMP0:%.*]] = load i8, ptr [[TMP2]], align 1
+; AUTOVF-NEXT: [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0
+; AUTOVF-NEXT: [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
+; AUTOVF-NEXT: store i8 [[DOT]], ptr [[TMP2]], align 1
+; AUTOVF-NEXT: [[INC]] = add nsw i32 [[INDEX]], 1
+; AUTOVF-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDEX]], 202
+; AUTOVF-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]]
; AUTOVF: [[FOR_END]]:
; AUTOVF-NEXT: ret i32 0
;
@@ -196,7 +192,7 @@ define void @scev4stride1(ptr noalias nocapture %a, ptr noalias nocapture readon
; AUTOVF-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; AUTOVF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; AUTOVF-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
-; AUTOVF-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; AUTOVF-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; AUTOVF: [[MIDDLE_BLOCK]]:
; AUTOVF-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
; AUTOVF: [[FOR_END_LOOPEXIT]]:
@@ -333,24 +329,19 @@ define void @tail_folded_store_avx512(ptr %start, ptr %end) #3 {
; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], 1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP3]], 63
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 63
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP3]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <64 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <64 x i32> [[BROADCAST_SPLATINSERT3]], <64 x i32> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 0, i32 [[TMP3]])
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi <64 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IV:%.*]] = phi <64 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <64 x i32> <i32 0, i32 -72, i32 -144, i32 -216, i32 -288, i32 -360, i32 -432, i32 -504, i32 -576, i32 -648, i32 -720, i32 -792, i32 -864, i32 -936, i32 -1008, i32 -1080, i32 -1152, i32 -1224, i32 -1296, i32 -1368, i32 -1440, i32 -1512, i32 -1584, i32 -1656, i32 -1728, i32 -1800, i32 -1872, i32 -1944, i32 -2016, i32 -2088, i32 -2160, i32 -2232, i32 -2304, i32 -2376, i32 -2448, i32 -2520, i32 -2592, i32 -2664, i32 -2736, i32 -2808, i32 -2880, i32 -2952, i32 -3024, i32 -3096, i32 -3168, i32 -3240, i32 -3312, i32 -3384, i32 -3456, i32 -3528, i32 -3600, i32 -3672, i32 -3744, i32 -3816, i32 -3888, i32 -3960, i32 -4032, i32 -4104, i32 -4176, i32 -4248, i32 -4320, i32 -4392, i32 -4464, i32 -4536>
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ule <64 x i32> [[VEC_IV]], [[BROADCAST_SPLAT4]]
; CHECK-NEXT: call void @llvm.masked.scatter.v64p0.v64p0(<64 x ptr> splat (ptr null), <64 x ptr> align 8 [[TMP5]], <64 x i1> [[TMP6]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 64
+; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 64
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 [[INDEX_NEXT]], i32 [[TMP3]])
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = xor i1 [[TMP9]], true
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 -4608
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <64 x i32> [[VEC_IV]], splat (i32 64)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
@@ -368,25 +359,20 @@ define void @tail_folded_store_avx512(ptr %start, ptr %end) #3 {
; AUTOVF-NEXT: [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], 1
; AUTOVF-NEXT: br label %[[VECTOR_PH:.*]]
; AUTOVF: [[VECTOR_PH]]:
-; AUTOVF-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP3]], 7
-; AUTOVF-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; AUTOVF-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; AUTOVF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP3]], 1
-; AUTOVF-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; AUTOVF-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT3]], <8 x i32> poison, <8 x i32> zeroinitializer
+; AUTOVF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[TMP3]])
; AUTOVF-NEXT: br label %[[VECTOR_BODY:.*]]
; AUTOVF: [[VECTOR_BODY]]:
; AUTOVF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AUTOVF-NEXT: [[TMP6:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; AUTOVF-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
-; AUTOVF-NEXT: [[VEC_IV:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; AUTOVF-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <8 x i32> <i32 0, i32 -72, i32 -144, i32 -216, i32 -288, i32 -360, i32 -432, i32 -504>
-; AUTOVF-NEXT: [[TMP6:%.*]] = icmp ule <8 x i32> [[VEC_IV]], [[BROADCAST_SPLAT4]]
; AUTOVF-NEXT: call void @llvm.masked.scatter.v8p0.v8p0(<8 x ptr> splat (ptr null), <8 x ptr> align 8 [[TMP5]], <8 x i1> [[TMP6]])
-; AUTOVF-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; AUTOVF-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; AUTOVF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[TMP3]])
+; AUTOVF-NEXT: [[TMP9:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; AUTOVF-NEXT: [[TMP7:%.*]] = xor i1 [[TMP9]], true
; AUTOVF-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 -576
-; AUTOVF-NEXT: [[VEC_IND_NEXT]] = add nuw <8 x i32> [[VEC_IV]], splat (i32 8)
-; AUTOVF-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; AUTOVF-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; AUTOVF-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; AUTOVF: [[MIDDLE_BLOCK]]:
; AUTOVF-NEXT: br label %[[EXIT:.*]]
; AUTOVF: [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll b/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll
index 16e65d3f1b3526..e2c4996f51701c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll
@@ -22,12 +22,9 @@ define void @value_defined_in_loop1_used_for_trip_counts(i32 %start, i1 %c, ptr
; CHECK: [[LOOP_3_PREHEADER]]:
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[IV_1]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 [[IV_1]])
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[BROADCAST_SPLAT]]
; CHECK-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> zeroinitializer, ptr align 1 [[DST]], <8 x i1> [[TMP0]])
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
index ead2f8481e5398..c9a74ae42a8382 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
@@ -9,11 +9,11 @@ define void @test_pr59090(ptr %l_out, ptr noalias %b) #0 {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 10001)
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i16> [ <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <8 x i16> [[VEC_IND]], splat (i16 10000)
+; CHECK-NEXT: [[TMP1:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = mul nuw i64 [[INDEX]], 6
; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[B:%.*]], align 1, !llvm.access.group [[ACC_GRP0:![0-9]+]]
; CHECK-NEXT: store i8 [[TMP3]], ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0]]
@@ -21,10 +21,11 @@ define void @test_pr59090(ptr %l_out, ptr noalias %b) #0 {
; CHECK-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> poison, <48 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
; CHECK-NEXT: [[TMP15:%.*]] = and <48 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false>
; CHECK-NEXT: call void @llvm.masked.store.v48i8.p0(<48 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison>, ptr align 1 [[TMP13]], <48 x i1> [[TMP15]]), !llvm.access.group [[ACC_GRP0]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <8 x i16> [[VEC_IND]], splat (i16 8)
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10008
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT]], i64 10001)
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: exit:
@@ -56,13 +57,14 @@ define void @test_pr59090_interleave(ptr %l_out, ptr noalias %b) #0 {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 10001)
+; CHECK-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <8 x i64> [[VEC_IND]], splat (i64 8)
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IND]], splat (i64 10000)
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <8 x i64> [[STEP_ADD]], splat (i64 10000)
+; CHECK-NEXT: [[TMP1:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[INDEX]], 6
; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP0]], 6
@@ -76,10 +78,13 @@ define void @test_pr59090_interleave(ptr %l_out, ptr noalias %b) #0 {
; CHECK-NEXT: [[INTERLEAVED_MASK1:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <48 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
; CHECK-NEXT: [[TMP9:%.*]] = and <48 x i1> [[INTERLEAVED_MASK1]], <i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false>
; CHECK-NEXT: call void @llvm.masked.store.v48i8.p0(<48 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison>, ptr align 1 [[TMP7]], <48 x i1> [[TMP9]]), !llvm.access.group [[ACC_GRP0]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <8 x i64> [[STEP_ADD]], splat (i64 8)
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10016
-; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[INDEX_NEXT]], i64 10001)
+; CHECK-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-NEXT: [[EXTRACT_NEXT_ALM_PART4]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[EXIT:%.*]]
; CHECK: exit:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/widen-canonical-iv-register-pressure.ll b/llvm/test/Transforms/LoopVectorize/X86/widen-canonical-iv-register-pressure.ll
index f147f9f9beca28..e4683219283432 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/widen-canonical-iv-register-pressure.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/widen-canonical-iv-register-pressure.ll
@@ -12,15 +12,14 @@ define i32 @many_reductions(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, i64
; UF2-NEXT: [[ENTRY:.*:]]
; UF2-NEXT: br label %[[VECTOR_PH:.*]]
; UF2: [[VECTOR_PH]]:
-; UF2-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 15
-; UF2-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 15
-; UF2-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; UF2-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; UF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; UF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; UF2-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 [[N]])
+; UF2-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; UF2-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
; UF2-NEXT: br label %[[VECTOR_BODY:.*]]
; UF2: [[VECTOR_BODY]]:
; UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UF2-NEXT: [[TMP0:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], %[[VECTOR_BODY]] ]
+; UF2-NEXT: [[TMP1:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART25:%.*]], %[[VECTOR_BODY]] ]
; UF2-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
; UF2-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
; UF2-NEXT: [[VEC_PHI2:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
@@ -33,10 +32,6 @@ define i32 @many_reductions(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, i64
; UF2-NEXT: [[VEC_PHI9:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
; UF2-NEXT: [[VEC_PHI10:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
; UF2-NEXT: [[VEC_PHI11:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
-; UF2-NEXT: [[VEC_IV:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UF2-NEXT: [[VEC_IV16:%.*]] = add nuw <8 x i64> [[VEC_IV]], splat (i64 8)
-; UF2-NEXT: [[TMP0:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; UF2-NEXT: [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IV16]], [[BROADCAST_SPLAT]]
; UF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; UF2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; UF2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
@@ -74,9 +69,12 @@ define i32 @many_reductions(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, i64
; UF2-NEXT: [[TMP24]] = add <8 x i32> [[VEC_PHI10]], [[WIDE_MASKED_LOAD26]]
; UF2-NEXT: [[TMP25]] = add <8 x i32> [[VEC_PHI11]], [[WIDE_MASKED_LOAD27]]
; UF2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 16
-; UF2-NEXT: [[VEC_IND_NEXT]] = add nuw <8 x i64> [[VEC_IV16]], splat (i64 8)
-; UF2-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UF2-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; UF2-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; UF2-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; UF2-NEXT: [[EXTRACT_NEXT_ALM_PART25]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
+; UF2-NEXT: [[TMP26:%.*]] = extractelement <8 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; UF2-NEXT: [[TMP45:%.*]] = xor i1 [[TMP26]], true
+; UF2-NEXT: br i1 [[TMP45]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; UF2: [[MIDDLE_BLOCK]]:
; UF2-NEXT: [[TMP27:%.*]] = select <8 x i1> [[TMP0]], <8 x i32> [[TMP14]], <8 x i32> [[VEC_PHI]]
; UF2-NEXT: [[TMP28:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> [[TMP15]], <8 x i32> [[VEC_PHI1]]
@@ -116,15 +114,18 @@ define i32 @many_reductions(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, i64
; UF4-NEXT: [[ENTRY:.*:]]
; UF4-NEXT: br label %[[VECTOR_PH:.*]]
; UF4: [[VECTOR_PH]]:
-; UF4-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 31
-; UF4-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 31
-; UF4-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; UF4-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; UF4-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; UF4-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; UF4-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 [[N]])
+; UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
+; UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
+; UF4-NEXT: [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 24)
; UF4-NEXT: br label %[[VECTOR_BODY:.*]]
; UF4: [[VECTOR_BODY]]:
; UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UF4-NEXT: [[TMP0:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], %[[VECTOR_BODY]] ]
+; UF4-NEXT: [[TMP1:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART53:%.*]], %[[VECTOR_BODY]] ]
+; UF4-NEXT: [[TMP2:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART2]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART54:%.*]], %[[VECTOR_BODY]] ]
+; UF4-NEXT: [[TMP3:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART3]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART55:%.*]], %[[VECTOR_BODY]] ]
; UF4-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
; UF4-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
; UF4-NEXT: [[VEC_PHI2:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
@@ -149,16 +150,6 @@ define i32 @many_reductions(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, i64
; UF4-NEXT: [[VEC_PHI21:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP49:%.*]], %[[VECTOR_BODY]] ]
; UF4-NEXT: [[VEC_PHI22:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP50:%.*]], %[[VECTOR_BODY]] ]
; UF4-NEXT: [[VEC_PHI23:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP51:%.*]], %[[VECTOR_BODY]] ]
-; UF4-NEXT: [[BROADCAST_SPLATINSERT24:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0
-; UF4-NEXT: [[BROADCAST_SPLAT25:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT24]], <8 x i64> poison, <8 x i32> zeroinitializer
-; UF4-NEXT: [[VEC_IV:%.*]] = add nuw <8 x i64> [[BROADCAST_SPLAT25]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
-; UF4-NEXT: [[VEC_IV28:%.*]] = add nuw <8 x i64> [[BROADCAST_SPLAT25]], <i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>
-; UF4-NEXT: [[VEC_IV31:%.*]] = add nuw <8 x i64> [[BROADCAST_SPLAT25]], <i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23>
-; UF4-NEXT: [[VEC_IV34:%.*]] = add nuw <8 x i64> [[BROADCAST_SPLAT25]], <i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>
-; UF4-NEXT: [[TMP0:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; UF4-NEXT: [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IV28]], [[BROADCAST_SPLAT]]
-; UF4-NEXT: [[TMP2:%.*]] = icmp ule <8 x i64> [[VEC_IV31]], [[BROADCAST_SPLAT]]
-; UF4-NEXT: [[TMP3:%.*]] = icmp ule <8 x i64> [[VEC_IV34]], [[BROADCAST_SPLAT]]
; UF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; UF4-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; UF4-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
@@ -232,8 +223,14 @@ define i32 @many_reductions(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, i64
; UF4-NEXT: [[TMP50]] = add <8 x i32> [[VEC_PHI22]], [[WIDE_MASKED_LOAD56]]
; UF4-NEXT: [[TMP51]] = add <8 x i32> [[VEC_PHI23]], [[WIDE_MASKED_LOAD57]]
; UF4-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 32
-; UF4-NEXT: [[TMP52:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UF4-NEXT: br i1 [[TMP52]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; UF4-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; UF4-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; UF4-NEXT: [[EXTRACT_NEXT_ALM_PART53]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
+; UF4-NEXT: [[EXTRACT_NEXT_ALM_PART54]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 16)
+; UF4-NEXT: [[EXTRACT_NEXT_ALM_PART55]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 24)
+; UF4-NEXT: [[TMP52:%.*]] = extractelement <8 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; UF4-NEXT: [[TMP83:%.*]] = xor i1 [[TMP52]], true
+; UF4-NEXT: br i1 [[TMP83]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; UF4: [[MIDDLE_BLOCK]]:
; UF4-NEXT: [[TMP53:%.*]] = select <8 x i1> [[TMP0]], <8 x i32> [[TMP28]], <8 x i32> [[VEC_PHI]]
; UF4-NEXT: [[TMP54:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> [[TMP29]], <8 x i32> [[VEC_PHI1]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
index 6f5b9d4403af12..123ee72235a7b8 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
@@ -413,19 +413,14 @@ define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %
; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; DISABLED_MASKED_STRIDED: [[VECTOR_PH]]:
-; DISABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; DISABLED_MASKED_STRIDED-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; DISABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
@@ -511,9 +506,11 @@ define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %
; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], %[[PRED_LOAD_IF15]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP2]])
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP54:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = xor i1 [[TMP54]], true
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP53]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; DISABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
; DISABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
@@ -531,19 +528,14 @@ define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %
; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; ENABLED_MASKED_STRIDED: [[VECTOR_PH]]:
-; ENABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; ENABLED_MASKED_STRIDED-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; ENABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = shl i32 [[INDEX]], 1
@@ -554,9 +546,11 @@ define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %
; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP6]], <8 x i1> [[TMP2]])
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = xor i1 [[TMP8]], true
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; ENABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
; ENABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
@@ -633,19 +627,14 @@ define void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture readonly %
; DISABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; DISABLED_MASKED_STRIDED: [[VECTOR_PH]]:
-; DISABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; DISABLED_MASKED_STRIDED-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; DISABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = mul nsw <8 x i32> [[VEC_IND]], splat (i32 3)
@@ -731,9 +720,11 @@ define void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture readonly %
; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], %[[PRED_LOAD_IF15]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP2]])
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP54:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = xor i1 [[TMP54]], true
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP53]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; DISABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
; DISABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
@@ -751,19 +742,14 @@ define void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture readonly %
; ENABLED_MASKED_STRIDED-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32
; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; ENABLED_MASKED_STRIDED: [[VECTOR_PH]]:
-; ENABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; ENABLED_MASKED_STRIDED-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; ENABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16:.*]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = mul nsw <8 x i32> [[VEC_IND]], splat (i32 3)
@@ -849,9 +835,11 @@ define void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture readonly %
; ENABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], %[[PRED_LOAD_IF15]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP2]])
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP54:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP53:%.*]] = xor i1 [[TMP54]], true
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP53]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; ENABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
; ENABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
@@ -1038,17 +1026,12 @@ define void @unconditional_strided1_optsize_unknown_tc(ptr noalias nocapture rea
; DISABLED_MASKED_STRIDED: [[FOR_BODY_PREHEADER]]:
; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; DISABLED_MASKED_STRIDED: [[VECTOR_PH]]:
-; DISABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; DISABLED_MASKED_STRIDED-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; DISABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE14:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_LOAD_CONTINUE14]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE14]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
@@ -1132,9 +1115,11 @@ define void @unconditional_strided1_optsize_unknown_tc(ptr noalias nocapture rea
; DISABLED_MASKED_STRIDED-NEXT: [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], %[[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], %[[PRED_LOAD_IF13]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP50:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP49]], ptr align 1 [[TMP50]], <8 x i1> [[TMP0]])
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP52:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP51:%.*]] = xor i1 [[TMP52]], true
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP51]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; DISABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
; DISABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
@@ -1151,17 +1136,11 @@ define void @unconditional_strided1_optsize_unknown_tc(ptr noalias nocapture rea
; ENABLED_MASKED_STRIDED: [[FOR_BODY_PREHEADER]]:
; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; ENABLED_MASKED_STRIDED: [[VECTOR_PH]]:
-; ENABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; ENABLED_MASKED_STRIDED-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; ENABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IV:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw i32 [[INDEX]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP1]]
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
@@ -1170,9 +1149,10 @@ define void @unconditional_strided1_optsize_unknown_tc(ptr noalias nocapture rea
; ENABLED_MASKED_STRIDED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP4]], <8 x i1> [[TMP0]])
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw <8 x i32> [[VEC_IV]], splat (i32 8)
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = xor i1 [[TMP6]], true
; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; ENABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
; ENABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
@@ -2309,19 +2289,14 @@ define void @masked_strided2_unknown_tc(ptr noalias nocapture readonly %p, ptr n
; DISABLED_MASKED_STRIDED: [[FOR_BODY_PREHEADER]]:
; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; DISABLED_MASKED_STRIDED: [[VECTOR_PH]]:
-; DISABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; DISABLED_MASKED_STRIDED-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; DISABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[GUARD]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE62:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE62]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE62]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp sgt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
; DISABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
@@ -2609,10 +2584,12 @@ define void @masked_strided2_unknown_tc(ptr noalias nocapture readonly %p, ptr n
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP158]], ptr [[TMP157]], align 1
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE62]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE62]]:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP159]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP159:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP160:%.*]] = xor i1 [[TMP159]], true
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP160]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; DISABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
; DISABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
; DISABLED_MASKED_STRIDED: [[FOR_END_LOOPEXIT]]:
@@ -2628,19 +2605,14 @@ define void @masked_strided2_unknown_tc(ptr noalias nocapture readonly %p, ptr n
; ENABLED_MASKED_STRIDED: [[FOR_BODY_PREHEADER]]:
; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; ENABLED_MASKED_STRIDED: [[VECTOR_PH]]:
-; ENABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; ENABLED_MASKED_STRIDED-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; ENABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[GUARD]], i64 0
; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = icmp sgt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
; ENABLED_MASKED_STRIDED-NEXT: [[TMP3:%.*]] = shl i32 [[INDEX]], 1
@@ -2656,10 +2628,12 @@ define void @masked_strided2_unknown_tc(ptr noalias nocapture readonly %p, ptr n
; ENABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = shufflevector <8 x i8> [[TMP5]], <8 x i8> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i8> [[TMP9]], <16 x i8> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP6]], <16 x i1> [[INTERLEAVED_MASK]])
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP11:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true
+; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; ENABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
; ENABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
; ENABLED_MASKED_STRIDED: [[FOR_END_LOOPEXIT]]:
@@ -2742,17 +2716,12 @@ define void @unconditional_masked_strided2_unknown_tc(ptr noalias nocapture read
; DISABLED_MASKED_STRIDED: [[FOR_BODY_PREHEADER]]:
; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; DISABLED_MASKED_STRIDED: [[VECTOR_PH]]:
-; DISABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; DISABLED_MASKED_STRIDED-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; DISABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; DISABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE60:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE60]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE60]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
@@ -3038,10 +3007,12 @@ define void @unconditional_masked_strided2_unknown_tc(ptr noalias nocapture read
; DISABLED_MASKED_STRIDED-NEXT: store i8 [[TMP156]], ptr [[TMP155]], align 1
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE60]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE60]]:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP157]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP157:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP158:%.*]] = xor i1 [[TMP157]], true
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP158]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; DISABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
; DISABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
; DISABLED_MASKED_STRIDED: [[FOR_END_LOOPEXIT]]:
@@ -3057,17 +3028,11 @@ define void @unconditional_masked_strided2_unknown_tc(ptr noalias nocapture read
; ENABLED_MASKED_STRIDED: [[FOR_BODY_PREHEADER]]:
; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; ENABLED_MASKED_STRIDED: [[VECTOR_PH]]:
-; ENABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; ENABLED_MASKED_STRIDED-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; ENABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; ENABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = shl nuw nsw i32 [[INDEX]], 1
; ENABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP1]]
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
@@ -3081,10 +3046,11 @@ define void @unconditional_masked_strided2_unknown_tc(ptr noalias nocapture read
; ENABLED_MASKED_STRIDED-NEXT: [[TMP7:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> [[TMP5]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP4]], <16 x i1> [[INTERLEAVED_MASK]])
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw <8 x i32> [[VEC_IND]], splat (i32 8)
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; ENABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
; ENABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
; ENABLED_MASKED_STRIDED: [[FOR_END_LOOPEXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
index ce1b579dba671a..986d4692f93821 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
@@ -137,19 +137,15 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: [[CMP15:%.*]] = icmp sgt i32 [[NUMPOINTS]], 0
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[CMP15]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
; DISABLED_MASKED_STRIDED: [[FOR_BODY_PREHEADER]]:
-; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
-; DISABLED_MASKED_STRIDED: [[VECTOR_PH]]:
; DISABLED_MASKED_STRIDED-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[NUMPOINTS]] to i64
-; DISABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw nsw i64 [[WIDE_TRIP_COUNT]], 3
-; DISABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 4294967292
-; DISABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
-; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; DISABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE15:.*]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE15]] ]
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; DISABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[WIDE_TRIP_COUNT]])
+; DISABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY1:.*]]
+; DISABLED_MASKED_STRIDED: [[VECTOR_BODY1]]:
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE15:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_BODY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE15]] ]
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE15]] ]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 [[INDEX]]
; DISABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP1]], <4 x i1> [[TMP0]], <4 x i16> poison)
; DISABLED_MASKED_STRIDED-NEXT: [[TMP2:%.*]] = shl nsw <4 x i64> [[VEC_IND]], splat (i64 2)
@@ -157,7 +153,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP4]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP4]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP6:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP6]], ptr [[TMP5]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE]]
@@ -166,7 +162,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF1]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP8]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP8]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP10]], ptr [[TMP9]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE2]]
@@ -175,7 +171,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF3]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP12]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP12]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP14:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP14]], ptr [[TMP13]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE4]]
@@ -184,7 +180,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF5]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP16]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP16]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP18:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP18]], ptr [[TMP17]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE6]]
@@ -195,7 +191,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF8]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP20]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP21]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP21]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP23:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 0
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP23]], ptr [[TMP22]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE9]]
@@ -203,7 +199,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF10]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[TMP20]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP24]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP24]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP26:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 1
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP26]], ptr [[TMP25]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE11]]
@@ -211,7 +207,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF12]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[TMP20]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP27]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP27]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP29:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 2
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP29]], ptr [[TMP28]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE13]]
@@ -219,15 +215,16 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_IF14]]:
; DISABLED_MASKED_STRIDED-NEXT: [[TMP30:%.*]] = extractelement <4 x i64> [[TMP20]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP30]]
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP30]]
; DISABLED_MASKED_STRIDED-NEXT: [[TMP32:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 3
; DISABLED_MASKED_STRIDED-NEXT: store i16 [[TMP32]], ptr [[TMP31]], align 2
; DISABLED_MASKED_STRIDED-NEXT: br label %[[PRED_STORE_CONTINUE15]]
; DISABLED_MASKED_STRIDED: [[PRED_STORE_CONTINUE15]]:
-; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; DISABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; DISABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[WIDE_TRIP_COUNT]])
+; DISABLED_MASKED_STRIDED-NEXT: [[TMP33:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; DISABLED_MASKED_STRIDED-NEXT: br i1 [[TMP33]], label %[[VECTOR_BODY1]], label %[[MIDDLE_BLOCK:.*]], !llvm.loop [[LOOP3:![0-9]+]]
; DISABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
; DISABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
; DISABLED_MASKED_STRIDED: [[FOR_END_LOOPEXIT]]:
@@ -241,19 +238,14 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; ENABLED_MASKED_STRIDED-NEXT: [[CMP15:%.*]] = icmp sgt i32 [[NUMPOINTS]], 0
; ENABLED_MASKED_STRIDED-NEXT: br i1 [[CMP15]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
; ENABLED_MASKED_STRIDED: [[FOR_BODY_PREHEADER]]:
+; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[NUMPOINTS]] to i64
; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_PH:.*]]
; ENABLED_MASKED_STRIDED: [[VECTOR_PH]]:
-; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[NUMPOINTS]] to i64
-; ENABLED_MASKED_STRIDED-NEXT: [[N_RND_UP:%.*]] = add nuw nsw i64 [[WIDE_TRIP_COUNT]], 3
-; ENABLED_MASKED_STRIDED-NEXT: [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 4294967292
-; ENABLED_MASKED_STRIDED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
-; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[WIDE_TRIP_COUNT]])
; ENABLED_MASKED_STRIDED-NEXT: br label %[[VECTOR_BODY:.*]]
; ENABLED_MASKED_STRIDED: [[VECTOR_BODY]]:
; ENABLED_MASKED_STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ENABLED_MASKED_STRIDED-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 [[INDEX]]
; ENABLED_MASKED_STRIDED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP1]], <4 x i1> [[TMP0]], <4 x i16> poison)
; ENABLED_MASKED_STRIDED-NEXT: [[DOTIDX:%.*]] = shl nsw i64 [[INDEX]], 3
@@ -264,10 +256,10 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
; ENABLED_MASKED_STRIDED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <4 x i1> [[TMP0]], <4 x i1> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
; ENABLED_MASKED_STRIDED-NEXT: [[TMP4:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>
; ENABLED_MASKED_STRIDED-NEXT: call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr align 2 [[TMP2]], <16 x i1> [[TMP4]])
-; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; ENABLED_MASKED_STRIDED-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; ENABLED_MASKED_STRIDED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; ENABLED_MASKED_STRIDED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[WIDE_TRIP_COUNT]])
+; ENABLED_MASKED_STRIDED-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT: br i1 [[TMP5]], label %[[VECTOR_BODY]], label %[[MIDDLE_BLOCK:.*]], !llvm.loop [[LOOP3:![0-9]+]]
; ENABLED_MASKED_STRIDED: [[MIDDLE_BLOCK]]:
; ENABLED_MASKED_STRIDED-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
; ENABLED_MASKED_STRIDED: [[FOR_END_LOOPEXIT]]:
>From 2fcd8842688c3fd9fdde894460d0151f66c4780a Mon Sep 17 00:00:00 2001
From: william <we3223 at gmail.com>
Date: Mon, 5 Oct 2026 23:05:38 +0800
Subject: [PATCH 4/5] [X86][TTI] Enable DataAndControlFlow tail folding style
on AVX-512-1
---
llvm/lib/Target/X86/X86TargetTransformInfo.h | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index f87cd2fee49271..47476fa2770c05 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -57,9 +57,8 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
/// \name Vector TTI Implementations
/// @{
-
- TailFoldingStyle
- getPreferredTailFoldingStyle() const {
+
+ TailFoldingStyle getPreferredTailFoldingStyle() const override {
return ST->hasAVX512() ? TailFoldingStyle::DataAndControlFlow
: TailFoldingStyle::DataWithoutLaneMask;
}
>From 0632ecfb74092c5633200487656136ec6d2aa4cb Mon Sep 17 00:00:00 2001
From: william <we3223 at gmail.com>
Date: Fri, 9 Oct 2026 15:57:40 +0800
Subject: [PATCH 5/5] [Trigger CI] Re-run pre-merge tests
---
.../X86/vectorize-interleaved-accesses-gap.ll | 70 +++++++++++--------
1 file changed, 40 insertions(+), 30 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
index c9a74ae42a8382..296a1ad7435dc4 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
@@ -1,23 +1,24 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -passes=loop-vectorize -enable-masked-interleaved-mem-accesses -enable-interleaved-mem-accesses -tail-folding-policy=must-fold-tail -force-vector-width=8 -S %s | FileCheck %s
target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx13.0.0"
define void @test_pr59090(ptr %l_out, ptr noalias %b) #0 {
-; CHECK-LABEL: @test_pr59090(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define void @test_pr59090(
+; CHECK-SAME: ptr [[L_OUT:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 10001)
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = mul nuw i64 [[INDEX]], 6
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[B:%.*]], align 1, !llvm.access.group [[ACC_GRP0:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0:![0-9]+]]
; CHECK-NEXT: store i8 [[TMP3]], ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0]]
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[L_OUT:%.*]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[L_OUT]], i64 [[TMP2]]
; CHECK-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> poison, <48 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
; CHECK-NEXT: [[TMP15:%.*]] = and <48 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false>
; CHECK-NEXT: call void @llvm.masked.store.v48i8.p0(<48 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison>, ptr align 1 [[TMP13]], <48 x i1> [[TMP15]]), !llvm.access.group [[ACC_GRP0]]
@@ -25,10 +26,10 @@ define void @test_pr59090(ptr %l_out, ptr noalias %b) #0 {
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT]], i64 10001)
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP5:%.*]] = xor i1 [[TMP4]], true
-; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[LOOP:%.*]]
-; CHECK: exit:
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
@@ -53,24 +54,25 @@ exit:
}
define void @test_pr59090_interleave(ptr %l_out, ptr noalias %b) #0 {
-; CHECK-LABEL: @test_pr59090_interleave(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define void @test_pr59090_interleave(
+; CHECK-SAME: ptr [[L_OUT:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 10001)
; CHECK-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
; CHECK-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[INDEX]], 6
; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP0]], 6
-; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[B:%.*]], align 1, !llvm.access.group [[ACC_GRP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0]]
; CHECK-NEXT: store i8 [[TMP5]], ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[L_OUT:%.*]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[L_OUT]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[L_OUT]], i64 [[TMP4]]
; CHECK-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> poison, <48 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
; CHECK-NEXT: [[TMP8:%.*]] = and <48 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false>
@@ -84,10 +86,10 @@ define void @test_pr59090_interleave(ptr %l_out, ptr noalias %b) #0 {
; CHECK-NEXT: [[EXTRACT_NEXT_ALM_PART4]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true
-; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[EXIT:%.*]]
-; CHECK: exit:
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
@@ -118,3 +120,11 @@ attributes #0 = { "target-cpu"="skx" }
!3 = !{!"llvm.loop.parallel_accesses", !0}
!4 = distinct !{!4, !3, !5}
!5 = !{!"llvm.loop.interleave.count", i32 2}
+;.
+; CHECK: [[ACC_GRP0]] = distinct !{}
+; CHECK: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]], [[META4:![0-9]+]]}
+; CHECK: [[META2]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP0]]}
+; CHECK: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META3]], [[META4]]}
+;.
More information about the llvm-commits
mailing list