[llvm] [X86] Improve llvm.get.active.lane.mask codegen (PR #225832)

via llvm-commits llvm-commits at lists.llvm.org
Fri Oct 9 01:27:33 PDT 2026


https://github.com/houngkoungting updated https://github.com/llvm/llvm-project/pull/225832

>From ee9343085718f3be84634632716b7306d3988c03 Mon Sep 17 00:00:00 2001
From: william <we3223 at gmail.com>
Date: Thu, 24 Sep 2026 00:07:36 +0800
Subject: [PATCH 1/5] [CodeGen] Clean up scalar bitmask type calculation in
 TargetLowering

---
 llvm/include/llvm/CodeGen/TargetLowering.h    |   18 +
 .../SelectionDAG/SelectionDAGBuilder.cpp      |    8 +
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |   45 +
 llvm/lib/Target/X86/X86ISelLowering.cpp       |   19 +
 llvm/lib/Target/X86/X86ISelLowering.h         |    3 +
 llvm/test/CodeGen/X86/active_lane_mask.ll     | 3241 +++--------------
 llvm/test/CodeGen/X86/pr47299.ll              |   70 +-
 7 files changed, 635 insertions(+), 2769 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 46521fa8c544a6..1dfd2b63ca2fbf 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -5759,6 +5759,24 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
   /// \returns The expansion if successful, SDValue() otherwise
   SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const;
 
+  /// Expand get_active_lane_mask into a scalar bitmask sequence.
+  /// \param DAG The SelectionDAG.
+  /// \param VT The target vector boolean type.
+  /// \param Index The loop iteration index.
+  /// \param TripCount The loop trip count.
+  /// \param DL The source location.
+  /// \returns The expanded SDValue, or an empty SDValue if expansion is not
+  /// supported.
+  SDValue expandGetActiveLaneMask(SelectionDAG &DAG, EVT VT, SDValue Index,
+                                  SDValue TripCount, const SDLoc &DL) const;
+
+  /// Return true if the target prefers to expand get_active_lane_mask
+  /// into a scalar bitmask sequence.
+  /// \param VT The vector mask type.
+  virtual bool shouldExpandGetActiveLaneMaskUsingScalar(EVT VT) const {
+    return false;
+  }
+
   /// Expand shift-by-parts.
   /// \param N Node to expand
   /// \param Lo lower-output-part after conversion
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index f7eec8c5088d48..b19dff3a87f422 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -8541,6 +8541,14 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     SDValue Index = getValue(I.getOperand(0));
     SDValue TripCount = getValue(I.getOperand(1));
     EVT ElementVT = Index.getValueType();
+    // Attempt to expand into a scalar bitmask if preferred by target.
+    if (TLI.shouldExpandGetActiveLaneMaskUsingScalar(CCVT)) {
+      if (SDValue Res =
+              TLI.expandGetActiveLaneMask(DAG, CCVT, Index, TripCount, sdl)) {
+        setValue(&I, Res);
+        return;
+      }
+    }
 
     if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
       setValue(&I, DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, sdl, CCVT, Index,
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 1eeab43bd480e1..f00c371af10097 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -8768,6 +8768,51 @@ static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
       /*AllowUndefs=*/true, /*AllowTruncation=*/true);
 }
 
+SDValue TargetLowering::expandGetActiveLaneMask(SelectionDAG &DAG, EVT VT,
+                                                SDValue Index,
+                                                SDValue TripCount,
+                                                const SDLoc &DL) const {
+  if (!VT.isFixedLengthVector())
+    return SDValue();
+
+  unsigned NumElts = VT.getVectorNumElements();
+
+  if (NumElts > 64)
+    return SDValue();
+
+  EVT IndexVT = Index.getValueType();
+  // Scalar integer type wide enough to hold all lane bits (at least i8).
+  unsigned MaskBits = llvm::PowerOf2Ceil(std::max(8u, NumElts));
+  EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), MaskBits);
+
+  // Clamp remaining lanes: Diff = (TripCount <= Index) ? 0 : (TripCount -
+  // Index).
+  SDValue Sub = DAG.getNode(ISD::SUB, DL, IndexVT, TripCount, Index);
+  SDValue IsLe = DAG.getSetCC(
+      DL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), IndexVT),
+      TripCount, Index, ISD::SETULE);
+  SDValue Zero = DAG.getConstant(0, DL, IndexVT);
+  SDValue Diff = DAG.getSelect(DL, IndexVT, IsLe, Zero, Sub);
+  // Clamp to vector length: k = umin(Diff, NumElts).
+  SDValue MaxLanes = DAG.getConstant(NumElts, DL, IndexVT);
+  SDValue Clamped = DAG.getNode(ISD::UMIN, DL, IndexVT, Diff, MaxLanes);
+  // Build the scalar bitmask: (1 << k) - 1.
+  SDValue ShiftAmt = DAG.getZExtOrTrunc(Clamped, DL, MaskVT);
+  SDValue One = DAG.getConstant(1, DL, MaskVT);
+  SDValue Shl = DAG.getNode(ISD::SHL, DL, MaskVT, One, ShiftAmt);
+  SDValue ScalarMask = DAG.getNode(ISD::SUB, DL, MaskVT, Shl, One);
+  // Bitcast to a boolean vector matching the integer bit width.
+  EVT BoolVecVT =
+      EVT::getVectorVT(*DAG.getContext(), MVT::i1, MaskVT.getSizeInBits());
+  SDValue Bitcast = DAG.getNode(ISD::BITCAST, DL, BoolVecVT, ScalarMask);
+
+  if (BoolVecVT == VT)
+    return Bitcast;
+  // Extract the original subvector if padded (e.g. v4i1 from i8 -> v8i1).
+  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Bitcast,
+                     DAG.getVectorIdxConstant(0, DL));
+}
+
 SDValue TargetLowering::expandFunnelShift(SDNode *Node,
                                           SelectionDAG &DAG) const {
   EVT VT = Node->getValueType(0);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7817c86155e5b0..30ff5844739bc8 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -32768,6 +32768,25 @@ X86TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
                                  : AtomicExpansionKind::None;
 }
 
+bool X86TargetLowering::shouldExpandGetActiveLaneMaskUsingScalar(EVT VT) const {
+  if (!VT.isFixedLengthVector())
+    return false;
+
+  unsigned NumElts = VT.getVectorNumElements();
+
+  // Only consider power-of-2 vector lengths up to 64.
+  if (NumElts > 64 || !isPowerOf2_32(NumElts))
+    return false;
+
+  // AVX-512 natively supports mask registers (kmov/bzhi) for >=16.
+  if (Subtarget.hasAVX512())
+    return NumElts >= 16;
+
+  // Without AVX-512, bitcasting to vXi1 requires costly unpacking (e.g. pinsrb)
+  // on smaller vectors. Only expand v64i1 to avoid its 80+ vector instructions.
+  return NumElts == 64;
+}
+
 enum BitTestKind : unsigned {
   UndefBit,
   ConstantBit,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index ba04080573c421..caedf954ff068d 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -110,6 +110,9 @@ namespace llvm {
 
     unsigned getJumpTableEncoding() const override;
     bool useSoftFloat() const override;
+    /// Return true if X86 prefers to expand get_active_lane_mask into a scalar
+    /// bitmask sequence.
+    bool shouldExpandGetActiveLaneMaskUsingScalar(EVT VT) const override;
 
     void markLibCallAttributes(MachineFunction *MF, unsigned CC,
                                ArgListTy &Args) const override;
diff --git a/llvm/test/CodeGen/X86/active_lane_mask.ll b/llvm/test/CodeGen/X86/active_lane_mask.ll
index be4264daffef03..74a37c8ba6b4a8 100644
--- a/llvm/test/CodeGen/X86/active_lane_mask.ll
+++ b/llvm/test/CodeGen/X86/active_lane_mask.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64    | FileCheck %s --check-prefixes=SSE2
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE42
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64    | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE,SSE42
 ; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=AVX2
 ; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512
 
@@ -272,34 +272,34 @@ define <16 x i1> @lane_mask_v16i1_i32(i32 %index, i32 %TC) {
 ; SSE42-LABEL: lane_mask_v16i1_i32:
 ; SSE42:       # %bb.0:
 ; SSE42-NEXT:    movd %edi, %xmm0
-; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm0 = [4294967295,4294967294,4294967293,4294967292]
-; SSE42-NEXT:    pminud %xmm2, %xmm0
+; SSE42-NEXT:    pminud %xmm1, %xmm0
 ; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [0,1,2,3]
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [4294967291,4294967290,4294967289,4294967288]
-; SSE42-NEXT:    pminud %xmm2, %xmm1
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [4,5,6,7]
+; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [4294967291,4294967290,4294967289,4294967288]
+; SSE42-NEXT:    pminud %xmm1, %xmm2
+; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [4,5,6,7]
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [4294967287,4294967286,4294967285,4294967284]
-; SSE42-NEXT:    pminud %xmm2, %xmm3
+; SSE42-NEXT:    pminud %xmm1, %xmm3
 ; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [8,9,10,11]
 ; SSE42-NEXT:    movd %esi, %xmm4
-; SSE42-NEXT:    pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE42-NEXT:    pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [12,13,14,15]
-; SSE42-NEXT:    movdqa %xmm2, %xmm5
+; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [12,13,14,15]
+; SSE42-NEXT:    movdqa %xmm1, %xmm5
 ; SSE42-NEXT:    pmaxud %xmm4, %xmm5
-; SSE42-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm3, %xmm2
-; SSE42-NEXT:    pmaxud %xmm4, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE42-NEXT:    packssdw %xmm5, %xmm2
-; SSE42-NEXT:    movdqa %xmm1, %xmm3
+; SSE42-NEXT:    pcmpeqd %xmm1, %xmm5
+; SSE42-NEXT:    movdqa %xmm3, %xmm1
+; SSE42-NEXT:    pmaxud %xmm4, %xmm1
+; SSE42-NEXT:    pcmpeqd %xmm3, %xmm1
+; SSE42-NEXT:    packssdw %xmm5, %xmm1
+; SSE42-NEXT:    movdqa %xmm2, %xmm3
 ; SSE42-NEXT:    pmaxud %xmm4, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm3
+; SSE42-NEXT:    pcmpeqd %xmm2, %xmm3
 ; SSE42-NEXT:    pmaxud %xmm0, %xmm4
 ; SSE42-NEXT:    pcmpeqd %xmm0, %xmm4
 ; SSE42-NEXT:    packssdw %xmm3, %xmm4
-; SSE42-NEXT:    packsswb %xmm2, %xmm4
+; SSE42-NEXT:    packsswb %xmm1, %xmm4
 ; SSE42-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE42-NEXT:    pxor %xmm4, %xmm0
 ; SSE42-NEXT:    retq
@@ -329,13 +329,16 @@ define <16 x i1> @lane_mask_v16i1_i32(i32 %index, i32 %TC) {
 ;
 ; AVX512-LABEL: lane_mask_v16i1_i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd %esi, %zmm0
-; AVX512-NEXT:    vpbroadcastd %edi, %zmm1
-; AVX512-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512-NEXT:    vpcmpltud %zmm0, %zmm1, %k0
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    subl %edi, %esi
+; AVX512-NEXT:    cmovbel %eax, %esi
+; AVX512-NEXT:    cmpl $16, %esi
+; AVX512-NEXT:    movl $16, %eax
+; AVX512-NEXT:    cmovbl %esi, %eax
+; AVX512-NEXT:    movl $-1, %ecx
+; AVX512-NEXT:    bzhil %eax, %ecx, %eax
+; AVX512-NEXT:    kmovd %eax, %k0
 ; AVX512-NEXT:    vpmovm2b %k0, %xmm0
-; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %TC)
   ret <16 x i1> %active.lane.mask
@@ -405,20 +408,9 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
 ; SSE2-NEXT:    por %xmm4, %xmm6
 ; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [24,25,26,27]
-; SSE2-NEXT:    paddd %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    por %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm7
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    packssdw %xmm5, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [20,21,22,23]
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [24,25,26,27]
 ; SSE2-NEXT:    paddd %xmm2, %xmm5
 ; SSE2-NEXT:    movdqa %xmm5, %xmm6
 ; SSE2-NEXT:    pxor %xmm1, %xmm6
@@ -428,6 +420,17 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
 ; SSE2-NEXT:    pxor %xmm1, %xmm7
 ; SSE2-NEXT:    movdqa %xmm3, %xmm5
 ; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSE2-NEXT:    packssdw %xmm4, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [20,21,22,23]
+; SSE2-NEXT:    paddd %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm6
+; SSE2-NEXT:    pxor %xmm1, %xmm6
+; SSE2-NEXT:    movdqa %xmm0, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE2-NEXT:    por %xmm4, %xmm7
+; SSE2-NEXT:    pxor %xmm1, %xmm7
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
 ; SSE2-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [16,17,18,19]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm6
 ; SSE2-NEXT:    pxor %xmm1, %xmm6
@@ -435,8 +438,8 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
 ; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm0
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    packssdw %xmm5, %xmm3
-; SSE2-NEXT:    packsswb %xmm4, %xmm3
+; SSE2-NEXT:    packssdw %xmm4, %xmm3
+; SSE2-NEXT:    packsswb %xmm5, %xmm3
 ; SSE2-NEXT:    pmovmskb %xmm3, %edx
 ; SSE2-NEXT:    shll $16, %edx
 ; SSE2-NEXT:    orl %ecx, %edx
@@ -456,28 +459,28 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
 ; SSE42-NEXT:    movdqa %xmm2, %xmm3
 ; SSE42-NEXT:    pmaxud %xmm0, %xmm3
 ; SSE42-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [4294967287,4294967286,4294967285,4294967284]
-; SSE42-NEXT:    pminud %xmm1, %xmm4
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [8,9,10,11]
-; SSE42-NEXT:    movdqa %xmm4, %xmm2
-; SSE42-NEXT:    pmaxud %xmm0, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE42-NEXT:    packssdw %xmm3, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [4294967291,4294967290,4294967289,4294967288]
-; SSE42-NEXT:    pminud %xmm1, %xmm3
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [4,5,6,7]
-; SSE42-NEXT:    movdqa %xmm3, %xmm4
+; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [4294967287,4294967286,4294967285,4294967284]
+; SSE42-NEXT:    pminud %xmm1, %xmm2
+; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [8,9,10,11]
+; SSE42-NEXT:    movdqa %xmm2, %xmm4
 ; SSE42-NEXT:    pmaxud %xmm0, %xmm4
+; SSE42-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE42-NEXT:    packssdw %xmm3, %xmm4
+; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [4294967291,4294967290,4294967289,4294967288]
+; SSE42-NEXT:    pminud %xmm1, %xmm2
+; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [4,5,6,7]
+; SSE42-NEXT:    movdqa %xmm2, %xmm3
+; SSE42-NEXT:    pmaxud %xmm0, %xmm3
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [4294967295,4294967294,4294967293,4294967292]
 ; SSE42-NEXT:    pminud %xmm1, %xmm5
 ; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5 # [0,1,2,3]
-; SSE42-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSE42-NEXT:    movdqa %xmm5, %xmm3
-; SSE42-NEXT:    pmaxud %xmm0, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE42-NEXT:    packssdw %xmm4, %xmm3
-; SSE42-NEXT:    packsswb %xmm2, %xmm3
-; SSE42-NEXT:    pmovmskb %xmm3, %ecx
+; SSE42-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE42-NEXT:    movdqa %xmm5, %xmm2
+; SSE42-NEXT:    pmaxud %xmm0, %xmm2
+; SSE42-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE42-NEXT:    packssdw %xmm3, %xmm2
+; SSE42-NEXT:    packsswb %xmm4, %xmm2
+; SSE42-NEXT:    pmovmskb %xmm2, %ecx
 ; SSE42-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [4294967267,4294967266,4294967265,4294967264]
 ; SSE42-NEXT:    pminud %xmm1, %xmm2
@@ -546,15 +549,15 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
 ;
 ; AVX512-LABEL: lane_mask_v32i1_i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd %esi, %zmm0
-; AVX512-NEXT:    vpbroadcastd %edi, %zmm1
-; AVX512-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT:    vpcmpltud %zmm0, %zmm2, %k0
-; AVX512-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
-; AVX512-NEXT:    vpcmpltud %zmm0, %zmm1, %k1
-; AVX512-NEXT:    kunpckwd %k0, %k1, %k0
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    subl %edi, %esi
+; AVX512-NEXT:    cmovbel %eax, %esi
+; AVX512-NEXT:    cmpl $32, %esi
+; AVX512-NEXT:    movl $32, %eax
+; AVX512-NEXT:    cmovbl %esi, %eax
+; AVX512-NEXT:    movl $-1, %ecx
+; AVX512-NEXT:    bzhil %eax, %ecx, %eax
+; AVX512-NEXT:    kmovd %eax, %k0
 ; AVX512-NEXT:    vpmovm2b %k0, %ymm0
 ; AVX512-NEXT:    retq
   %active.lane.mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i32(i32 %index, i32 %TC)
@@ -562,434 +565,69 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
 }
 
 define <64 x i1> @lane_mask_v64i1_i32(i32 %index, i32 %TC) {
-; SSE2-LABEL: lane_mask_v64i1_i32:
+; SSE-LABEL: lane_mask_v64i1_i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorl %eax, %eax
+; SSE-NEXT:    subl %esi, %edx
+; SSE-NEXT:    cmoval %edx, %eax
+; SSE-NEXT:    cmpl $64, %eax
+; SSE-NEXT:    movl $64, %ecx
+; SSE-NEXT:    cmovbl %eax, %ecx
+; SSE-NEXT:    movl $1, %edx
+; SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
+; SSE-NEXT:    shlq %cl, %rdx
+; SSE-NEXT:    movq %rdi, %rax
+; SSE-NEXT:    decq %rdx
+; SSE-NEXT:    movq %rdx, (%rdi)
+; SSE-NEXT:    retq
+;
+; AVX2-LABEL: lane_mask_v64i1_i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    xorl %eax, %eax
+; AVX2-NEXT:    subl %esi, %edx
+; AVX2-NEXT:    cmovbel %eax, %edx
+; AVX2-NEXT:    cmpl $64, %edx
+; AVX2-NEXT:    movl $64, %ecx
+; AVX2-NEXT:    cmovbl %edx, %ecx
+; AVX2-NEXT:    movq %rdi, %rax
+; AVX2-NEXT:    movq $-1, %rdx
+; AVX2-NEXT:    bzhiq %rcx, %rdx, %rcx
+; AVX2-NEXT:    movq %rcx, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: lane_mask_v64i1_i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    subl %edi, %esi
+; AVX512-NEXT:    cmovbel %eax, %esi
+; AVX512-NEXT:    cmpl $64, %esi
+; AVX512-NEXT:    movl $64, %eax
+; AVX512-NEXT:    cmovbl %esi, %eax
+; AVX512-NEXT:    movq $-1, %rcx
+; AVX512-NEXT:    bzhiq %rax, %rcx, %rax
+; AVX512-NEXT:    kmovq %rax, %k0
+; AVX512-NEXT:    vpmovm2b %k0, %zmm0
+; AVX512-NEXT:    retq
+  %active.lane.mask = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 %index, i32 %TC)
+  ret <64 x i1> %active.lane.mask
+}
+
+;
+; i32 (zero index)
+;
+
+define <2 x i1> @lane_mask_v2i1_i32_zero(i32 %TC) {
+; SSE2-LABEL: lane_mask_v2i1_i32_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq %rdi, %rax
-; SSE2-NEXT:    movd %esi, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [12,13,14,15]
-; SSE2-NEXT:    paddd %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT:    pxor %xmm1, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [8,9,10,11]
-; SSE2-NEXT:    paddd %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
-; SSE2-NEXT:    por %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    packssdw %xmm6, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [4,5,6,7]
-; SSE2-NEXT:    paddd %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [0,1,2,3]
-; SSE2-NEXT:    paddd %xmm2, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm0, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    por %xmm6, %xmm8
-; SSE2-NEXT:    pxor %xmm1, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT:    packssdw %xmm5, %xmm6
-; SSE2-NEXT:    packsswb %xmm4, %xmm6
-; SSE2-NEXT:    pmovmskb %xmm6, %edx
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [28,29,30,31]
-; SSE2-NEXT:    paddd %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [24,25,26,27]
-; SSE2-NEXT:    paddd %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    por %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    packssdw %xmm5, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [20,21,22,23]
-; SSE2-NEXT:    paddd %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [16,17,18,19]
-; SSE2-NEXT:    paddd %xmm2, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm0, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    por %xmm6, %xmm8
-; SSE2-NEXT:    pxor %xmm1, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT:    packssdw %xmm5, %xmm6
-; SSE2-NEXT:    packsswb %xmm4, %xmm6
-; SSE2-NEXT:    pmovmskb %xmm6, %ecx
-; SSE2-NEXT:    shll $16, %ecx
-; SSE2-NEXT:    orl %edx, %ecx
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [44,45,46,47]
-; SSE2-NEXT:    paddd %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [40,41,42,43]
-; SSE2-NEXT:    paddd %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    por %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    packssdw %xmm5, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [36,37,38,39]
-; SSE2-NEXT:    paddd %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [32,33,34,35]
-; SSE2-NEXT:    paddd %xmm2, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm0, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    por %xmm6, %xmm8
-; SSE2-NEXT:    pxor %xmm1, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT:    packssdw %xmm5, %xmm6
-; SSE2-NEXT:    packsswb %xmm4, %xmm6
-; SSE2-NEXT:    pmovmskb %xmm6, %edx
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [60,61,62,63]
-; SSE2-NEXT:    paddd %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [56,57,58,59]
-; SSE2-NEXT:    paddd %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    por %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    packssdw %xmm5, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [52,53,54,55]
-; SSE2-NEXT:    paddd %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [48,49,50,51]
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    packssdw %xmm5, %xmm3
-; SSE2-NEXT:    packsswb %xmm4, %xmm3
-; SSE2-NEXT:    pmovmskb %xmm3, %esi
-; SSE2-NEXT:    shll $16, %esi
-; SSE2-NEXT:    orl %edx, %esi
-; SSE2-NEXT:    shlq $32, %rsi
-; SSE2-NEXT:    orq %rcx, %rsi
-; SSE2-NEXT:    movq %rsi, (%rdi)
+; SSE2-NEXT:    movd %edi, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSE42-LABEL: lane_mask_v64i1_i32:
+; SSE42-LABEL: lane_mask_v2i1_i32_zero:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    movq %rdi, %rax
-; SSE42-NEXT:    movd %edx, %xmm1
-; SSE42-NEXT:    movd %esi, %xmm0
-; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [4294967283,4294967282,4294967281,4294967280]
-; SSE42-NEXT:    pminud %xmm0, %xmm2
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [12,13,14,15]
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE42-NEXT:    movdqa %xmm2, %xmm3
-; SSE42-NEXT:    pmaxud %xmm1, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [4294967287,4294967286,4294967285,4294967284]
-; SSE42-NEXT:    pminud %xmm0, %xmm4
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [8,9,10,11]
-; SSE42-NEXT:    movdqa %xmm4, %xmm2
-; SSE42-NEXT:    pmaxud %xmm1, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE42-NEXT:    packssdw %xmm3, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [4294967291,4294967290,4294967289,4294967288]
-; SSE42-NEXT:    pminud %xmm0, %xmm3
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [4,5,6,7]
-; SSE42-NEXT:    movdqa %xmm3, %xmm4
-; SSE42-NEXT:    pmaxud %xmm1, %xmm4
-; SSE42-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [4294967295,4294967294,4294967293,4294967292]
-; SSE42-NEXT:    pminud %xmm0, %xmm3
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [0,1,2,3]
-; SSE42-NEXT:    movdqa %xmm3, %xmm5
-; SSE42-NEXT:    pmaxud %xmm1, %xmm5
-; SSE42-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSE42-NEXT:    packssdw %xmm4, %xmm5
-; SSE42-NEXT:    packsswb %xmm2, %xmm5
-; SSE42-NEXT:    pmovmskb %xmm5, %edx
-; SSE42-NEXT:    xorl $65535, %edx # imm = 0xFFFF
-; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [4294967267,4294967266,4294967265,4294967264]
-; SSE42-NEXT:    pminud %xmm0, %xmm2
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [28,29,30,31]
-; SSE42-NEXT:    movdqa %xmm2, %xmm3
-; SSE42-NEXT:    pmaxud %xmm1, %xmm3
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [4294967271,4294967270,4294967269,4294967268]
-; SSE42-NEXT:    pminud %xmm0, %xmm4
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [24,25,26,27]
-; SSE42-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT:    movdqa %xmm4, %xmm2
-; SSE42-NEXT:    pmaxud %xmm1, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [4294967275,4294967274,4294967273,4294967272]
-; SSE42-NEXT:    pminud %xmm0, %xmm4
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [20,21,22,23]
-; SSE42-NEXT:    packssdw %xmm3, %xmm2
-; SSE42-NEXT:    movdqa %xmm4, %xmm3
-; SSE42-NEXT:    pmaxud %xmm1, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [4294967279,4294967278,4294967277,4294967276]
-; SSE42-NEXT:    pminud %xmm0, %xmm4
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [16,17,18,19]
-; SSE42-NEXT:    movdqa %xmm4, %xmm5
-; SSE42-NEXT:    pmaxud %xmm1, %xmm5
-; SSE42-NEXT:    pcmpeqd %xmm4, %xmm5
-; SSE42-NEXT:    packssdw %xmm3, %xmm5
-; SSE42-NEXT:    packsswb %xmm2, %xmm5
-; SSE42-NEXT:    pmovmskb %xmm5, %ecx
-; SSE42-NEXT:    notl %ecx
-; SSE42-NEXT:    shll $16, %ecx
-; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [4294967251,4294967250,4294967249,4294967248]
-; SSE42-NEXT:    pminud %xmm0, %xmm2
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [44,45,46,47]
-; SSE42-NEXT:    orl %edx, %ecx
-; SSE42-NEXT:    movdqa %xmm2, %xmm3
-; SSE42-NEXT:    pmaxud %xmm1, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [4294967255,4294967254,4294967253,4294967252]
-; SSE42-NEXT:    pminud %xmm0, %xmm4
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [40,41,42,43]
-; SSE42-NEXT:    movdqa %xmm4, %xmm2
-; SSE42-NEXT:    pmaxud %xmm1, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE42-NEXT:    packssdw %xmm3, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [4294967259,4294967258,4294967257,4294967256]
-; SSE42-NEXT:    pminud %xmm0, %xmm3
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [36,37,38,39]
-; SSE42-NEXT:    movdqa %xmm3, %xmm4
-; SSE42-NEXT:    pmaxud %xmm1, %xmm4
-; SSE42-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [4294967263,4294967262,4294967261,4294967260]
-; SSE42-NEXT:    pminud %xmm0, %xmm3
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [32,33,34,35]
-; SSE42-NEXT:    movdqa %xmm3, %xmm5
-; SSE42-NEXT:    pmaxud %xmm1, %xmm5
-; SSE42-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSE42-NEXT:    packssdw %xmm4, %xmm5
-; SSE42-NEXT:    packsswb %xmm2, %xmm5
-; SSE42-NEXT:    pmovmskb %xmm5, %edx
-; SSE42-NEXT:    xorl $65535, %edx # imm = 0xFFFF
-; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [4294967235,4294967234,4294967233,4294967232]
-; SSE42-NEXT:    pminud %xmm0, %xmm2
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [60,61,62,63]
-; SSE42-NEXT:    movdqa %xmm2, %xmm3
-; SSE42-NEXT:    pmaxud %xmm1, %xmm3
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [4294967239,4294967238,4294967237,4294967236]
-; SSE42-NEXT:    pminud %xmm0, %xmm4
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [56,57,58,59]
-; SSE42-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT:    movdqa %xmm4, %xmm2
-; SSE42-NEXT:    pmaxud %xmm1, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [4294967243,4294967242,4294967241,4294967240]
-; SSE42-NEXT:    pminud %xmm0, %xmm4
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [52,53,54,55]
-; SSE42-NEXT:    packssdw %xmm3, %xmm2
-; SSE42-NEXT:    movdqa %xmm4, %xmm3
-; SSE42-NEXT:    pmaxud %xmm1, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE42-NEXT:    pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [48,49,50,51]
-; SSE42-NEXT:    pmaxud %xmm0, %xmm1
-; SSE42-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE42-NEXT:    packssdw %xmm3, %xmm1
-; SSE42-NEXT:    packsswb %xmm2, %xmm1
-; SSE42-NEXT:    pmovmskb %xmm1, %esi
-; SSE42-NEXT:    notl %esi
-; SSE42-NEXT:    shll $16, %esi
-; SSE42-NEXT:    orl %edx, %esi
-; SSE42-NEXT:    shlq $32, %rsi
-; SSE42-NEXT:    orq %rcx, %rsi
-; SSE42-NEXT:    movq %rsi, (%rdi)
-; SSE42-NEXT:    retq
-;
-; AVX2-LABEL: lane_mask_v64i1_i32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    movq %rdi, %rax
-; AVX2-NEXT:    vmovd %edx, %xmm0
-; AVX2-NEXT:    vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT:    vmovd %esi, %xmm1
-; AVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
-; AVX2-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
-; AVX2-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [24,25,26,27,28,29,30,31]
-; AVX2-NEXT:    vpmaxud %ymm0, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
-; AVX2-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3
-; AVX2-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [16,17,18,19,20,21,22,23]
-; AVX2-NEXT:    vpmaxud %ymm0, %ymm3, %ymm4
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpackssdw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[0,2,1,3]
-; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX2-NEXT:    vpxor %ymm2, %ymm3, %ymm3
-; AVX2-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm4
-; AVX2-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4 # [8,9,10,11,12,13,14,15]
-; AVX2-NEXT:    vpmaxud %ymm0, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm4, %ymm4
-; AVX2-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5
-; AVX2-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm5 # [0,1,2,3,4,5,6,7]
-; AVX2-NEXT:    vpmaxud %ymm0, %ymm5, %ymm6
-; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpackssdw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
-; AVX2-NEXT:    vpxor %ymm2, %ymm4, %ymm4
-; AVX2-NEXT:    vpacksswb %ymm3, %ymm4, %ymm3
-; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
-; AVX2-NEXT:    vpmovmskb %ymm3, %ecx
-; AVX2-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3
-; AVX2-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [56,57,58,59,60,61,62,63]
-; AVX2-NEXT:    vpmaxud %ymm0, %ymm3, %ymm4
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm4
-; AVX2-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4 # [48,49,50,51,52,53,54,55]
-; AVX2-NEXT:    vpmaxud %ymm0, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm4, %ymm4
-; AVX2-NEXT:    vpackssdw %ymm3, %ymm4, %ymm3
-; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
-; AVX2-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm4
-; AVX2-NEXT:    vpxor %ymm2, %ymm3, %ymm3
-; AVX2-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4 # [40,41,42,43,44,45,46,47]
-; AVX2-NEXT:    vpmaxud %ymm0, %ymm4, %ymm5
-; AVX2-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX2-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [32,33,34,35,36,37,38,39]
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm4, %ymm4
-; AVX2-NEXT:    vpmaxud %ymm0, %ymm1, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm1, %ymm0
-; AVX2-NEXT:    vpackssdw %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpacksswb %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpmovmskb %ymm0, %edx
-; AVX2-NEXT:    shlq $32, %rdx
-; AVX2-NEXT:    orq %rcx, %rdx
-; AVX2-NEXT:    movq %rdx, (%rdi)
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: lane_mask_v64i1_i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd %esi, %zmm0
-; AVX512-NEXT:    vpbroadcastd %edi, %zmm1
-; AVX512-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm3
-; AVX512-NEXT:    vpcmpltud %zmm0, %zmm2, %k0
-; AVX512-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm2 # [16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
-; AVX512-NEXT:    vpcmpltud %zmm0, %zmm2, %k1
-; AVX512-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT:    kunpckwd %k0, %k1, %k0
-; AVX512-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47]
-; AVX512-NEXT:    vpcmpltud %zmm0, %zmm2, %k1
-; AVX512-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63]
-; AVX512-NEXT:    vpcmpltud %zmm0, %zmm1, %k2
-; AVX512-NEXT:    kunpckwd %k1, %k2, %k1
-; AVX512-NEXT:    kunpckdq %k0, %k1, %k0
-; AVX512-NEXT:    vpmovm2b %k0, %zmm0
-; AVX512-NEXT:    retq
-  %active.lane.mask = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 %index, i32 %TC)
-  ret <64 x i1> %active.lane.mask
-}
-
-;
-; i32 (zero index)
-;
-
-define <2 x i1> @lane_mask_v2i1_i32_zero(i32 %TC) {
-; SSE2-LABEL: lane_mask_v2i1_i32_zero:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movd %edi, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: lane_mask_v2i1_i32_zero:
-; SSE42:       # %bb.0:
-; SSE42-NEXT:    movd %edi, %xmm0
+; SSE42-NEXT:    movd %edi, %xmm0
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; SSE42-NEXT:    movq {{.*#+}} xmm1 = [0,1,0,0]
 ; SSE42-NEXT:    pmaxud %xmm1, %xmm0
@@ -1180,10 +818,13 @@ define <16 x i1> @lane_mask_v16i1_i32_zero(i32 %TC) {
 ;
 ; AVX512-LABEL: lane_mask_v16i1_i32_zero:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd %edi, %zmm0
-; AVX512-NEXT:    vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
+; AVX512-NEXT:    cmpl $16, %edi
+; AVX512-NEXT:    movl $16, %eax
+; AVX512-NEXT:    cmovbl %edi, %eax
+; AVX512-NEXT:    movl $-1, %ecx
+; AVX512-NEXT:    bzhil %eax, %ecx, %eax
+; AVX512-NEXT:    kmovd %eax, %k0
 ; AVX512-NEXT:    vpmovm2b %k0, %xmm0
-; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 0, i32 %TC)
   ret <16 x i1> %active.lane.mask
@@ -1304,10 +945,12 @@ define <32 x i1> @lane_mask_v32i1_i32_zero(i32 %TC) {
 ;
 ; AVX512-LABEL: lane_mask_v32i1_i32_zero:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd %edi, %zmm0
-; AVX512-NEXT:    vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
-; AVX512-NEXT:    vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT:    kunpckwd %k0, %k1, %k0
+; AVX512-NEXT:    cmpl $32, %edi
+; AVX512-NEXT:    movl $32, %eax
+; AVX512-NEXT:    cmovbl %edi, %eax
+; AVX512-NEXT:    movl $-1, %ecx
+; AVX512-NEXT:    bzhil %eax, %ecx, %eax
+; AVX512-NEXT:    kmovd %eax, %k0
 ; AVX512-NEXT:    vpmovm2b %k0, %ymm0
 ; AVX512-NEXT:    retq
   %active.lane.mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i32(i32 0, i32 %TC)
@@ -1315,229 +958,38 @@ define <32 x i1> @lane_mask_v32i1_i32_zero(i32 %TC) {
 }
 
 define <64 x i1> @lane_mask_v64i1_i32_zero(i32 %TC) {
-; SSE2-LABEL: lane_mask_v64i1_i32_zero:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movd %esi, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    movq %rdi, %rax
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    packssdw %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    packssdw %xmm1, %xmm3
-; SSE2-NEXT:    packsswb %xmm2, %xmm3
-; SSE2-NEXT:    pmovmskb %xmm3, %edx
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    packssdw %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    packssdw %xmm1, %xmm3
-; SSE2-NEXT:    packsswb %xmm2, %xmm3
-; SSE2-NEXT:    pmovmskb %xmm3, %ecx
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    shll $16, %ecx
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    orl %edx, %ecx
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    packssdw %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    packsswb %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    pmovmskb %xmm1, %edx
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    packsswb %xmm2, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %esi
-; SSE2-NEXT:    shll $16, %esi
-; SSE2-NEXT:    orl %edx, %esi
-; SSE2-NEXT:    shlq $32, %rsi
-; SSE2-NEXT:    orq %rcx, %rsi
-; SSE2-NEXT:    movq %rsi, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: lane_mask_v64i1_i32_zero:
-; SSE42:       # %bb.0:
-; SSE42-NEXT:    movq %rdi, %rax
-; SSE42-NEXT:    movd %esi, %xmm0
-; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [12,13,14,15]
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pmaxud %xmm1, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [8,9,10,11]
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pmaxud %xmm1, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE42-NEXT:    packssdw %xmm2, %xmm3
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [4,5,6,7]
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pmaxud %xmm1, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [0,1,2,3]
-; SSE42-NEXT:    movdqa %xmm0, %xmm4
-; SSE42-NEXT:    pmaxud %xmm1, %xmm4
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE42-NEXT:    packssdw %xmm2, %xmm4
-; SSE42-NEXT:    packsswb %xmm3, %xmm4
-; SSE42-NEXT:    pmovmskb %xmm4, %edx
-; SSE42-NEXT:    xorl $65535, %edx # imm = 0xFFFF
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [28,29,30,31]
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pmaxud %xmm1, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [24,25,26,27]
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pmaxud %xmm1, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE42-NEXT:    packssdw %xmm2, %xmm3
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [20,21,22,23]
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pmaxud %xmm1, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [16,17,18,19]
-; SSE42-NEXT:    movdqa %xmm0, %xmm4
-; SSE42-NEXT:    pmaxud %xmm1, %xmm4
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE42-NEXT:    packssdw %xmm2, %xmm4
-; SSE42-NEXT:    packsswb %xmm3, %xmm4
-; SSE42-NEXT:    pmovmskb %xmm4, %ecx
-; SSE42-NEXT:    notl %ecx
-; SSE42-NEXT:    shll $16, %ecx
-; SSE42-NEXT:    orl %edx, %ecx
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [44,45,46,47]
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pmaxud %xmm1, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [40,41,42,43]
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pmaxud %xmm1, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE42-NEXT:    packssdw %xmm2, %xmm3
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [36,37,38,39]
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pmaxud %xmm1, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [32,33,34,35]
-; SSE42-NEXT:    movdqa %xmm0, %xmm4
-; SSE42-NEXT:    pmaxud %xmm1, %xmm4
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE42-NEXT:    packssdw %xmm2, %xmm4
-; SSE42-NEXT:    packsswb %xmm3, %xmm4
-; SSE42-NEXT:    pmovmskb %xmm4, %edx
-; SSE42-NEXT:    xorl $65535, %edx # imm = 0xFFFF
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [60,61,62,63]
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pmaxud %xmm1, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [56,57,58,59]
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pmaxud %xmm1, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm3
-; SSE42-NEXT:    packssdw %xmm2, %xmm3
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [52,53,54,55]
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pmaxud %xmm1, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [48,49,50,51]
-; SSE42-NEXT:    pmaxud %xmm1, %xmm0
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE42-NEXT:    packssdw %xmm2, %xmm0
-; SSE42-NEXT:    packsswb %xmm3, %xmm0
-; SSE42-NEXT:    pmovmskb %xmm0, %esi
-; SSE42-NEXT:    notl %esi
-; SSE42-NEXT:    shll $16, %esi
-; SSE42-NEXT:    orl %edx, %esi
-; SSE42-NEXT:    shlq $32, %rsi
-; SSE42-NEXT:    orq %rcx, %rsi
-; SSE42-NEXT:    movq %rsi, (%rdi)
-; SSE42-NEXT:    retq
+; SSE-LABEL: lane_mask_v64i1_i32_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpl $64, %esi
+; SSE-NEXT:    movl $64, %ecx
+; SSE-NEXT:    cmovbl %esi, %ecx
+; SSE-NEXT:    movl $1, %edx
+; SSE-NEXT:    # kill: def $cl killed $cl killed $ecx
+; SSE-NEXT:    shlq %cl, %rdx
+; SSE-NEXT:    movq %rdi, %rax
+; SSE-NEXT:    decq %rdx
+; SSE-NEXT:    movq %rdx, (%rdi)
+; SSE-NEXT:    retq
 ;
 ; AVX2-LABEL: lane_mask_v64i1_i32_zero:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    cmpl $64, %esi
+; AVX2-NEXT:    movl $64, %ecx
+; AVX2-NEXT:    cmovbl %esi, %ecx
 ; AVX2-NEXT:    movq %rdi, %rax
-; AVX2-NEXT:    vmovd %esi, %xmm0
-; AVX2-NEXT:    vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [24,25,26,27,28,29,30,31]
-; AVX2-NEXT:    vpmaxud %ymm1, %ymm0, %ymm2
-; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [16,17,18,19,20,21,22,23]
-; AVX2-NEXT:    vpmaxud %ymm2, %ymm0, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX2-NEXT:    vpxor %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [8,9,10,11,12,13,14,15]
-; AVX2-NEXT:    vpmaxud %ymm3, %ymm0, %ymm4
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm4, %ymm3
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,2,3,4,5,6,7]
-; AVX2-NEXT:    vpmaxud %ymm4, %ymm0, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpackssdw %ymm3, %ymm4, %ymm3
-; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
-; AVX2-NEXT:    vpxor %ymm2, %ymm3, %ymm3
-; AVX2-NEXT:    vpacksswb %ymm1, %ymm3, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT:    vpmovmskb %ymm1, %ecx
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [56,57,58,59,60,61,62,63]
-; AVX2-NEXT:    vpmaxud %ymm1, %ymm0, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm3, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [48,49,50,51,52,53,54,55]
-; AVX2-NEXT:    vpmaxud %ymm3, %ymm0, %ymm4
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm4, %ymm3
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm3, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT:    vpxor %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [40,41,42,43,44,45,46,47]
-; AVX2-NEXT:    vpmaxud %ymm3, %ymm0, %ymm4
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm4, %ymm3
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [32,33,34,35,36,37,38,39]
-; AVX2-NEXT:    vpmaxud %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpackssdw %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpmovmskb %ymm0, %edx
-; AVX2-NEXT:    shlq $32, %rdx
-; AVX2-NEXT:    orq %rcx, %rdx
-; AVX2-NEXT:    movq %rdx, (%rdi)
-; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    movq $-1, %rdx
+; AVX2-NEXT:    bzhiq %rcx, %rdx, %rcx
+; AVX2-NEXT:    movq %rcx, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: lane_mask_v64i1_i32_zero:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd %edi, %zmm0
-; AVX512-NEXT:    vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
-; AVX512-NEXT:    vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT:    vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k2
-; AVX512-NEXT:    kunpckwd %k0, %k1, %k0
-; AVX512-NEXT:    vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT:    kunpckwd %k2, %k1, %k1
-; AVX512-NEXT:    kunpckdq %k0, %k1, %k0
+; AVX512-NEXT:    cmpl $64, %edi
+; AVX512-NEXT:    movl $64, %eax
+; AVX512-NEXT:    cmovbl %edi, %eax
+; AVX512-NEXT:    movq $-1, %rcx
+; AVX512-NEXT:    bzhiq %rax, %rcx, %rax
+; AVX512-NEXT:    kmovq %rax, %k0
 ; AVX512-NEXT:    vpmovm2b %k0, %zmm0
 ; AVX512-NEXT:    retq
   %active.lane.mask = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 0, i32 %TC)
@@ -1948,195 +1400,195 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
 ; SSE2-LABEL: lane_mask_v16i1_i64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %rdi, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,1,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,1,0,1]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm0, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pxor %xmm0, %xmm6
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
-; SSE2-NEXT:    paddq %xmm4, %xmm2
+; SSE2-NEXT:    paddq %xmm5, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    movdqa %xmm6, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
+; SSE2-NEXT:    pand %xmm4, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    por %xmm2, %xmm1
 ; SSE2-NEXT:    por %xmm7, %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2,3]
-; SSE2-NEXT:    paddq %xmm4, %xmm3
+; SSE2-NEXT:    paddq %xmm5, %xmm3
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm6, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    por %xmm3, %xmm2
 ; SSE2-NEXT:    por %xmm8, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [4,5]
-; SSE2-NEXT:    paddq %xmm4, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [4,5]
+; SSE2-NEXT:    paddq %xmm5, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm5, %xmm7
+; SSE2-NEXT:    movdqa %xmm6, %xmm7
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm8, %xmm9
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm3
+; SSE2-NEXT:    por %xmm4, %xmm3
 ; SSE2-NEXT:    por %xmm9, %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [6,7]
-; SSE2-NEXT:    paddq %xmm4, %xmm7
-; SSE2-NEXT:    movdqa %xmm7, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm5, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm8
+; SSE2-NEXT:    paddq %xmm5, %xmm7
+; SSE2-NEXT:    movdqa %xmm7, %xmm4
+; SSE2-NEXT:    pxor %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm6, %xmm8
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm8
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm9, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm6
-; SSE2-NEXT:    por %xmm10, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm4
+; SSE2-NEXT:    por %xmm10, %xmm4
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [8,9]
-; SSE2-NEXT:    paddq %xmm4, %xmm8
+; SSE2-NEXT:    paddq %xmm5, %xmm8
 ; SSE2-NEXT:    movdqa %xmm8, %xmm7
 ; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm5, %xmm9
+; SSE2-NEXT:    movdqa %xmm6, %xmm9
 ; SSE2-NEXT:    pcmpgtd %xmm7, %xmm9
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm10, %xmm11
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm9[1,1,3,3]
 ; SSE2-NEXT:    por %xmm8, %xmm7
 ; SSE2-NEXT:    por %xmm11, %xmm7
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [10,11]
-; SSE2-NEXT:    paddq %xmm4, %xmm9
+; SSE2-NEXT:    paddq %xmm5, %xmm9
 ; SSE2-NEXT:    movdqa %xmm9, %xmm8
 ; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm5, %xmm10
+; SSE2-NEXT:    movdqa %xmm6, %xmm10
 ; SSE2-NEXT:    pcmpgtd %xmm8, %xmm10
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm8
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm8
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm8[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm11, %xmm12
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm10[1,1,3,3]
 ; SSE2-NEXT:    por %xmm9, %xmm8
 ; SSE2-NEXT:    por %xmm12, %xmm8
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [12,13]
-; SSE2-NEXT:    paddq %xmm4, %xmm10
+; SSE2-NEXT:    paddq %xmm5, %xmm10
 ; SSE2-NEXT:    movdqa %xmm10, %xmm9
 ; SSE2-NEXT:    pxor %xmm0, %xmm9
-; SSE2-NEXT:    movdqa %xmm5, %xmm11
+; SSE2-NEXT:    movdqa %xmm6, %xmm11
 ; SSE2-NEXT:    pcmpgtd %xmm9, %xmm11
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm9
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm9
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm9[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm12, %xmm13
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm11[1,1,3,3]
 ; SSE2-NEXT:    por %xmm10, %xmm9
 ; SSE2-NEXT:    por %xmm13, %xmm9
-; SSE2-NEXT:    paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [14,15]
-; SSE2-NEXT:    movdqa %xmm4, %xmm10
+; SSE2-NEXT:    paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5 # [14,15]
+; SSE2-NEXT:    movdqa %xmm5, %xmm10
 ; SSE2-NEXT:    pxor %xmm0, %xmm10
-; SSE2-NEXT:    movdqa %xmm5, %xmm11
+; SSE2-NEXT:    movdqa %xmm6, %xmm11
 ; SSE2-NEXT:    pcmpgtd %xmm10, %xmm11
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm10[1,1,3,3]
-; SSE2-NEXT:    pand %xmm12, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm10[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm10
 ; SSE2-NEXT:    por %xmm5, %xmm10
-; SSE2-NEXT:    movq %rsi, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
+; SSE2-NEXT:    por %xmm6, %xmm10
+; SSE2-NEXT:    movq %rsi, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,1,0,1]
 ; SSE2-NEXT:    pxor %xmm0, %xmm10
-; SSE2-NEXT:    pxor %xmm0, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm10
+; SSE2-NEXT:    pxor %xmm0, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm10
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm11, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm10, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm10, %xmm6
 ; SSE2-NEXT:    pxor %xmm0, %xmm9
-; SSE2-NEXT:    movdqa %xmm4, %xmm10
+; SSE2-NEXT:    movdqa %xmm5, %xmm10
 ; SSE2-NEXT:    pcmpgtd %xmm9, %xmm10
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm9
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm9
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm9[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm11, %xmm12
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
 ; SSE2-NEXT:    por %xmm12, %xmm9
-; SSE2-NEXT:    packssdw %xmm5, %xmm9
+; SSE2-NEXT:    packssdw %xmm6, %xmm9
 ; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm8
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm8
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm10, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm8, %xmm10
 ; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm7
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm8, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm5
-; SSE2-NEXT:    packssdw %xmm10, %xmm5
-; SSE2-NEXT:    packssdw %xmm9, %xmm5
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm4, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
+; SSE2-NEXT:    por %xmm7, %xmm6
+; SSE2-NEXT:    packssdw %xmm10, %xmm6
+; SSE2-NEXT:    packssdw %xmm9, %xmm6
+; SSE2-NEXT:    pxor %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm5, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm7
+; SSE2-NEXT:    por %xmm4, %xmm7
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm8, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    por %xmm9, %xmm3
 ; SSE2-NEXT:    packssdw %xmm7, %xmm3
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm4
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm5, %xmm0
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    packssdw %xmm6, %xmm0
+; SSE2-NEXT:    packssdw %xmm4, %xmm0
 ; SSE2-NEXT:    packssdw %xmm3, %xmm0
-; SSE2-NEXT:    packsswb %xmm5, %xmm0
+; SSE2-NEXT:    packsswb %xmm6, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: lane_mask_v16i1_i64:
@@ -2144,72 +1596,72 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
 ; SSE42-NEXT:    movq %rdi, %xmm0
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
-; SSE42-NEXT:    movdqa %xmm0, %xmm4
-; SSE42-NEXT:    pxor %xmm1, %xmm4
+; SSE42-NEXT:    movdqa %xmm0, %xmm5
+; SSE42-NEXT:    pxor %xmm1, %xmm5
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
 ; SSE42-NEXT:    paddq %xmm0, %xmm3
-; SSE42-NEXT:    movdqa %xmm3, %xmm5
-; SSE42-NEXT:    pxor %xmm1, %xmm5
-; SSE42-NEXT:    movdqa %xmm4, %xmm2
-; SSE42-NEXT:    pcmpgtq %xmm5, %xmm2
+; SSE42-NEXT:    movdqa %xmm3, %xmm4
+; SSE42-NEXT:    pxor %xmm1, %xmm4
+; SSE42-NEXT:    movdqa %xmm5, %xmm2
+; SSE42-NEXT:    pcmpgtq %xmm4, %xmm2
 ; SSE42-NEXT:    por %xmm3, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [2,3]
-; SSE42-NEXT:    paddq %xmm0, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm6
+; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [2,3]
+; SSE42-NEXT:    paddq %xmm0, %xmm4
+; SSE42-NEXT:    movdqa %xmm4, %xmm6
 ; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm4, %xmm3
+; SSE42-NEXT:    movdqa %xmm5, %xmm3
 ; SSE42-NEXT:    pcmpgtq %xmm6, %xmm3
-; SSE42-NEXT:    por %xmm5, %xmm3
+; SSE42-NEXT:    por %xmm4, %xmm3
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm6 = [4,5]
 ; SSE42-NEXT:    paddq %xmm0, %xmm6
 ; SSE42-NEXT:    movdqa %xmm6, %xmm7
 ; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm4, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm5
-; SSE42-NEXT:    por %xmm6, %xmm5
+; SSE42-NEXT:    movdqa %xmm5, %xmm4
+; SSE42-NEXT:    pcmpgtq %xmm7, %xmm4
+; SSE42-NEXT:    por %xmm6, %xmm4
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm7 = [6,7]
 ; SSE42-NEXT:    paddq %xmm0, %xmm7
 ; SSE42-NEXT:    movdqa %xmm7, %xmm8
 ; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm4, %xmm6
+; SSE42-NEXT:    movdqa %xmm5, %xmm6
 ; SSE42-NEXT:    pcmpgtq %xmm8, %xmm6
 ; SSE42-NEXT:    por %xmm7, %xmm6
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm8 = [8,9]
 ; SSE42-NEXT:    paddq %xmm0, %xmm8
 ; SSE42-NEXT:    movdqa %xmm8, %xmm9
 ; SSE42-NEXT:    pxor %xmm1, %xmm9
-; SSE42-NEXT:    movdqa %xmm4, %xmm7
+; SSE42-NEXT:    movdqa %xmm5, %xmm7
 ; SSE42-NEXT:    pcmpgtq %xmm9, %xmm7
 ; SSE42-NEXT:    por %xmm8, %xmm7
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm9 = [10,11]
 ; SSE42-NEXT:    paddq %xmm0, %xmm9
 ; SSE42-NEXT:    movdqa %xmm9, %xmm10
 ; SSE42-NEXT:    pxor %xmm1, %xmm10
-; SSE42-NEXT:    movdqa %xmm4, %xmm8
+; SSE42-NEXT:    movdqa %xmm5, %xmm8
 ; SSE42-NEXT:    pcmpgtq %xmm10, %xmm8
 ; SSE42-NEXT:    por %xmm9, %xmm8
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm9 = [12,13]
 ; SSE42-NEXT:    paddq %xmm0, %xmm9
 ; SSE42-NEXT:    movdqa %xmm9, %xmm10
 ; SSE42-NEXT:    pxor %xmm1, %xmm10
-; SSE42-NEXT:    movdqa %xmm4, %xmm11
+; SSE42-NEXT:    movdqa %xmm5, %xmm11
 ; SSE42-NEXT:    pcmpgtq %xmm10, %xmm11
 ; SSE42-NEXT:    por %xmm9, %xmm11
 ; SSE42-NEXT:    paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [14,15]
 ; SSE42-NEXT:    movdqa %xmm0, %xmm9
 ; SSE42-NEXT:    pxor %xmm1, %xmm9
-; SSE42-NEXT:    pcmpgtq %xmm9, %xmm4
-; SSE42-NEXT:    por %xmm0, %xmm4
+; SSE42-NEXT:    pcmpgtq %xmm9, %xmm5
+; SSE42-NEXT:    por %xmm0, %xmm5
 ; SSE42-NEXT:    movq %rsi, %xmm0
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE42-NEXT:    pxor %xmm1, %xmm4
+; SSE42-NEXT:    pxor %xmm1, %xmm5
 ; SSE42-NEXT:    pxor %xmm1, %xmm0
 ; SSE42-NEXT:    movdqa %xmm0, %xmm9
-; SSE42-NEXT:    pcmpgtq %xmm4, %xmm9
+; SSE42-NEXT:    pcmpgtq %xmm5, %xmm9
 ; SSE42-NEXT:    pxor %xmm1, %xmm11
-; SSE42-NEXT:    movdqa %xmm0, %xmm4
-; SSE42-NEXT:    pcmpgtq %xmm11, %xmm4
-; SSE42-NEXT:    packssdw %xmm9, %xmm4
+; SSE42-NEXT:    movdqa %xmm0, %xmm5
+; SSE42-NEXT:    pcmpgtq %xmm11, %xmm5
+; SSE42-NEXT:    packssdw %xmm9, %xmm5
 ; SSE42-NEXT:    pxor %xmm1, %xmm8
 ; SSE42-NEXT:    movdqa %xmm0, %xmm9
 ; SSE42-NEXT:    pcmpgtq %xmm8, %xmm9
@@ -2217,14 +1669,14 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
 ; SSE42-NEXT:    movdqa %xmm0, %xmm8
 ; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
 ; SSE42-NEXT:    packssdw %xmm9, %xmm8
-; SSE42-NEXT:    packssdw %xmm4, %xmm8
+; SSE42-NEXT:    packssdw %xmm5, %xmm8
 ; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm4
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm4
-; SSE42-NEXT:    pxor %xmm1, %xmm5
+; SSE42-NEXT:    movdqa %xmm0, %xmm5
+; SSE42-NEXT:    pcmpgtq %xmm6, %xmm5
+; SSE42-NEXT:    pxor %xmm1, %xmm4
 ; SSE42-NEXT:    movdqa %xmm0, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm5, %xmm6
-; SSE42-NEXT:    packssdw %xmm4, %xmm6
+; SSE42-NEXT:    pcmpgtq %xmm4, %xmm6
+; SSE42-NEXT:    packssdw %xmm5, %xmm6
 ; SSE42-NEXT:    pxor %xmm1, %xmm3
 ; SSE42-NEXT:    movdqa %xmm0, %xmm4
 ; SSE42-NEXT:    pcmpgtq %xmm3, %xmm4
@@ -2281,17 +1733,16 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
 ;
 ; AVX512-LABEL: lane_mask_v16i1_i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %rsi, %zmm0
-; AVX512-NEXT:    vpbroadcastq %rdi, %zmm1
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [0,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm2, %k0
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [8,9,10,11,12,13,14,15]
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm1, %k1
-; AVX512-NEXT:    kunpckbw %k0, %k1, %k0
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    subq %rdi, %rsi
+; AVX512-NEXT:    cmovaq %rsi, %rax
+; AVX512-NEXT:    cmpq $16, %rax
+; AVX512-NEXT:    movl $16, %ecx
+; AVX512-NEXT:    cmovbq %rax, %rcx
+; AVX512-NEXT:    movl $-1, %eax
+; AVX512-NEXT:    bzhil %ecx, %eax, %eax
+; AVX512-NEXT:    kmovd %eax, %k0
 ; AVX512-NEXT:    vpmovm2b %k0, %xmm0
-; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 %index, i64 %TC)
   ret <16 x i1> %active.lane.mask
@@ -2831,20 +2282,9 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
 ; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
 ; SSE42-NEXT:    por %xmm5, %xmm7
 ; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [20,21]
-; SSE42-NEXT:    paddq %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm5, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
 ; SSE42-NEXT:    movdqa %xmm3, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm5
-; SSE42-NEXT:    packssdw %xmm6, %xmm5
-; SSE42-NEXT:    movdqa {{.*#+}} xmm6 = [18,19]
+; SSE42-NEXT:    pcmpgtq %xmm7, %xmm5
+; SSE42-NEXT:    movdqa {{.*#+}} xmm6 = [20,21]
 ; SSE42-NEXT:    paddq %xmm2, %xmm6
 ; SSE42-NEXT:    movdqa %xmm6, %xmm7
 ; SSE42-NEXT:    pxor %xmm1, %xmm7
@@ -2854,6 +2294,17 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
 ; SSE42-NEXT:    pxor %xmm1, %xmm8
 ; SSE42-NEXT:    movdqa %xmm3, %xmm6
 ; SSE42-NEXT:    pcmpgtq %xmm8, %xmm6
+; SSE42-NEXT:    packssdw %xmm5, %xmm6
+; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [18,19]
+; SSE42-NEXT:    paddq %xmm2, %xmm5
+; SSE42-NEXT:    movdqa %xmm5, %xmm7
+; SSE42-NEXT:    pxor %xmm1, %xmm7
+; SSE42-NEXT:    movdqa %xmm0, %xmm8
+; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
+; SSE42-NEXT:    por %xmm5, %xmm8
+; SSE42-NEXT:    pxor %xmm1, %xmm8
+; SSE42-NEXT:    movdqa %xmm3, %xmm5
+; SSE42-NEXT:    pcmpgtq %xmm8, %xmm5
 ; SSE42-NEXT:    paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [16,17]
 ; SSE42-NEXT:    movdqa %xmm2, %xmm7
 ; SSE42-NEXT:    pxor %xmm1, %xmm7
@@ -2861,8 +2312,8 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
 ; SSE42-NEXT:    por %xmm2, %xmm0
 ; SSE42-NEXT:    pxor %xmm1, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm0, %xmm3
-; SSE42-NEXT:    packssdw %xmm6, %xmm3
 ; SSE42-NEXT:    packssdw %xmm5, %xmm3
+; SSE42-NEXT:    packssdw %xmm6, %xmm3
 ; SSE42-NEXT:    packsswb %xmm4, %xmm3
 ; SSE42-NEXT:    pmovmskb %xmm3, %edx
 ; SSE42-NEXT:    shll $16, %edx
@@ -2945,23 +2396,15 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
 ;
 ; AVX512-LABEL: lane_mask_v32i1_i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %rsi, %zmm0
-; AVX512-NEXT:    vpbroadcastq %rdi, %zmm1
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [0,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm3
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm2, %k0
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm2 # [8,9,10,11,12,13,14,15]
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm2, %k1
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT:    kunpckbw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [16,17,18,19,20,21,22,23]
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm2, %k1
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [24,25,26,27,28,29,30,31]
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm1, %k2
-; AVX512-NEXT:    kunpckbw %k1, %k2, %k1
-; AVX512-NEXT:    kunpckwd %k0, %k1, %k0
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    subq %rdi, %rsi
+; AVX512-NEXT:    cmovaq %rsi, %rax
+; AVX512-NEXT:    cmpq $32, %rax
+; AVX512-NEXT:    movl $32, %ecx
+; AVX512-NEXT:    cmovbq %rax, %rcx
+; AVX512-NEXT:    movl $-1, %eax
+; AVX512-NEXT:    bzhil %ecx, %eax, %eax
+; AVX512-NEXT:    kmovd %eax, %k0
 ; AVX512-NEXT:    vpmovm2b %k0, %ymm0
 ; AVX512-NEXT:    retq
   %active.lane.mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 %index, i64 %TC)
@@ -2969,1367 +2412,105 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
 }
 
 define <64 x i1> @lane_mask_v64i1_i64(i64 %index, i64 %TC) {
-; SSE2-LABEL: lane_mask_v64i1_i64:
+; SSE-LABEL: lane_mask_v64i1_i64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorl %eax, %eax
+; SSE-NEXT:    subq %rsi, %rdx
+; SSE-NEXT:    cmovaq %rdx, %rax
+; SSE-NEXT:    cmpq $64, %rax
+; SSE-NEXT:    movl $64, %ecx
+; SSE-NEXT:    cmovbq %rax, %rcx
+; SSE-NEXT:    movl $1, %edx
+; SSE-NEXT:    # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT:    shlq %cl, %rdx
+; SSE-NEXT:    movq %rdi, %rax
+; SSE-NEXT:    decq %rdx
+; SSE-NEXT:    movq %rdx, (%rdi)
+; SSE-NEXT:    retq
+;
+; AVX2-LABEL: lane_mask_v64i1_i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    xorl %eax, %eax
+; AVX2-NEXT:    subq %rsi, %rdx
+; AVX2-NEXT:    cmovaq %rdx, %rax
+; AVX2-NEXT:    cmpq $64, %rax
+; AVX2-NEXT:    movl $64, %ecx
+; AVX2-NEXT:    cmovbq %rax, %rcx
+; AVX2-NEXT:    movq %rdi, %rax
+; AVX2-NEXT:    movq $-1, %rdx
+; AVX2-NEXT:    bzhiq %rcx, %rdx, %rcx
+; AVX2-NEXT:    movq %rcx, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: lane_mask_v64i1_i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    subq %rdi, %rsi
+; AVX512-NEXT:    cmovaq %rsi, %rax
+; AVX512-NEXT:    cmpq $64, %rax
+; AVX512-NEXT:    movl $64, %ecx
+; AVX512-NEXT:    cmovbq %rax, %rcx
+; AVX512-NEXT:    movq $-1, %rax
+; AVX512-NEXT:    bzhiq %rcx, %rax, %rax
+; AVX512-NEXT:    kmovq %rax, %k0
+; AVX512-NEXT:    vpmovm2b %k0, %zmm0
+; AVX512-NEXT:    retq
+  %active.lane.mask = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 %index, i64 %TC)
+  ret <64 x i1> %active.lane.mask
+}
+
+;
+; i64 (zero index)
+;
+
+define <2 x i1> @lane_mask_v2i1_i64_zero(i64 %TC) {
+; SSE2-LABEL: lane_mask_v2i1_i64_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq %rdi, %rax
-; SSE2-NEXT:    movq %rsi, %xmm0
+; SSE2-NEXT:    movq %rdi, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; SSE2-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE42-LABEL: lane_mask_v2i1_i64_zero:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movq %rdi, %xmm0
+; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; SSE42-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE42-NEXT:    retq
+;
+; AVX2-LABEL: lane_mask_v2i1_i64_zero:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovq %rdi, %xmm0
+; AVX2-NEXT:    vpbroadcastq %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: lane_mask_v2i1_i64_zero:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpbroadcastq %rdi, %xmm0
+; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %xmm0
+; AVX512-NEXT:    retq
+  %active.lane.mask = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 %TC)
+  ret <2 x i1> %active.lane.mask
+}
+
+define <4 x i1> @lane_mask_v4i1_i64_zero(i64 %TC) {
+; SSE2-LABEL: lane_mask_v4i1_i64_zero:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq %rdi, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,0,1]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [14,15]
-; SSE2-NEXT:    paddq %xmm1, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm0, %xmm4
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm0, %xmm5
-; SSE2-NEXT:    movq %rdx, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,0,1]
-; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [12,13]
-; SSE2-NEXT:    paddq %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm5
-; SSE2-NEXT:    packssdw %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [10,11]
-; SSE2-NEXT:    paddq %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm7
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [8,9]
-; SSE2-NEXT:    paddq %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm8
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm4
-; SSE2-NEXT:    packssdw %xmm6, %xmm4
-; SSE2-NEXT:    packssdw %xmm5, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [6,7]
-; SSE2-NEXT:    paddq %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [4,5]
-; SSE2-NEXT:    paddq %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm8
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm5
-; SSE2-NEXT:    packssdw %xmm6, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2,3]
-; SSE2-NEXT:    paddq %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm8
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
-; SSE2-NEXT:    paddq %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm2, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm9
-; SSE2-NEXT:    por %xmm8, %xmm9
-; SSE2-NEXT:    pxor %xmm0, %xmm9
-; SSE2-NEXT:    movdqa %xmm3, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm9, %xmm7
-; SSE2-NEXT:    packssdw %xmm6, %xmm7
-; SSE2-NEXT:    packssdw %xmm5, %xmm7
-; SSE2-NEXT:    packsswb %xmm4, %xmm7
-; SSE2-NEXT:    pmovmskb %xmm7, %edx
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [30,31]
-; SSE2-NEXT:    paddq %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    por %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [28,29]
-; SSE2-NEXT:    paddq %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm5
-; SSE2-NEXT:    packssdw %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [26,27]
-; SSE2-NEXT:    paddq %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm7
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [24,25]
-; SSE2-NEXT:    paddq %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm8
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm4
-; SSE2-NEXT:    packssdw %xmm6, %xmm4
-; SSE2-NEXT:    packssdw %xmm5, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [22,23]
-; SSE2-NEXT:    paddq %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [20,21]
-; SSE2-NEXT:    paddq %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm8
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm5
-; SSE2-NEXT:    packssdw %xmm6, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [18,19]
-; SSE2-NEXT:    paddq %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm8
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [16,17]
-; SSE2-NEXT:    paddq %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm2, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm9
-; SSE2-NEXT:    por %xmm8, %xmm9
-; SSE2-NEXT:    pxor %xmm0, %xmm9
-; SSE2-NEXT:    movdqa %xmm3, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm9, %xmm7
-; SSE2-NEXT:    packssdw %xmm6, %xmm7
-; SSE2-NEXT:    packssdw %xmm5, %xmm7
-; SSE2-NEXT:    packsswb %xmm4, %xmm7
-; SSE2-NEXT:    pmovmskb %xmm7, %ecx
-; SSE2-NEXT:    shll $16, %ecx
-; SSE2-NEXT:    orl %edx, %ecx
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [46,47]
-; SSE2-NEXT:    paddq %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    por %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [44,45]
-; SSE2-NEXT:    paddq %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm5
-; SSE2-NEXT:    packssdw %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [42,43]
-; SSE2-NEXT:    paddq %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm7
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [40,41]
-; SSE2-NEXT:    paddq %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm8
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm4
-; SSE2-NEXT:    packssdw %xmm6, %xmm4
-; SSE2-NEXT:    packssdw %xmm5, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [38,39]
-; SSE2-NEXT:    paddq %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [36,37]
-; SSE2-NEXT:    paddq %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm8
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm5
-; SSE2-NEXT:    packssdw %xmm6, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [34,35]
-; SSE2-NEXT:    paddq %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm8
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [32,33]
-; SSE2-NEXT:    paddq %xmm1, %xmm7
-; SSE2-NEXT:    movdqa %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm2, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm9
-; SSE2-NEXT:    por %xmm8, %xmm9
-; SSE2-NEXT:    pxor %xmm0, %xmm9
-; SSE2-NEXT:    movdqa %xmm3, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm9, %xmm7
-; SSE2-NEXT:    packssdw %xmm6, %xmm7
-; SSE2-NEXT:    packssdw %xmm5, %xmm7
-; SSE2-NEXT:    packsswb %xmm4, %xmm7
-; SSE2-NEXT:    pmovmskb %xmm7, %edx
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [62,63]
-; SSE2-NEXT:    paddq %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    por %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [60,61]
-; SSE2-NEXT:    paddq %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm5
-; SSE2-NEXT:    packssdw %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [58,59]
-; SSE2-NEXT:    paddq %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm7
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [56,57]
-; SSE2-NEXT:    paddq %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm8
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm4
-; SSE2-NEXT:    packssdw %xmm6, %xmm4
-; SSE2-NEXT:    packssdw %xmm5, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [54,55]
-; SSE2-NEXT:    paddq %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [52,53]
-; SSE2-NEXT:    paddq %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm8
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm5
-; SSE2-NEXT:    packssdw %xmm6, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [50,51]
-; SSE2-NEXT:    paddq %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm8
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm6
-; SSE2-NEXT:    paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [48,49]
-; SSE2-NEXT:    movdqa %xmm1, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm7
-; SSE2-NEXT:    por %xmm2, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    packssdw %xmm6, %xmm0
-; SSE2-NEXT:    packssdw %xmm5, %xmm0
-; SSE2-NEXT:    packsswb %xmm4, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %esi
-; SSE2-NEXT:    shll $16, %esi
-; SSE2-NEXT:    orl %edx, %esi
-; SSE2-NEXT:    shlq $32, %rsi
-; SSE2-NEXT:    orq %rcx, %rsi
-; SSE2-NEXT:    movq %rsi, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: lane_mask_v64i1_i64:
-; SSE42:       # %bb.0:
-; SSE42-NEXT:    movq %rdi, %rax
-; SSE42-NEXT:    movq %rsi, %xmm0
-; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,0,1]
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
-; SSE42-NEXT:    movdqa %xmm2, %xmm0
-; SSE42-NEXT:    pxor %xmm1, %xmm0
-; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [14,15]
-; SSE42-NEXT:    paddq %xmm2, %xmm3
-; SSE42-NEXT:    movdqa %xmm3, %xmm4
-; SSE42-NEXT:    pxor %xmm1, %xmm4
-; SSE42-NEXT:    movdqa %xmm0, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm4, %xmm5
-; SSE42-NEXT:    por %xmm3, %xmm5
-; SSE42-NEXT:    pxor %xmm1, %xmm5
-; SSE42-NEXT:    movq %rdx, %xmm3
-; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,1,0,1]
-; SSE42-NEXT:    pxor %xmm1, %xmm3
-; SSE42-NEXT:    movdqa %xmm3, %xmm4
-; SSE42-NEXT:    pcmpgtq %xmm5, %xmm4
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [12,13]
-; SSE42-NEXT:    paddq %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT:    por %xmm5, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm5
-; SSE42-NEXT:    packssdw %xmm4, %xmm5
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [10,11]
-; SSE42-NEXT:    paddq %xmm2, %xmm4
-; SSE42-NEXT:    movdqa %xmm4, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT:    por %xmm4, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [8,9]
-; SSE42-NEXT:    paddq %xmm2, %xmm4
-; SSE42-NEXT:    movdqa %xmm4, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm4, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm3, %xmm4
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm4
-; SSE42-NEXT:    packssdw %xmm6, %xmm4
-; SSE42-NEXT:    packssdw %xmm5, %xmm4
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [6,7]
-; SSE42-NEXT:    paddq %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT:    por %xmm5, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [4,5]
-; SSE42-NEXT:    paddq %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm5, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm3, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm5
-; SSE42-NEXT:    packssdw %xmm6, %xmm5
-; SSE42-NEXT:    movdqa {{.*#+}} xmm6 = [2,3]
-; SSE42-NEXT:    paddq %xmm2, %xmm6
-; SSE42-NEXT:    movdqa %xmm6, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm6, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm6
-; SSE42-NEXT:    movdqa {{.*#+}} xmm7 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
-; SSE42-NEXT:    paddq %xmm2, %xmm7
-; SSE42-NEXT:    movdqa %xmm7, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm0, %xmm9
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm9
-; SSE42-NEXT:    por %xmm7, %xmm9
-; SSE42-NEXT:    pxor %xmm1, %xmm9
-; SSE42-NEXT:    movdqa %xmm3, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm9, %xmm7
-; SSE42-NEXT:    packssdw %xmm6, %xmm7
-; SSE42-NEXT:    packssdw %xmm5, %xmm7
-; SSE42-NEXT:    packsswb %xmm4, %xmm7
-; SSE42-NEXT:    pmovmskb %xmm7, %edx
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [30,31]
-; SSE42-NEXT:    paddq %xmm2, %xmm4
-; SSE42-NEXT:    movdqa %xmm4, %xmm5
-; SSE42-NEXT:    pxor %xmm1, %xmm5
-; SSE42-NEXT:    movdqa %xmm0, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm5, %xmm6
-; SSE42-NEXT:    por %xmm4, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm3, %xmm4
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm4
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [28,29]
-; SSE42-NEXT:    paddq %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT:    por %xmm5, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm5
-; SSE42-NEXT:    packssdw %xmm4, %xmm5
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [26,27]
-; SSE42-NEXT:    paddq %xmm2, %xmm4
-; SSE42-NEXT:    movdqa %xmm4, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT:    por %xmm4, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [24,25]
-; SSE42-NEXT:    paddq %xmm2, %xmm4
-; SSE42-NEXT:    movdqa %xmm4, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm4, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm3, %xmm4
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm4
-; SSE42-NEXT:    packssdw %xmm6, %xmm4
-; SSE42-NEXT:    packssdw %xmm5, %xmm4
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [22,23]
-; SSE42-NEXT:    paddq %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT:    por %xmm5, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [20,21]
-; SSE42-NEXT:    paddq %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm5, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm3, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm5
-; SSE42-NEXT:    packssdw %xmm6, %xmm5
-; SSE42-NEXT:    movdqa {{.*#+}} xmm6 = [18,19]
-; SSE42-NEXT:    paddq %xmm2, %xmm6
-; SSE42-NEXT:    movdqa %xmm6, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm6, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm6
-; SSE42-NEXT:    movdqa {{.*#+}} xmm7 = [16,17]
-; SSE42-NEXT:    paddq %xmm2, %xmm7
-; SSE42-NEXT:    movdqa %xmm7, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm0, %xmm9
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm9
-; SSE42-NEXT:    por %xmm7, %xmm9
-; SSE42-NEXT:    pxor %xmm1, %xmm9
-; SSE42-NEXT:    movdqa %xmm3, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm9, %xmm7
-; SSE42-NEXT:    packssdw %xmm6, %xmm7
-; SSE42-NEXT:    packssdw %xmm5, %xmm7
-; SSE42-NEXT:    packsswb %xmm4, %xmm7
-; SSE42-NEXT:    pmovmskb %xmm7, %ecx
-; SSE42-NEXT:    shll $16, %ecx
-; SSE42-NEXT:    orl %edx, %ecx
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [46,47]
-; SSE42-NEXT:    paddq %xmm2, %xmm4
-; SSE42-NEXT:    movdqa %xmm4, %xmm5
-; SSE42-NEXT:    pxor %xmm1, %xmm5
-; SSE42-NEXT:    movdqa %xmm0, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm5, %xmm6
-; SSE42-NEXT:    por %xmm4, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm3, %xmm4
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm4
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [44,45]
-; SSE42-NEXT:    paddq %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT:    por %xmm5, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm5
-; SSE42-NEXT:    packssdw %xmm4, %xmm5
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [42,43]
-; SSE42-NEXT:    paddq %xmm2, %xmm4
-; SSE42-NEXT:    movdqa %xmm4, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT:    por %xmm4, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [40,41]
-; SSE42-NEXT:    paddq %xmm2, %xmm4
-; SSE42-NEXT:    movdqa %xmm4, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm4, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm3, %xmm4
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm4
-; SSE42-NEXT:    packssdw %xmm6, %xmm4
-; SSE42-NEXT:    packssdw %xmm5, %xmm4
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [38,39]
-; SSE42-NEXT:    paddq %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT:    por %xmm5, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [36,37]
-; SSE42-NEXT:    paddq %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm5, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm3, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm5
-; SSE42-NEXT:    packssdw %xmm6, %xmm5
-; SSE42-NEXT:    movdqa {{.*#+}} xmm6 = [34,35]
-; SSE42-NEXT:    paddq %xmm2, %xmm6
-; SSE42-NEXT:    movdqa %xmm6, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm6, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm6
-; SSE42-NEXT:    movdqa {{.*#+}} xmm7 = [32,33]
-; SSE42-NEXT:    paddq %xmm2, %xmm7
-; SSE42-NEXT:    movdqa %xmm7, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm0, %xmm9
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm9
-; SSE42-NEXT:    por %xmm7, %xmm9
-; SSE42-NEXT:    pxor %xmm1, %xmm9
-; SSE42-NEXT:    movdqa %xmm3, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm9, %xmm7
-; SSE42-NEXT:    packssdw %xmm6, %xmm7
-; SSE42-NEXT:    packssdw %xmm5, %xmm7
-; SSE42-NEXT:    packsswb %xmm4, %xmm7
-; SSE42-NEXT:    pmovmskb %xmm7, %edx
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [62,63]
-; SSE42-NEXT:    paddq %xmm2, %xmm4
-; SSE42-NEXT:    movdqa %xmm4, %xmm5
-; SSE42-NEXT:    pxor %xmm1, %xmm5
-; SSE42-NEXT:    movdqa %xmm0, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm5, %xmm6
-; SSE42-NEXT:    por %xmm4, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm3, %xmm4
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm4
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [60,61]
-; SSE42-NEXT:    paddq %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT:    por %xmm5, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm5
-; SSE42-NEXT:    packssdw %xmm4, %xmm5
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [58,59]
-; SSE42-NEXT:    paddq %xmm2, %xmm4
-; SSE42-NEXT:    movdqa %xmm4, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT:    por %xmm4, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [56,57]
-; SSE42-NEXT:    paddq %xmm2, %xmm4
-; SSE42-NEXT:    movdqa %xmm4, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm4, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm3, %xmm4
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm4
-; SSE42-NEXT:    packssdw %xmm6, %xmm4
-; SSE42-NEXT:    packssdw %xmm5, %xmm4
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [54,55]
-; SSE42-NEXT:    paddq %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm6
-; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
-; SSE42-NEXT:    por %xmm5, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm6
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [52,53]
-; SSE42-NEXT:    paddq %xmm2, %xmm5
-; SSE42-NEXT:    movdqa %xmm5, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm5, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm3, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm5
-; SSE42-NEXT:    packssdw %xmm6, %xmm5
-; SSE42-NEXT:    movdqa {{.*#+}} xmm6 = [50,51]
-; SSE42-NEXT:    paddq %xmm2, %xmm6
-; SSE42-NEXT:    movdqa %xmm6, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm6, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm6
-; SSE42-NEXT:    paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [48,49]
-; SSE42-NEXT:    movdqa %xmm2, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm0
-; SSE42-NEXT:    por %xmm2, %xmm0
-; SSE42-NEXT:    pxor %xmm1, %xmm0
-; SSE42-NEXT:    pcmpgtq %xmm0, %xmm3
-; SSE42-NEXT:    packssdw %xmm6, %xmm3
-; SSE42-NEXT:    packssdw %xmm5, %xmm3
-; SSE42-NEXT:    packsswb %xmm4, %xmm3
-; SSE42-NEXT:    pmovmskb %xmm3, %esi
-; SSE42-NEXT:    shll $16, %esi
-; SSE42-NEXT:    orl %edx, %esi
-; SSE42-NEXT:    shlq $32, %rsi
-; SSE42-NEXT:    orq %rcx, %rsi
-; SSE42-NEXT:    movq %rsi, (%rdi)
-; SSE42-NEXT:    retq
-;
-; AVX2-LABEL: lane_mask_v64i1_i64:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    movq %rdi, %rax
-; AVX2-NEXT:    vmovq %rsi, %xmm0
-; AVX2-NEXT:    vpbroadcastq %xmm0, %ymm1
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpxor %ymm0, %ymm1, %ymm2
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3 # [28,29,30,31]
-; AVX2-NEXT:    vpxor %ymm0, %ymm3, %ymm4
-; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm2, %ymm4
-; AVX2-NEXT:    vpor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpxor %ymm0, %ymm3, %ymm4
-; AVX2-NEXT:    vmovq %rdx, %xmm3
-; AVX2-NEXT:    vpbroadcastq %xmm3, %ymm3
-; AVX2-NEXT:    vpxor %ymm0, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm3, %ymm4
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5 # [24,25,26,27]
-; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm6
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm2, %ymm6
-; AVX2-NEXT:    vpor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtq %ymm5, %ymm3, %ymm5
-; AVX2-NEXT:    vpackssdw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5 # [20,21,22,23]
-; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm6
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm2, %ymm6
-; AVX2-NEXT:    vpor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtq %ymm5, %ymm3, %ymm5
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm6 # [16,17,18,19]
-; AVX2-NEXT:    vpxor %ymm0, %ymm6, %ymm7
-; AVX2-NEXT:    vpcmpgtq %ymm7, %ymm2, %ymm7
-; AVX2-NEXT:    vpor %ymm7, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm0, %ymm6, %ymm6
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm3, %ymm6
-; AVX2-NEXT:    vpackssdw %ymm5, %ymm6, %ymm5
-; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,2,1,3]
-; AVX2-NEXT:    vpackssdw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5 # [12,13,14,15]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
-; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm6
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm2, %ymm6
-; AVX2-NEXT:    vpor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtq %ymm5, %ymm3, %ymm5
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm6 # [8,9,10,11]
-; AVX2-NEXT:    vpxor %ymm0, %ymm6, %ymm7
-; AVX2-NEXT:    vpcmpgtq %ymm7, %ymm2, %ymm7
-; AVX2-NEXT:    vpor %ymm7, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm0, %ymm6, %ymm6
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm3, %ymm6
-; AVX2-NEXT:    vpackssdw %ymm5, %ymm6, %ymm5
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm6 # [4,5,6,7]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,2,1,3]
-; AVX2-NEXT:    vpxor %ymm0, %ymm6, %ymm7
-; AVX2-NEXT:    vpcmpgtq %ymm7, %ymm2, %ymm7
-; AVX2-NEXT:    vpor %ymm7, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm0, %ymm6, %ymm6
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm3, %ymm6
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm7 # [0,1,2,3]
-; AVX2-NEXT:    vpxor %ymm0, %ymm7, %ymm8
-; AVX2-NEXT:    vpcmpgtq %ymm8, %ymm2, %ymm8
-; AVX2-NEXT:    vpor %ymm7, %ymm8, %ymm7
-; AVX2-NEXT:    vpxor %ymm0, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpgtq %ymm7, %ymm3, %ymm7
-; AVX2-NEXT:    vpackssdw %ymm6, %ymm7, %ymm6
-; AVX2-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,2,1,3]
-; AVX2-NEXT:    vpackssdw %ymm5, %ymm6, %ymm5
-; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,2,1,3]
-; AVX2-NEXT:    vpacksswb %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
-; AVX2-NEXT:    vpsllw $7, %ymm4, %ymm4
-; AVX2-NEXT:    vpmovmskb %ymm4, %ecx
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm4 # [60,61,62,63]
-; AVX2-NEXT:    vpxor %ymm0, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpgtq %ymm5, %ymm2, %ymm5
-; AVX2-NEXT:    vpor %ymm5, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm0, %ymm4, %ymm4
-; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm3, %ymm4
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5 # [56,57,58,59]
-; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm6
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm2, %ymm6
-; AVX2-NEXT:    vpor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtq %ymm5, %ymm3, %ymm5
-; AVX2-NEXT:    vpackssdw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5 # [52,53,54,55]
-; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm6
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm2, %ymm6
-; AVX2-NEXT:    vpor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtq %ymm5, %ymm3, %ymm5
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm6 # [48,49,50,51]
-; AVX2-NEXT:    vpxor %ymm0, %ymm6, %ymm7
-; AVX2-NEXT:    vpcmpgtq %ymm7, %ymm2, %ymm7
-; AVX2-NEXT:    vpor %ymm7, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm0, %ymm6, %ymm6
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm3, %ymm6
-; AVX2-NEXT:    vpackssdw %ymm5, %ymm6, %ymm5
-; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,2,1,3]
-; AVX2-NEXT:    vpackssdw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm5 # [44,45,46,47]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
-; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm6
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm2, %ymm6
-; AVX2-NEXT:    vpor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtq %ymm5, %ymm3, %ymm5
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm6 # [40,41,42,43]
-; AVX2-NEXT:    vpxor %ymm0, %ymm6, %ymm7
-; AVX2-NEXT:    vpcmpgtq %ymm7, %ymm2, %ymm7
-; AVX2-NEXT:    vpor %ymm7, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm0, %ymm6, %ymm6
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm3, %ymm6
-; AVX2-NEXT:    vpackssdw %ymm5, %ymm6, %ymm5
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm6 # [36,37,38,39]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,2,1,3]
-; AVX2-NEXT:    vpxor %ymm0, %ymm6, %ymm7
-; AVX2-NEXT:    vpcmpgtq %ymm7, %ymm2, %ymm7
-; AVX2-NEXT:    vpor %ymm7, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm0, %ymm6, %ymm6
-; AVX2-NEXT:    vpcmpgtq %ymm6, %ymm3, %ymm6
-; AVX2-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [32,33,34,35]
-; AVX2-NEXT:    vpxor %ymm0, %ymm1, %ymm7
-; AVX2-NEXT:    vpcmpgtq %ymm7, %ymm2, %ymm2
-; AVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpxor %ymm0, %ymm1, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm3, %ymm0
-; AVX2-NEXT:    vpackssdw %ymm6, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpackssdw %ymm5, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpacksswb %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpsllw $7, %ymm0, %ymm0
-; AVX2-NEXT:    vpmovmskb %ymm0, %edx
-; AVX2-NEXT:    shlq $32, %rdx
-; AVX2-NEXT:    orq %rcx, %rdx
-; AVX2-NEXT:    movq %rdx, (%rdi)
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: lane_mask_v64i1_i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %rsi, %zmm0
-; AVX512-NEXT:    vpbroadcastq %rdi, %zmm1
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [0,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm3
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm2, %k0
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm2 # [8,9,10,11,12,13,14,15]
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm2, %k1
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT:    kunpckbw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [16,17,18,19,20,21,22,23]
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm2, %k1
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [24,25,26,27,28,29,30,31]
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm2, %k2
-; AVX512-NEXT:    kunpckbw %k1, %k2, %k1
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT:    kunpckwd %k0, %k1, %k0
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [32,33,34,35,36,37,38,39]
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm2, %k1
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [40,41,42,43,44,45,46,47]
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm2, %k2
-; AVX512-NEXT:    kunpckbw %k1, %k2, %k1
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2 # [48,49,50,51,52,53,54,55]
-; AVX512-NEXT:    vpminuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT:    vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [56,57,58,59,60,61,62,63]
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm2, %k2
-; AVX512-NEXT:    vpcmpltuq %zmm0, %zmm1, %k3
-; AVX512-NEXT:    kunpckbw %k2, %k3, %k2
-; AVX512-NEXT:    kunpckwd %k1, %k2, %k1
-; AVX512-NEXT:    kunpckdq %k0, %k1, %k0
-; AVX512-NEXT:    vpmovm2b %k0, %zmm0
-; AVX512-NEXT:    retq
-  %active.lane.mask = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 %index, i64 %TC)
-  ret <64 x i1> %active.lane.mask
-}
-
-;
-; i64 (zero index)
-;
-
-define <2 x i1> @lane_mask_v2i1_i64_zero(i64 %TC) {
-; SSE2-LABEL: lane_mask_v2i1_i64_zero:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq %rdi, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE2-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: lane_mask_v2i1_i64_zero:
-; SSE42:       # %bb.0:
-; SSE42-NEXT:    movq %rdi, %xmm0
-; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE42-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT:    retq
-;
-; AVX2-LABEL: lane_mask_v2i1_i64_zero:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq %rdi, %xmm0
-; AVX2-NEXT:    vpbroadcastq %xmm0, %xmm0
-; AVX2-NEXT:    vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: lane_mask_v2i1_i64_zero:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %rdi, %xmm0
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %k0
-; AVX512-NEXT:    vpmovm2q %k0, %xmm0
-; AVX512-NEXT:    retq
-  %active.lane.mask = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 %TC)
-  ret <2 x i1> %active.lane.mask
-}
-
-define <4 x i1> @lane_mask_v4i1_i64_zero(i64 %TC) {
-; SSE2-LABEL: lane_mask_v4i1_i64_zero:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq %rdi, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,0,1]
-; SSE2-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,1,3]
@@ -4520,223 +2701,10 @@ define <16 x i1> @lane_mask_v16i1_i64_zero(i64 %TC) {
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: lane_mask_v16i1_i64_zero:
-; SSE42:       # %bb.0:
-; SSE42-NEXT:    movq %rdi, %xmm0
-; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE42-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT:    movdqa %xmm0, %xmm1
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT:    packssdw %xmm1, %xmm2
-; SSE42-NEXT:    movdqa %xmm0, %xmm1
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT:    packssdw %xmm1, %xmm3
-; SSE42-NEXT:    packssdw %xmm2, %xmm3
-; SSE42-NEXT:    movdqa %xmm0, %xmm1
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT:    packssdw %xmm1, %xmm2
-; SSE42-NEXT:    movdqa %xmm0, %xmm1
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT:    packssdw %xmm1, %xmm0
-; SSE42-NEXT:    packssdw %xmm2, %xmm0
-; SSE42-NEXT:    packsswb %xmm3, %xmm0
-; SSE42-NEXT:    retq
-;
-; AVX2-LABEL: lane_mask_v16i1_i64_zero:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq %rdi, %xmm0
-; AVX2-NEXT:    vpbroadcastq %xmm0, %ymm0
-; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2-NEXT:    vpackssdw %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: lane_mask_v16i1_i64_zero:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %rdi, %zmm0
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT:    kunpckbw %k0, %k1, %k0
-; AVX512-NEXT:    vpmovm2b %k0, %xmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
-  %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 %TC)
-  ret <16 x i1> %active.lane.mask
-}
-
-define <32 x i1> @lane_mask_v32i1_i64_zero(i64 %TC) {
-; SSE2-LABEL: lane_mask_v32i1_i64_zero:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq %rsi, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE2-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    movq %rdi, %rax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm3
-; SSE2-NEXT:    packssdw %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm2
-; SSE2-NEXT:    packssdw %xmm5, %xmm2
-; SSE2-NEXT:    packssdw %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    packssdw %xmm3, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6
-; SSE2-NEXT:    por %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm6
-; SSE2-NEXT:    packssdw %xmm3, %xmm6
-; SSE2-NEXT:    packssdw %xmm4, %xmm6
-; SSE2-NEXT:    packsswb %xmm2, %xmm6
-; SSE2-NEXT:    pmovmskb %xmm6, %ecx
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm3
-; SSE2-NEXT:    packssdw %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm2
-; SSE2-NEXT:    packssdw %xmm5, %xmm2
-; SSE2-NEXT:    packssdw %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    packssdw %xmm3, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm0
-; SSE2-NEXT:    packssdw %xmm3, %xmm0
-; SSE2-NEXT:    packssdw %xmm4, %xmm0
-; SSE2-NEXT:    packsswb %xmm2, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %edx
-; SSE2-NEXT:    shll $16, %edx
-; SSE2-NEXT:    orl %ecx, %edx
-; SSE2-NEXT:    movl %edx, (%rdi)
-; SSE2-NEXT:    retq
-;
-; SSE42-LABEL: lane_mask_v32i1_i64_zero:
-; SSE42:       # %bb.0:
-; SSE42-NEXT:    movq %rsi, %xmm0
-; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE42-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT:    movdqa %xmm0, %xmm1
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT:    movdqa %xmm0, %xmm4
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE42-NEXT:    movq %rdi, %rax
-; SSE42-NEXT:    movdqa %xmm0, %xmm5
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE42-NEXT:    packssdw %xmm1, %xmm2
-; SSE42-NEXT:    movdqa %xmm0, %xmm1
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT:    packssdw %xmm3, %xmm4
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT:    packssdw %xmm2, %xmm4
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT:    packssdw %xmm5, %xmm1
-; SSE42-NEXT:    packssdw %xmm3, %xmm2
-; SSE42-NEXT:    packssdw %xmm1, %xmm2
-; SSE42-NEXT:    packsswb %xmm4, %xmm2
-; SSE42-NEXT:    pmovmskb %xmm2, %ecx
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movq %rdi, %xmm0
+; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; SSE42-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE42-NEXT:    movdqa %xmm0, %xmm1
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE42-NEXT:    movdqa %xmm0, %xmm2
@@ -4759,13 +2727,9 @@ define <32 x i1> @lane_mask_v32i1_i64_zero(i64 %TC) {
 ; SSE42-NEXT:    packssdw %xmm1, %xmm0
 ; SSE42-NEXT:    packssdw %xmm2, %xmm0
 ; SSE42-NEXT:    packsswb %xmm3, %xmm0
-; SSE42-NEXT:    pmovmskb %xmm0, %edx
-; SSE42-NEXT:    shll $16, %edx
-; SSE42-NEXT:    orl %ecx, %edx
-; SSE42-NEXT:    movl %edx, (%rdi)
 ; SSE42-NEXT:    retq
 ;
-; AVX2-LABEL: lane_mask_v32i1_i64_zero:
+; AVX2-LABEL: lane_mask_v16i1_i64_zero:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovq %rdi, %xmm0
 ; AVX2-NEXT:    vpbroadcastq %xmm0, %ymm0
@@ -4776,43 +2740,32 @@ define <32 x i1> @lane_mask_v32i1_i64_zero(i64 %TC) {
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
 ; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
-; AVX2-NEXT:    vpackssdw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
-; AVX2-NEXT:    vpackssdw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
 ; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2-NEXT:    vpackssdw %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpackssdw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: lane_mask_v32i1_i64_zero:
+; AVX512-LABEL: lane_mask_v16i1_i64_zero:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %rdi, %zmm0
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k2
-; AVX512-NEXT:    kunpckbw %k0, %k1, %k0
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT:    kunpckbw %k2, %k1, %k1
-; AVX512-NEXT:    kunpckwd %k0, %k1, %k0
-; AVX512-NEXT:    vpmovm2b %k0, %ymm0
+; AVX512-NEXT:    cmpq $16, %rdi
+; AVX512-NEXT:    movl $16, %eax
+; AVX512-NEXT:    cmovbq %rdi, %rax
+; AVX512-NEXT:    movl $-1, %ecx
+; AVX512-NEXT:    bzhil %eax, %ecx, %eax
+; AVX512-NEXT:    kmovd %eax, %k0
+; AVX512-NEXT:    vpmovm2b %k0, %xmm0
 ; AVX512-NEXT:    retq
-  %active.lane.mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 %TC)
-  ret <32 x i1> %active.lane.mask
+  %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 %TC)
+  ret <16 x i1> %active.lane.mask
 }
 
-define <64 x i1> @lane_mask_v64i1_i64_zero(i64 %TC) {
-; SSE2-LABEL: lane_mask_v64i1_i64_zero:
+define <32 x i1> @lane_mask_v32i1_i64_zero(i64 %TC) {
+; SSE2-LABEL: lane_mask_v32i1_i64_zero:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %rsi, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
@@ -4875,7 +2828,7 @@ define <64 x i1> @lane_mask_v64i1_i64_zero(i64 %TC) {
 ; SSE2-NEXT:    packssdw %xmm3, %xmm6
 ; SSE2-NEXT:    packssdw %xmm4, %xmm6
 ; SSE2-NEXT:    packsswb %xmm2, %xmm6
-; SSE2-NEXT:    pmovmskb %xmm6, %edx
+; SSE2-NEXT:    pmovmskb %xmm6, %ecx
 ; SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
@@ -4921,120 +2874,6 @@ define <64 x i1> @lane_mask_v64i1_i64_zero(i64 %TC) {
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
 ; SSE2-NEXT:    pand %xmm1, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6
-; SSE2-NEXT:    por %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm6
-; SSE2-NEXT:    packssdw %xmm3, %xmm6
-; SSE2-NEXT:    packssdw %xmm4, %xmm6
-; SSE2-NEXT:    packsswb %xmm2, %xmm6
-; SSE2-NEXT:    pmovmskb %xmm6, %ecx
-; SSE2-NEXT:    shll $16, %ecx
-; SSE2-NEXT:    orl %edx, %ecx
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    packssdw %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm2
-; SSE2-NEXT:    packssdw %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    packssdw %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    packssdw %xmm3, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm5
-; SSE2-NEXT:    packssdw %xmm3, %xmm5
-; SSE2-NEXT:    packssdw %xmm4, %xmm5
-; SSE2-NEXT:    packsswb %xmm2, %xmm5
-; SSE2-NEXT:    pmovmskb %xmm5, %edx
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    packssdw %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm2
-; SSE2-NEXT:    packssdw %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    packssdw %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    packssdw %xmm3, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    por %xmm5, %xmm3
 ; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
@@ -5044,203 +2883,147 @@ define <64 x i1> @lane_mask_v64i1_i64_zero(i64 %TC) {
 ; SSE2-NEXT:    packssdw %xmm3, %xmm0
 ; SSE2-NEXT:    packssdw %xmm4, %xmm0
 ; SSE2-NEXT:    packsswb %xmm2, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %esi
-; SSE2-NEXT:    shll $16, %esi
-; SSE2-NEXT:    orl %edx, %esi
-; SSE2-NEXT:    shlq $32, %rsi
-; SSE2-NEXT:    orq %rcx, %rsi
-; SSE2-NEXT:    movq %rsi, (%rdi)
+; SSE2-NEXT:    pmovmskb %xmm0, %edx
+; SSE2-NEXT:    shll $16, %edx
+; SSE2-NEXT:    orl %ecx, %edx
+; SSE2-NEXT:    movl %edx, (%rdi)
 ; SSE2-NEXT:    retq
 ;
-; SSE42-LABEL: lane_mask_v64i1_i64_zero:
+; SSE42-LABEL: lane_mask_v32i1_i64_zero:
 ; SSE42:       # %bb.0:
 ; SSE42-NEXT:    movq %rsi, %xmm0
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; SSE42-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE42-NEXT:    movdqa %xmm0, %xmm1
+; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE42-NEXT:    movdqa %xmm0, %xmm2
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
 ; SSE42-NEXT:    movdqa %xmm0, %xmm3
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; SSE42-NEXT:    movdqa %xmm0, %xmm4
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE42-NEXT:    movdqa %xmm0, %xmm1
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE42-NEXT:    movq %rdi, %rax
 ; SSE42-NEXT:    movdqa %xmm0, %xmm5
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE42-NEXT:    packssdw %xmm2, %xmm3
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT:    packssdw %xmm4, %xmm1
-; SSE42-NEXT:    movdqa %xmm0, %xmm4
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE42-NEXT:    packssdw %xmm3, %xmm1
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT:    packssdw %xmm5, %xmm2
-; SSE42-NEXT:    movdqa %xmm0, %xmm5
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE42-NEXT:    packssdw %xmm4, %xmm3
-; SSE42-NEXT:    movdqa %xmm0, %xmm4
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE42-NEXT:    packssdw %xmm2, %xmm3
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT:    packsswb %xmm1, %xmm3
+; SSE42-NEXT:    packssdw %xmm1, %xmm2
 ; SSE42-NEXT:    movdqa %xmm0, %xmm1
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT:    pmovmskb %xmm3, %edx
+; SSE42-NEXT:    packssdw %xmm3, %xmm4
 ; SSE42-NEXT:    movdqa %xmm0, %xmm3
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT:    packssdw %xmm5, %xmm4
-; SSE42-NEXT:    movdqa %xmm0, %xmm5
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE42-NEXT:    packssdw %xmm2, %xmm1
+; SSE42-NEXT:    packssdw %xmm2, %xmm4
 ; SSE42-NEXT:    movdqa %xmm0, %xmm2
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT:    packssdw %xmm4, %xmm1
-; SSE42-NEXT:    movdqa %xmm0, %xmm4
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE42-NEXT:    packssdw %xmm3, %xmm5
-; SSE42-NEXT:    packssdw %xmm2, %xmm4
-; SSE42-NEXT:    packssdw %xmm5, %xmm4
-; SSE42-NEXT:    packsswb %xmm1, %xmm4
-; SSE42-NEXT:    pmovmskb %xmm4, %ecx
+; SSE42-NEXT:    packssdw %xmm5, %xmm1
+; SSE42-NEXT:    packssdw %xmm3, %xmm2
+; SSE42-NEXT:    packssdw %xmm1, %xmm2
+; SSE42-NEXT:    packsswb %xmm4, %xmm2
+; SSE42-NEXT:    pmovmskb %xmm2, %ecx
 ; SSE42-NEXT:    movdqa %xmm0, %xmm1
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT:    shll $16, %ecx
 ; SSE42-NEXT:    movdqa %xmm0, %xmm2
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT:    orl %edx, %ecx
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; SSE42-NEXT:    packssdw %xmm1, %xmm2
 ; SSE42-NEXT:    movdqa %xmm0, %xmm1
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT:    packssdw %xmm3, %xmm1
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT:    packssdw %xmm2, %xmm1
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT:    packssdw %xmm3, %xmm2
 ; SSE42-NEXT:    movdqa %xmm0, %xmm3
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT:    movdqa %xmm0, %xmm4
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE42-NEXT:    packssdw %xmm3, %xmm4
-; SSE42-NEXT:    packssdw %xmm2, %xmm4
-; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT:    packsswb %xmm1, %xmm4
+; SSE42-NEXT:    packssdw %xmm1, %xmm3
+; SSE42-NEXT:    packssdw %xmm2, %xmm3
 ; SSE42-NEXT:    movdqa %xmm0, %xmm1
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT:    pmovmskb %xmm4, %edx
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE42-NEXT:    packssdw %xmm2, %xmm1
 ; SSE42-NEXT:    movdqa %xmm0, %xmm2
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE42-NEXT:    packssdw %xmm3, %xmm2
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; SSE42-NEXT:    packssdw %xmm1, %xmm2
 ; SSE42-NEXT:    movdqa %xmm0, %xmm1
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE42-NEXT:    packssdw %xmm3, %xmm1
-; SSE42-NEXT:    movdqa %xmm0, %xmm3
-; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; SSE42-NEXT:    pcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE42-NEXT:    packssdw %xmm3, %xmm0
 ; SSE42-NEXT:    packssdw %xmm1, %xmm0
-; SSE42-NEXT:    packsswb %xmm2, %xmm0
-; SSE42-NEXT:    pmovmskb %xmm0, %esi
-; SSE42-NEXT:    shll $16, %esi
-; SSE42-NEXT:    orl %edx, %esi
-; SSE42-NEXT:    shlq $32, %rsi
-; SSE42-NEXT:    orq %rcx, %rsi
-; SSE42-NEXT:    movq %rsi, (%rdi)
+; SSE42-NEXT:    packssdw %xmm2, %xmm0
+; SSE42-NEXT:    packsswb %xmm3, %xmm0
+; SSE42-NEXT:    pmovmskb %xmm0, %edx
+; SSE42-NEXT:    shll $16, %edx
+; SSE42-NEXT:    orl %ecx, %edx
+; SSE42-NEXT:    movl %edx, (%rdi)
 ; SSE42-NEXT:    retq
 ;
-; AVX2-LABEL: lane_mask_v64i1_i64_zero:
+; AVX2-LABEL: lane_mask_v32i1_i64_zero:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq %rsi, %xmm0
+; AVX2-NEXT:    vmovq %rdi, %xmm0
 ; AVX2-NEXT:    vpbroadcastq %xmm0, %ymm0
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1
 ; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT:    movq %rdi, %rax
 ; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
-; AVX2-NEXT:    vpackssdw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
 ; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
 ; AVX2-NEXT:    vpackssdw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm4
-; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT:    vpackssdw %ymm3, %ymm4, %ymm3
-; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
-; AVX2-NEXT:    vpackssdw %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT:    vpacksswb %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT:    vpsllw $7, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
-; AVX2-NEXT:    vpmovmskb %ymm1, %ecx
-; AVX2-NEXT:    vpackssdw %ymm2, %ymm3, %ymm1
 ; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
 ; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
 ; AVX2-NEXT:    vpackssdw %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
-; AVX2-NEXT:    vpackssdw %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
 ; AVX2-NEXT:    vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
 ; AVX2-NEXT:    vpackssdw %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpackssdw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT:    vpsllw $7, %ymm0, %ymm0
-; AVX2-NEXT:    vpmovmskb %ymm0, %edx
-; AVX2-NEXT:    shlq $32, %rdx
-; AVX2-NEXT:    orq %rcx, %rdx
-; AVX2-NEXT:    movq %rdx, (%rdi)
-; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: lane_mask_v32i1_i64_zero:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    cmpq $32, %rdi
+; AVX512-NEXT:    movl $32, %eax
+; AVX512-NEXT:    cmovbq %rdi, %rax
+; AVX512-NEXT:    movl $-1, %ecx
+; AVX512-NEXT:    bzhil %eax, %ecx, %eax
+; AVX512-NEXT:    kmovd %eax, %k0
+; AVX512-NEXT:    vpmovm2b %k0, %ymm0
+; AVX512-NEXT:    retq
+  %active.lane.mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 %TC)
+  ret <32 x i1> %active.lane.mask
+}
+
+define <64 x i1> @lane_mask_v64i1_i64_zero(i64 %TC) {
+; SSE-LABEL: lane_mask_v64i1_i64_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpq $64, %rsi
+; SSE-NEXT:    movl $64, %ecx
+; SSE-NEXT:    cmovbq %rsi, %rcx
+; SSE-NEXT:    movl $1, %edx
+; SSE-NEXT:    # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT:    shlq %cl, %rdx
+; SSE-NEXT:    movq %rdi, %rax
+; SSE-NEXT:    decq %rdx
+; SSE-NEXT:    movq %rdx, (%rdi)
+; SSE-NEXT:    retq
+;
+; AVX2-LABEL: lane_mask_v64i1_i64_zero:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    cmpq $64, %rsi
+; AVX2-NEXT:    movl $64, %ecx
+; AVX2-NEXT:    cmovbq %rsi, %rcx
+; AVX2-NEXT:    movq %rdi, %rax
+; AVX2-NEXT:    movq $-1, %rdx
+; AVX2-NEXT:    bzhiq %rcx, %rdx, %rcx
+; AVX2-NEXT:    movq %rcx, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: lane_mask_v64i1_i64_zero:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %rdi, %zmm0
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k2
-; AVX512-NEXT:    kunpckbw %k0, %k1, %k0
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT:    kunpckbw %k2, %k1, %k1
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k2
-; AVX512-NEXT:    kunpckwd %k0, %k1, %k0
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
-; AVX512-NEXT:    kunpckbw %k2, %k1, %k1
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k2
-; AVX512-NEXT:    vpcmpnleuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k3
-; AVX512-NEXT:    kunpckbw %k2, %k3, %k2
-; AVX512-NEXT:    kunpckwd %k1, %k2, %k1
-; AVX512-NEXT:    kunpckdq %k0, %k1, %k0
+; AVX512-NEXT:    cmpq $64, %rdi
+; AVX512-NEXT:    movl $64, %eax
+; AVX512-NEXT:    cmovbq %rdi, %rax
+; AVX512-NEXT:    movq $-1, %rcx
+; AVX512-NEXT:    bzhiq %rax, %rcx, %rax
+; AVX512-NEXT:    kmovq %rax, %k0
 ; AVX512-NEXT:    vpmovm2b %k0, %zmm0
 ; AVX512-NEXT:    retq
   %active.lane.mask = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 0, i64 %TC)
diff --git a/llvm/test/CodeGen/X86/pr47299.ll b/llvm/test/CodeGen/X86/pr47299.ll
index 7cb1112402ebe5..1e502fd52268b5 100644
--- a/llvm/test/CodeGen/X86/pr47299.ll
+++ b/llvm/test/CodeGen/X86/pr47299.ll
@@ -56,12 +56,13 @@ define <7 x i1> @create_mask7(i64 %0) {
 define <16 x i1> @create_mask16(i64 %0) {
 ; CHECK-LABEL: create_mask16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastq zmm0, rdi
-; CHECK-NEXT:    vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    kunpckbw k0, k1, k0
+; CHECK-NEXT:    cmp rdi, 16
+; CHECK-NEXT:    mov eax, 16
+; CHECK-NEXT:    cmovb rax, rdi
+; CHECK-NEXT:    mov ecx, -1
+; CHECK-NEXT:    bzhi eax, ecx, eax
+; CHECK-NEXT:    kmovd k0, eax
 ; CHECK-NEXT:    vpmovm2b xmm0, k0
-; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret
   %2 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 %0)
   ret <16 x i1> %2
@@ -70,14 +71,12 @@ define <16 x i1> @create_mask16(i64 %0) {
 define <32 x i1> @create_mask32(i64 %0) {
 ; CHECK-LABEL: create_mask32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastq zmm0, rdi
-; CHECK-NEXT:    vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    kunpckbw k0, k1, k0
-; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    kunpckbw k1, k1, k2
-; CHECK-NEXT:    kunpckwd k0, k1, k0
+; CHECK-NEXT:    cmp rdi, 32
+; CHECK-NEXT:    mov eax, 32
+; CHECK-NEXT:    cmovb rax, rdi
+; CHECK-NEXT:    mov ecx, -1
+; CHECK-NEXT:    bzhi eax, ecx, eax
+; CHECK-NEXT:    kmovd k0, eax
 ; CHECK-NEXT:    vpmovm2b ymm0, k0
 ; CHECK-NEXT:    ret
   %2 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 %0)
@@ -87,22 +86,12 @@ define <32 x i1> @create_mask32(i64 %0) {
 define <64 x i1> @create_mask64(i64 %0) {
 ; CHECK-LABEL: create_mask64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastq zmm0, rdi
-; CHECK-NEXT:    vpcmpnleuq k0, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    kunpckbw k0, k1, k0
-; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    kunpckbw k1, k1, k2
-; CHECK-NEXT:    vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    kunpckwd k0, k1, k0
-; CHECK-NEXT:    vpcmpnleuq k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    kunpckbw k1, k1, k2
-; CHECK-NEXT:    vpcmpnleuq k2, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    vpcmpnleuq k3, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    kunpckbw k2, k3, k2
-; CHECK-NEXT:    kunpckwd k1, k2, k1
-; CHECK-NEXT:    kunpckdq k0, k1, k0
+; CHECK-NEXT:    cmp rdi, 64
+; CHECK-NEXT:    mov eax, 64
+; CHECK-NEXT:    cmovb rax, rdi
+; CHECK-NEXT:    mov rcx, -1
+; CHECK-NEXT:    bzhi rax, rcx, rax
+; CHECK-NEXT:    kmovq k0, rax
 ; CHECK-NEXT:    vpmovm2b zmm0, k0
 ; CHECK-NEXT:    ret
   %2 = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 0, i64 %0)
@@ -112,10 +101,13 @@ define <64 x i1> @create_mask64(i64 %0) {
 define <16 x i1> @create_mask16_i32(i32 %0) {
 ; CHECK-LABEL: create_mask16_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastd zmm0, edi
-; CHECK-NEXT:    vpcmpnleud k0, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
+; CHECK-NEXT:    cmp edi, 16
+; CHECK-NEXT:    mov eax, 16
+; CHECK-NEXT:    cmovb eax, edi
+; CHECK-NEXT:    mov ecx, -1
+; CHECK-NEXT:    bzhi eax, ecx, eax
+; CHECK-NEXT:    kmovd k0, eax
 ; CHECK-NEXT:    vpmovm2b xmm0, k0
-; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    ret
   %2 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 0, i32 %0)
   ret <16 x i1> %2
@@ -124,14 +116,12 @@ define <16 x i1> @create_mask16_i32(i32 %0) {
 define <64 x i1> @create_mask64_i32(i32 %0) {
 ; CHECK-LABEL: create_mask64_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastd zmm0, edi
-; CHECK-NEXT:    vpcmpnleud k0, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    vpcmpnleud k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    vpcmpnleud k2, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    kunpckwd k0, k1, k0
-; CHECK-NEXT:    vpcmpnleud k1, zmm0, zmmword ptr [rip + {{\.?LCPI[0-9]+_[0-9]+}}]
-; CHECK-NEXT:    kunpckwd k1, k1, k2
-; CHECK-NEXT:    kunpckdq k0, k1, k0
+; CHECK-NEXT:    cmp edi, 64
+; CHECK-NEXT:    mov eax, 64
+; CHECK-NEXT:    cmovb eax, edi
+; CHECK-NEXT:    mov rcx, -1
+; CHECK-NEXT:    bzhi rax, rcx, rax
+; CHECK-NEXT:    kmovq k0, rax
 ; CHECK-NEXT:    vpmovm2b zmm0, k0
 ; CHECK-NEXT:    ret
   %2 = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 0, i32 %0)

>From 2c7897d59ccc46133c91f6a1137d80541ecf0b54 Mon Sep 17 00:00:00 2001
From: william <we3223 at gmail.com>
Date: Wed, 30 Sep 2026 23:54:13 +0800
Subject: [PATCH 2/5] [X86][test] Add common SSE check prefix in
 active_lane_mask.ll-1

---
 llvm/test/CodeGen/X86/active_lane_mask.ll | 376 +++++++++++-----------
 1 file changed, 188 insertions(+), 188 deletions(-)

diff --git a/llvm/test/CodeGen/X86/active_lane_mask.ll b/llvm/test/CodeGen/X86/active_lane_mask.ll
index 74a37c8ba6b4a8..d316db74246d9e 100644
--- a/llvm/test/CodeGen/X86/active_lane_mask.ll
+++ b/llvm/test/CodeGen/X86/active_lane_mask.ll
@@ -272,34 +272,34 @@ define <16 x i1> @lane_mask_v16i1_i32(i32 %index, i32 %TC) {
 ; SSE42-LABEL: lane_mask_v16i1_i32:
 ; SSE42:       # %bb.0:
 ; SSE42-NEXT:    movd %edi, %xmm0
-; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm0 = [4294967295,4294967294,4294967293,4294967292]
-; SSE42-NEXT:    pminud %xmm1, %xmm0
+; SSE42-NEXT:    pminud %xmm2, %xmm0
 ; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [0,1,2,3]
-; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [4294967291,4294967290,4294967289,4294967288]
-; SSE42-NEXT:    pminud %xmm1, %xmm2
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [4,5,6,7]
+; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [4294967291,4294967290,4294967289,4294967288]
+; SSE42-NEXT:    pminud %xmm2, %xmm1
+; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [4,5,6,7]
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [4294967287,4294967286,4294967285,4294967284]
-; SSE42-NEXT:    pminud %xmm1, %xmm3
+; SSE42-NEXT:    pminud %xmm2, %xmm3
 ; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [8,9,10,11]
 ; SSE42-NEXT:    movd %esi, %xmm4
-; SSE42-NEXT:    pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE42-NEXT:    pminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [12,13,14,15]
-; SSE42-NEXT:    movdqa %xmm1, %xmm5
+; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [12,13,14,15]
+; SSE42-NEXT:    movdqa %xmm2, %xmm5
 ; SSE42-NEXT:    pmaxud %xmm4, %xmm5
-; SSE42-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE42-NEXT:    movdqa %xmm3, %xmm1
-; SSE42-NEXT:    pmaxud %xmm4, %xmm1
-; SSE42-NEXT:    pcmpeqd %xmm3, %xmm1
-; SSE42-NEXT:    packssdw %xmm5, %xmm1
-; SSE42-NEXT:    movdqa %xmm2, %xmm3
+; SSE42-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSE42-NEXT:    movdqa %xmm3, %xmm2
+; SSE42-NEXT:    pmaxud %xmm4, %xmm2
+; SSE42-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE42-NEXT:    packssdw %xmm5, %xmm2
+; SSE42-NEXT:    movdqa %xmm1, %xmm3
 ; SSE42-NEXT:    pmaxud %xmm4, %xmm3
-; SSE42-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE42-NEXT:    pcmpeqd %xmm1, %xmm3
 ; SSE42-NEXT:    pmaxud %xmm0, %xmm4
 ; SSE42-NEXT:    pcmpeqd %xmm0, %xmm4
 ; SSE42-NEXT:    packssdw %xmm3, %xmm4
-; SSE42-NEXT:    packsswb %xmm1, %xmm4
+; SSE42-NEXT:    packsswb %xmm2, %xmm4
 ; SSE42-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE42-NEXT:    pxor %xmm4, %xmm0
 ; SSE42-NEXT:    retq
@@ -408,20 +408,9 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
 ; SSE2-NEXT:    por %xmm4, %xmm6
 ; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [24,25,26,27]
-; SSE2-NEXT:    paddd %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm1, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    pxor %xmm1, %xmm7
 ; SSE2-NEXT:    movdqa %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    packssdw %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [20,21,22,23]
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [24,25,26,27]
 ; SSE2-NEXT:    paddd %xmm2, %xmm4
 ; SSE2-NEXT:    movdqa %xmm4, %xmm6
 ; SSE2-NEXT:    pxor %xmm1, %xmm6
@@ -431,6 +420,17 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
 ; SSE2-NEXT:    pxor %xmm1, %xmm7
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
 ; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE2-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [20,21,22,23]
+; SSE2-NEXT:    paddd %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pxor %xmm1, %xmm6
+; SSE2-NEXT:    movdqa %xmm0, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE2-NEXT:    por %xmm5, %xmm7
+; SSE2-NEXT:    pxor %xmm1, %xmm7
+; SSE2-NEXT:    movdqa %xmm3, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
 ; SSE2-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [16,17,18,19]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm6
 ; SSE2-NEXT:    pxor %xmm1, %xmm6
@@ -438,8 +438,8 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
 ; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm0
 ; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    packssdw %xmm4, %xmm3
-; SSE2-NEXT:    packsswb %xmm5, %xmm3
+; SSE2-NEXT:    packssdw %xmm5, %xmm3
+; SSE2-NEXT:    packsswb %xmm4, %xmm3
 ; SSE2-NEXT:    pmovmskb %xmm3, %edx
 ; SSE2-NEXT:    shll $16, %edx
 ; SSE2-NEXT:    orl %ecx, %edx
@@ -459,28 +459,28 @@ define <32 x i1> @lane_mask_v32i1_i32(i32 %index, i32 %TC) {
 ; SSE42-NEXT:    movdqa %xmm2, %xmm3
 ; SSE42-NEXT:    pmaxud %xmm0, %xmm3
 ; SSE42-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [4294967287,4294967286,4294967285,4294967284]
-; SSE42-NEXT:    pminud %xmm1, %xmm2
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [8,9,10,11]
-; SSE42-NEXT:    movdqa %xmm2, %xmm4
+; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [4294967287,4294967286,4294967285,4294967284]
+; SSE42-NEXT:    pminud %xmm1, %xmm4
+; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [8,9,10,11]
+; SSE42-NEXT:    movdqa %xmm4, %xmm2
+; SSE42-NEXT:    pmaxud %xmm0, %xmm2
+; SSE42-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE42-NEXT:    packssdw %xmm3, %xmm2
+; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [4294967291,4294967290,4294967289,4294967288]
+; SSE42-NEXT:    pminud %xmm1, %xmm3
+; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 # [4,5,6,7]
+; SSE42-NEXT:    movdqa %xmm3, %xmm4
 ; SSE42-NEXT:    pmaxud %xmm0, %xmm4
-; SSE42-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE42-NEXT:    packssdw %xmm3, %xmm4
-; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [4294967291,4294967290,4294967289,4294967288]
-; SSE42-NEXT:    pminud %xmm1, %xmm2
-; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [4,5,6,7]
-; SSE42-NEXT:    movdqa %xmm2, %xmm3
-; SSE42-NEXT:    pmaxud %xmm0, %xmm3
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [4294967295,4294967294,4294967293,4294967292]
 ; SSE42-NEXT:    pminud %xmm1, %xmm5
 ; SSE42-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5 # [0,1,2,3]
-; SSE42-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE42-NEXT:    movdqa %xmm5, %xmm2
-; SSE42-NEXT:    pmaxud %xmm0, %xmm2
-; SSE42-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE42-NEXT:    packssdw %xmm3, %xmm2
-; SSE42-NEXT:    packsswb %xmm4, %xmm2
-; SSE42-NEXT:    pmovmskb %xmm2, %ecx
+; SSE42-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSE42-NEXT:    movdqa %xmm5, %xmm3
+; SSE42-NEXT:    pmaxud %xmm0, %xmm3
+; SSE42-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE42-NEXT:    packssdw %xmm4, %xmm3
+; SSE42-NEXT:    packsswb %xmm2, %xmm3
+; SSE42-NEXT:    pmovmskb %xmm3, %ecx
 ; SSE42-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [4294967267,4294967266,4294967265,4294967264]
 ; SSE42-NEXT:    pminud %xmm1, %xmm2
@@ -1400,195 +1400,195 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
 ; SSE2-LABEL: lane_mask_v16i1_i64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %rdi, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,1,0,1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,1,0,1]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pxor %xmm0, %xmm5
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
-; SSE2-NEXT:    paddq %xmm5, %xmm2
+; SSE2-NEXT:    paddq %xmm4, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm6, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm7
+; SSE2-NEXT:    pand %xmm6, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    por %xmm2, %xmm1
 ; SSE2-NEXT:    por %xmm7, %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2,3]
-; SSE2-NEXT:    paddq %xmm5, %xmm3
+; SSE2-NEXT:    paddq %xmm4, %xmm3
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm6, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm3, %xmm2
 ; SSE2-NEXT:    por %xmm8, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [4,5]
-; SSE2-NEXT:    paddq %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [4,5]
+; SSE2-NEXT:    paddq %xmm4, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm3
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
+; SSE2-NEXT:    movdqa %xmm5, %xmm7
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm8, %xmm9
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    por %xmm6, %xmm3
 ; SSE2-NEXT:    por %xmm9, %xmm3
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [6,7]
-; SSE2-NEXT:    paddq %xmm5, %xmm7
-; SSE2-NEXT:    movdqa %xmm7, %xmm4
-; SSE2-NEXT:    pxor %xmm0, %xmm4
-; SSE2-NEXT:    movdqa %xmm6, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm8
+; SSE2-NEXT:    paddq %xmm4, %xmm7
+; SSE2-NEXT:    movdqa %xmm7, %xmm6
+; SSE2-NEXT:    pxor %xmm0, %xmm6
+; SSE2-NEXT:    movdqa %xmm5, %xmm8
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm8
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm9, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm4
-; SSE2-NEXT:    por %xmm10, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm8[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm6
+; SSE2-NEXT:    por %xmm10, %xmm6
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [8,9]
-; SSE2-NEXT:    paddq %xmm5, %xmm8
+; SSE2-NEXT:    paddq %xmm4, %xmm8
 ; SSE2-NEXT:    movdqa %xmm8, %xmm7
 ; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm6, %xmm9
+; SSE2-NEXT:    movdqa %xmm5, %xmm9
 ; SSE2-NEXT:    pcmpgtd %xmm7, %xmm9
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm10, %xmm11
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm9[1,1,3,3]
 ; SSE2-NEXT:    por %xmm8, %xmm7
 ; SSE2-NEXT:    por %xmm11, %xmm7
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [10,11]
-; SSE2-NEXT:    paddq %xmm5, %xmm9
+; SSE2-NEXT:    paddq %xmm4, %xmm9
 ; SSE2-NEXT:    movdqa %xmm9, %xmm8
 ; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm6, %xmm10
+; SSE2-NEXT:    movdqa %xmm5, %xmm10
 ; SSE2-NEXT:    pcmpgtd %xmm8, %xmm10
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm8
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm8
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm8[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm11, %xmm12
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm10[1,1,3,3]
 ; SSE2-NEXT:    por %xmm9, %xmm8
 ; SSE2-NEXT:    por %xmm12, %xmm8
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [12,13]
-; SSE2-NEXT:    paddq %xmm5, %xmm10
+; SSE2-NEXT:    paddq %xmm4, %xmm10
 ; SSE2-NEXT:    movdqa %xmm10, %xmm9
 ; SSE2-NEXT:    pxor %xmm0, %xmm9
-; SSE2-NEXT:    movdqa %xmm6, %xmm11
+; SSE2-NEXT:    movdqa %xmm5, %xmm11
 ; SSE2-NEXT:    pcmpgtd %xmm9, %xmm11
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm9
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm9
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm9[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm12, %xmm13
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm11[1,1,3,3]
 ; SSE2-NEXT:    por %xmm10, %xmm9
 ; SSE2-NEXT:    por %xmm13, %xmm9
-; SSE2-NEXT:    paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5 # [14,15]
-; SSE2-NEXT:    movdqa %xmm5, %xmm10
+; SSE2-NEXT:    paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 # [14,15]
+; SSE2-NEXT:    movdqa %xmm4, %xmm10
 ; SSE2-NEXT:    pxor %xmm0, %xmm10
-; SSE2-NEXT:    movdqa %xmm6, %xmm11
+; SSE2-NEXT:    movdqa %xmm5, %xmm11
 ; SSE2-NEXT:    pcmpgtd %xmm10, %xmm11
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm10[1,1,3,3]
-; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm10[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm10
 ; SSE2-NEXT:    por %xmm5, %xmm10
-; SSE2-NEXT:    por %xmm6, %xmm10
-; SSE2-NEXT:    movq %rsi, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,1,0,1]
+; SSE2-NEXT:    movq %rsi, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
 ; SSE2-NEXT:    pxor %xmm0, %xmm10
-; SSE2-NEXT:    pxor %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm10
+; SSE2-NEXT:    pxor %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm10
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm11, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm10, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm10, %xmm5
 ; SSE2-NEXT:    pxor %xmm0, %xmm9
-; SSE2-NEXT:    movdqa %xmm5, %xmm10
+; SSE2-NEXT:    movdqa %xmm4, %xmm10
 ; SSE2-NEXT:    pcmpgtd %xmm9, %xmm10
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm9
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm9
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm9[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm11, %xmm12
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
 ; SSE2-NEXT:    por %xmm12, %xmm9
-; SSE2-NEXT:    packssdw %xmm6, %xmm9
+; SSE2-NEXT:    packssdw %xmm5, %xmm9
 ; SSE2-NEXT:    pxor %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm8
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm8
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm10, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm5[1,1,3,3]
 ; SSE2-NEXT:    por %xmm8, %xmm10
 ; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm7
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm8, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm6
-; SSE2-NEXT:    packssdw %xmm10, %xmm6
-; SSE2-NEXT:    packssdw %xmm9, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm4
-; SSE2-NEXT:    movdqa %xmm5, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm5
+; SSE2-NEXT:    packssdw %xmm10, %xmm5
+; SSE2-NEXT:    packssdw %xmm9, %xmm5
+; SSE2-NEXT:    pxor %xmm0, %xmm6
+; SSE2-NEXT:    movdqa %xmm4, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm7
+; SSE2-NEXT:    por %xmm6, %xmm7
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    movdqa %xmm4, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm8, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    por %xmm9, %xmm3
 ; SSE2-NEXT:    packssdw %xmm7, %xmm3
 ; SSE2-NEXT:    pxor %xmm0, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm4, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm6
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm5, %xmm0
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    packssdw %xmm4, %xmm0
+; SSE2-NEXT:    packssdw %xmm6, %xmm0
 ; SSE2-NEXT:    packssdw %xmm3, %xmm0
-; SSE2-NEXT:    packsswb %xmm6, %xmm0
+; SSE2-NEXT:    packsswb %xmm5, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: lane_mask_v16i1_i64:
@@ -1596,72 +1596,72 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
 ; SSE42-NEXT:    movq %rdi, %xmm0
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
-; SSE42-NEXT:    movdqa %xmm0, %xmm5
-; SSE42-NEXT:    pxor %xmm1, %xmm5
+; SSE42-NEXT:    movdqa %xmm0, %xmm4
+; SSE42-NEXT:    pxor %xmm1, %xmm4
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
 ; SSE42-NEXT:    paddq %xmm0, %xmm3
-; SSE42-NEXT:    movdqa %xmm3, %xmm4
-; SSE42-NEXT:    pxor %xmm1, %xmm4
-; SSE42-NEXT:    movdqa %xmm5, %xmm2
-; SSE42-NEXT:    pcmpgtq %xmm4, %xmm2
+; SSE42-NEXT:    movdqa %xmm3, %xmm5
+; SSE42-NEXT:    pxor %xmm1, %xmm5
+; SSE42-NEXT:    movdqa %xmm4, %xmm2
+; SSE42-NEXT:    pcmpgtq %xmm5, %xmm2
 ; SSE42-NEXT:    por %xmm3, %xmm2
-; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [2,3]
-; SSE42-NEXT:    paddq %xmm0, %xmm4
-; SSE42-NEXT:    movdqa %xmm4, %xmm6
+; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [2,3]
+; SSE42-NEXT:    paddq %xmm0, %xmm5
+; SSE42-NEXT:    movdqa %xmm5, %xmm6
 ; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm5, %xmm3
+; SSE42-NEXT:    movdqa %xmm4, %xmm3
 ; SSE42-NEXT:    pcmpgtq %xmm6, %xmm3
-; SSE42-NEXT:    por %xmm4, %xmm3
+; SSE42-NEXT:    por %xmm5, %xmm3
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm6 = [4,5]
 ; SSE42-NEXT:    paddq %xmm0, %xmm6
 ; SSE42-NEXT:    movdqa %xmm6, %xmm7
 ; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm5, %xmm4
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm4
-; SSE42-NEXT:    por %xmm6, %xmm4
+; SSE42-NEXT:    movdqa %xmm4, %xmm5
+; SSE42-NEXT:    pcmpgtq %xmm7, %xmm5
+; SSE42-NEXT:    por %xmm6, %xmm5
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm7 = [6,7]
 ; SSE42-NEXT:    paddq %xmm0, %xmm7
 ; SSE42-NEXT:    movdqa %xmm7, %xmm8
 ; SSE42-NEXT:    pxor %xmm1, %xmm8
-; SSE42-NEXT:    movdqa %xmm5, %xmm6
+; SSE42-NEXT:    movdqa %xmm4, %xmm6
 ; SSE42-NEXT:    pcmpgtq %xmm8, %xmm6
 ; SSE42-NEXT:    por %xmm7, %xmm6
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm8 = [8,9]
 ; SSE42-NEXT:    paddq %xmm0, %xmm8
 ; SSE42-NEXT:    movdqa %xmm8, %xmm9
 ; SSE42-NEXT:    pxor %xmm1, %xmm9
-; SSE42-NEXT:    movdqa %xmm5, %xmm7
+; SSE42-NEXT:    movdqa %xmm4, %xmm7
 ; SSE42-NEXT:    pcmpgtq %xmm9, %xmm7
 ; SSE42-NEXT:    por %xmm8, %xmm7
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm9 = [10,11]
 ; SSE42-NEXT:    paddq %xmm0, %xmm9
 ; SSE42-NEXT:    movdqa %xmm9, %xmm10
 ; SSE42-NEXT:    pxor %xmm1, %xmm10
-; SSE42-NEXT:    movdqa %xmm5, %xmm8
+; SSE42-NEXT:    movdqa %xmm4, %xmm8
 ; SSE42-NEXT:    pcmpgtq %xmm10, %xmm8
 ; SSE42-NEXT:    por %xmm9, %xmm8
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm9 = [12,13]
 ; SSE42-NEXT:    paddq %xmm0, %xmm9
 ; SSE42-NEXT:    movdqa %xmm9, %xmm10
 ; SSE42-NEXT:    pxor %xmm1, %xmm10
-; SSE42-NEXT:    movdqa %xmm5, %xmm11
+; SSE42-NEXT:    movdqa %xmm4, %xmm11
 ; SSE42-NEXT:    pcmpgtq %xmm10, %xmm11
 ; SSE42-NEXT:    por %xmm9, %xmm11
 ; SSE42-NEXT:    paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [14,15]
 ; SSE42-NEXT:    movdqa %xmm0, %xmm9
 ; SSE42-NEXT:    pxor %xmm1, %xmm9
-; SSE42-NEXT:    pcmpgtq %xmm9, %xmm5
-; SSE42-NEXT:    por %xmm0, %xmm5
+; SSE42-NEXT:    pcmpgtq %xmm9, %xmm4
+; SSE42-NEXT:    por %xmm0, %xmm4
 ; SSE42-NEXT:    movq %rsi, %xmm0
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE42-NEXT:    pxor %xmm1, %xmm5
+; SSE42-NEXT:    pxor %xmm1, %xmm4
 ; SSE42-NEXT:    pxor %xmm1, %xmm0
 ; SSE42-NEXT:    movdqa %xmm0, %xmm9
-; SSE42-NEXT:    pcmpgtq %xmm5, %xmm9
+; SSE42-NEXT:    pcmpgtq %xmm4, %xmm9
 ; SSE42-NEXT:    pxor %xmm1, %xmm11
-; SSE42-NEXT:    movdqa %xmm0, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm11, %xmm5
-; SSE42-NEXT:    packssdw %xmm9, %xmm5
+; SSE42-NEXT:    movdqa %xmm0, %xmm4
+; SSE42-NEXT:    pcmpgtq %xmm11, %xmm4
+; SSE42-NEXT:    packssdw %xmm9, %xmm4
 ; SSE42-NEXT:    pxor %xmm1, %xmm8
 ; SSE42-NEXT:    movdqa %xmm0, %xmm9
 ; SSE42-NEXT:    pcmpgtq %xmm8, %xmm9
@@ -1669,14 +1669,14 @@ define <16 x i1> @lane_mask_v16i1_i64(i64 %index, i64 %TC) {
 ; SSE42-NEXT:    movdqa %xmm0, %xmm8
 ; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
 ; SSE42-NEXT:    packssdw %xmm9, %xmm8
-; SSE42-NEXT:    packssdw %xmm5, %xmm8
+; SSE42-NEXT:    packssdw %xmm4, %xmm8
 ; SSE42-NEXT:    pxor %xmm1, %xmm6
-; SSE42-NEXT:    movdqa %xmm0, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm6, %xmm5
-; SSE42-NEXT:    pxor %xmm1, %xmm4
+; SSE42-NEXT:    movdqa %xmm0, %xmm4
+; SSE42-NEXT:    pcmpgtq %xmm6, %xmm4
+; SSE42-NEXT:    pxor %xmm1, %xmm5
 ; SSE42-NEXT:    movdqa %xmm0, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm4, %xmm6
-; SSE42-NEXT:    packssdw %xmm5, %xmm6
+; SSE42-NEXT:    pcmpgtq %xmm5, %xmm6
+; SSE42-NEXT:    packssdw %xmm4, %xmm6
 ; SSE42-NEXT:    pxor %xmm1, %xmm3
 ; SSE42-NEXT:    movdqa %xmm0, %xmm4
 ; SSE42-NEXT:    pcmpgtq %xmm3, %xmm4
@@ -2282,20 +2282,9 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
 ; SSE42-NEXT:    pcmpgtq %xmm6, %xmm7
 ; SSE42-NEXT:    por %xmm5, %xmm7
 ; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm3, %xmm5
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm5
-; SSE42-NEXT:    movdqa {{.*#+}} xmm6 = [20,21]
-; SSE42-NEXT:    paddq %xmm2, %xmm6
-; SSE42-NEXT:    movdqa %xmm6, %xmm7
-; SSE42-NEXT:    pxor %xmm1, %xmm7
-; SSE42-NEXT:    movdqa %xmm0, %xmm8
-; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
-; SSE42-NEXT:    por %xmm6, %xmm8
-; SSE42-NEXT:    pxor %xmm1, %xmm8
 ; SSE42-NEXT:    movdqa %xmm3, %xmm6
-; SSE42-NEXT:    pcmpgtq %xmm8, %xmm6
-; SSE42-NEXT:    packssdw %xmm5, %xmm6
-; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [18,19]
+; SSE42-NEXT:    pcmpgtq %xmm7, %xmm6
+; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [20,21]
 ; SSE42-NEXT:    paddq %xmm2, %xmm5
 ; SSE42-NEXT:    movdqa %xmm5, %xmm7
 ; SSE42-NEXT:    pxor %xmm1, %xmm7
@@ -2305,6 +2294,17 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
 ; SSE42-NEXT:    pxor %xmm1, %xmm8
 ; SSE42-NEXT:    movdqa %xmm3, %xmm5
 ; SSE42-NEXT:    pcmpgtq %xmm8, %xmm5
+; SSE42-NEXT:    packssdw %xmm6, %xmm5
+; SSE42-NEXT:    movdqa {{.*#+}} xmm6 = [18,19]
+; SSE42-NEXT:    paddq %xmm2, %xmm6
+; SSE42-NEXT:    movdqa %xmm6, %xmm7
+; SSE42-NEXT:    pxor %xmm1, %xmm7
+; SSE42-NEXT:    movdqa %xmm0, %xmm8
+; SSE42-NEXT:    pcmpgtq %xmm7, %xmm8
+; SSE42-NEXT:    por %xmm6, %xmm8
+; SSE42-NEXT:    pxor %xmm1, %xmm8
+; SSE42-NEXT:    movdqa %xmm3, %xmm6
+; SSE42-NEXT:    pcmpgtq %xmm8, %xmm6
 ; SSE42-NEXT:    paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 # [16,17]
 ; SSE42-NEXT:    movdqa %xmm2, %xmm7
 ; SSE42-NEXT:    pxor %xmm1, %xmm7
@@ -2312,8 +2312,8 @@ define <32 x i1> @lane_mask_v32i1_i64(i64 %index, i64 %TC) {
 ; SSE42-NEXT:    por %xmm2, %xmm0
 ; SSE42-NEXT:    pxor %xmm1, %xmm0
 ; SSE42-NEXT:    pcmpgtq %xmm0, %xmm3
-; SSE42-NEXT:    packssdw %xmm5, %xmm3
 ; SSE42-NEXT:    packssdw %xmm6, %xmm3
+; SSE42-NEXT:    packssdw %xmm5, %xmm3
 ; SSE42-NEXT:    packsswb %xmm4, %xmm3
 ; SSE42-NEXT:    pmovmskb %xmm3, %edx
 ; SSE42-NEXT:    shll $16, %edx

>From 81f07d0929f89c354deca8f056ec731793c686af Mon Sep 17 00:00:00 2001
From: william <we3223 at gmail.com>
Date: Mon, 5 Oct 2026 22:59:13 +0800
Subject: [PATCH 3/5] [X86][TTI] Enable DataAndControlFlow tail folding style
 on AVX-512

---
 llvm/lib/Target/X86/X86TargetTransformInfo.h  |   7 +-
 .../LoopVectorize/X86/cost-model.ll           |   8 +-
 .../X86/divs-with-tail-folding.ll             |  47 ++---
 .../X86/fold-tail-low-trip-count.ll           |  34 ++--
 .../Transforms/LoopVectorize/X86/optsize.ll   |  96 ++++------
 .../X86/scev-checks-unprofitable.ll           |   5 +-
 .../X86/vectorize-interleaved-accesses-gap.ll |  33 ++--
 .../widen-canonical-iv-register-pressure.ll   |  59 +++---
 .../x86-interleaved-accesses-masked-group.ll  | 178 +++++++-----------
 ...86-interleaved-store-accesses-with-gaps.ll |  60 +++---
 10 files changed, 228 insertions(+), 299 deletions(-)

diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 41fbcc77304fb2..f87cd2fee49271 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -57,7 +57,12 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
 
   /// \name Vector TTI Implementations
   /// @{
-
+  
+  TailFoldingStyle
+  getPreferredTailFoldingStyle() const {
+    return ST->hasAVX512() ? TailFoldingStyle::DataAndControlFlow
+                           : TailFoldingStyle::DataWithoutLaneMask;
+  }
   unsigned getNumberOfRegisters(unsigned ClassID) const override;
   unsigned getRegisterClassForType(bool Vector, Type *Ty) const override;
   bool hasConditionalLoadStoreForType(Type *Ty, bool IsStore) const override;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 42c8fe3011f7d2..26dc78ff94b76b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -981,20 +981,22 @@ define void @replicating_sdiv_operand_profitable_to_scalarize(i32 %x) #3 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 0, i32 13)
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[X]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> splat (i32 2), <16 x i32> [[BROADCAST_SPLAT]], <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>)
+; CHECK-NEXT:    [[TMP0:%.*]] = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> splat (i32 2), <16 x i32> [[BROADCAST_SPLAT]], <16 x i1> [[ACTIVE_LANE_MASK_ENTRY]])
 ; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> zeroinitializer, <16 x i32> [[TMP0]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub <16 x i32> [[BROADCAST_SPLAT]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i32> [[TMP2]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = or <16 x i32> [[TMP0]], splat (i32 3)
-; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> [[TMP0]], <16 x i32> [[TMP4]], <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> [[TMP0]], <16 x i32> [[TMP4]], <16 x i1> [[ACTIVE_LANE_MASK_ENTRY]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i32> [[TMP3]], [[TMP5]]
-; CHECK-NEXT:    [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> <i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true>, i1 false)
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], splat (i1 true)
+; CHECK-NEXT:    [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> [[TMP8]], i1 false)
 ; CHECK-NEXT:    [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x i32> [[TMP6]], i64 [[LAST_ACTIVE_LANE]]
 ; CHECK-NEXT:    br label %[[EXIT1:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
index fada3253c14717..0d7a3dc5163ce4 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
@@ -16,17 +16,11 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[N]])
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP16]], i64 [[CONV6]], i64 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = sdiv i64 [[M]], [[TMP9]]
@@ -40,9 +34,10 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
 ; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP25]]
 ; CHECK-NEXT:    call void @llvm.masked.store.v4f64.p0(<4 x double> zeroinitializer, ptr align 8 [[TMP26]], <4 x i1> [[TMP6]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IV]], splat (i64 4)
-; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP18]], true
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -103,19 +98,14 @@ define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i
 ; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[N]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP8]], i64 0
@@ -131,9 +121,11 @@ define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i
 ; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP27]]
 ; CHECK-NEXT:    call void @llvm.masked.store.v4f64.p0(<4 x double> zeroinitializer, ptr align 8 [[TMP28]], <4 x i1> [[TMP8]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[TMP5]], 4
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = xor i1 [[TMP18]], true
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -207,19 +199,14 @@ define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[N_RND_UP]], 3
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP5]]
-; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP0]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[MUL_2_I]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = udiv <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP7]], i64 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = urem i64 [[INDEX]], [[MUL_2_I]]
@@ -238,9 +225,11 @@ define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP21]]
 ; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP7]], ptr align 4 [[TMP22]], <4 x i1> [[TMP6]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = xor i1 [[TMP23]], true
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll
index b92a591ae5147c..6636b358c3a8e0 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll
@@ -50,27 +50,22 @@ define ptr @low_trip_count_via_profile_info_with_iv_live_out(ptr align 16 %start
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP1]], 63
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 63
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP1]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <64 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <64 x i64> [[BROADCAST_SPLATINSERT]], <64 x i64> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 0, i64 [[TMP1]])
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IV:%.*]] = phi <64 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31, i64 32, i64 33, i64 34, i64 35, i64 36, i64 37, i64 38, i64 39, i64 40, i64 41, i64 42, i64 43, i64 44, i64 45, i64 46, i64 47, i64 48, i64 49, i64 50, i64 51, i64 52, i64 53, i64 54, i64 55, i64 56, i64 57, i64 58, i64 59, i64 60, i64 61, i64 62, i64 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <64 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi <64 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[TMP5]], <64 x i1> [[TMP3]], <64 x i8> poison)
 ; CHECK-NEXT:    call void @llvm.masked.store.v64i8.p0(<64 x i8> [[WIDE_MASKED_LOAD]], ptr align 1 [[NEXT_GEP]], <64 x i1> [[TMP3]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 64
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <64 x i64> [[VEC_IV]], splat (i64 64)
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:
@@ -104,26 +99,21 @@ define void @low_trip_count_via_profile_info(ptr align 16 %start, ptr align 16 %
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP7]], [[TMP6]]
 ; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP1]], 63
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 63
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <64 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <64 x i64> [[BROADCAST_SPLATINSERT]], <64 x i64> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 0, i64 [[TMP1]])
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IV:%.*]] = phi <64 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31, i64 32, i64 33, i64 34, i64 35, i64 36, i64 37, i64 38, i64 39, i64 40, i64 41, i64 42, i64 43, i64 44, i64 45, i64 46, i64 47, i64 48, i64 49, i64 50, i64 51, i64 52, i64 53, i64 54, i64 55, i64 56, i64 57, i64 58, i64 59, i64 60, i64 61, i64 62, i64 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule <64 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <64 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[TMP4]], <64 x i1> [[TMP2]], <64 x i8> poison)
 ; CHECK-NEXT:    call void @llvm.masked.store.v64i8.p0(<64 x i8> [[WIDE_MASKED_LOAD]], ptr align 1 [[NEXT_GEP]], <64 x i1> [[TMP2]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 64
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <64 x i64> [[VEC_IV]], splat (i64 64)
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP1]])
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i1 [[TMP8]], true
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/optsize.ll b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
index c78688c348b840..0dc598e7b4e7bb 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
@@ -15,19 +15,20 @@ define i32 @foo_optsize() #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 0, i32 203)
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <64 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 39, i8 40, i8 41, i8 42, i8 43, i8 44, i8 45, i8 46, i8 47, i8 48, i8 49, i8 50, i8 51, i8 52, i8 53, i8 54, i8 55, i8 56, i8 57, i8 58, i8 59, i8 60, i8 61, i8 62, i8 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <64 x i8> [[VEC_IND]], splat (i8 -54)
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <64 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[TMP2]], <64 x i1> [[TMP1]], <64 x i8> poison)
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <64 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = select <64 x i1> [[TMP4]], <64 x i8> splat (i8 2), <64 x i8> splat (i8 1)
 ; CHECK-NEXT:    call void @llvm.masked.store.v64i8.p0(<64 x i8> [[TMP5]], ptr align 1 [[TMP2]], <64 x i1> [[TMP1]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 64
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <64 x i8> [[VEC_IND]], splat (i8 64)
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 64
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 [[INDEX_NEXT]], i32 203)
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i1 [[TMP3]], true
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[FOR_END:.*]]
@@ -39,19 +40,20 @@ define i32 @foo_optsize() #0 {
 ; AUTOVF-NEXT:  [[ENTRY:.*:]]
 ; AUTOVF-NEXT:    br label %[[VECTOR_PH:.*]]
 ; AUTOVF:       [[VECTOR_PH]]:
+; AUTOVF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i32(i32 0, i32 203)
 ; AUTOVF-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; AUTOVF:       [[VECTOR_BODY]]:
 ; AUTOVF-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AUTOVF-NEXT:    [[VEC_IND:%.*]] = phi <32 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AUTOVF-NEXT:    [[TMP1:%.*]] = icmp ule <32 x i8> [[VEC_IND]], splat (i8 -54)
+; AUTOVF-NEXT:    [[TMP1:%.*]] = phi <32 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; AUTOVF-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[INDEX]]
 ; AUTOVF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[TMP2]], <32 x i1> [[TMP1]], <32 x i8> poison)
 ; AUTOVF-NEXT:    [[TMP4:%.*]] = icmp eq <32 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
 ; AUTOVF-NEXT:    [[TMP5:%.*]] = select <32 x i1> [[TMP4]], <32 x i8> splat (i8 2), <32 x i8> splat (i8 1)
 ; AUTOVF-NEXT:    call void @llvm.masked.store.v32i8.p0(<32 x i8> [[TMP5]], ptr align 1 [[TMP2]], <32 x i1> [[TMP1]])
-; AUTOVF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
-; AUTOVF-NEXT:    [[VEC_IND_NEXT]] = add nuw <32 x i8> [[VEC_IND]], splat (i8 32)
-; AUTOVF-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 224
+; AUTOVF-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 32
+; AUTOVF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i32(i32 [[INDEX_NEXT]], i32 203)
+; AUTOVF-NEXT:    [[TMP3:%.*]] = extractelement <32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; AUTOVF-NEXT:    [[TMP6:%.*]] = xor i1 [[TMP3]], true
 ; AUTOVF-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; AUTOVF:       [[MIDDLE_BLOCK]]:
 ; AUTOVF-NEXT:    br label %[[FOR_END:.*]]
@@ -85,19 +87,20 @@ define i32 @foo_minsize() #1 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 0, i32 203)
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <64 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 39, i8 40, i8 41, i8 42, i8 43, i8 44, i8 45, i8 46, i8 47, i8 48, i8 49, i8 50, i8 51, i8 52, i8 53, i8 54, i8 55, i8 56, i8 57, i8 58, i8 59, i8 60, i8 61, i8 62, i8 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <64 x i8> [[VEC_IND]], splat (i8 -54)
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <64 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[TMP2]], <64 x i1> [[TMP1]], <64 x i8> poison)
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <64 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = select <64 x i1> [[TMP4]], <64 x i8> splat (i8 2), <64 x i8> splat (i8 1)
 ; CHECK-NEXT:    call void @llvm.masked.store.v64i8.p0(<64 x i8> [[TMP5]], ptr align 1 [[TMP2]], <64 x i1> [[TMP1]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 64
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <64 x i8> [[VEC_IND]], splat (i8 64)
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 64
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 [[INDEX_NEXT]], i32 203)
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i1 [[TMP3]], true
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[FOR_END:.*]]
@@ -106,25 +109,18 @@ define i32 @foo_minsize() #1 {
 ;
 ; AUTOVF-LABEL: define i32 @foo_minsize(
 ; AUTOVF-SAME: ) #[[ATTR1:[0-9]+]] {
-; AUTOVF-NEXT:  [[ENTRY:.*:]]
-; AUTOVF-NEXT:    br label %[[VECTOR_PH:.*]]
-; AUTOVF:       [[VECTOR_PH]]:
+; AUTOVF-NEXT:  [[VECTOR_PH:.*]]:
 ; AUTOVF-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; AUTOVF:       [[VECTOR_BODY]]:
-; AUTOVF-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AUTOVF-NEXT:    [[VEC_IND:%.*]] = phi <16 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; AUTOVF-NEXT:    [[TMP0:%.*]] = icmp ule <16 x i8> [[VEC_IND]], splat (i8 -54)
+; AUTOVF-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INC:%.*]], %[[VECTOR_BODY]] ]
 ; AUTOVF-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[INDEX]]
-; AUTOVF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[TMP0]], <16 x i8> poison)
-; AUTOVF-NEXT:    [[TMP5:%.*]] = icmp eq <16 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
-; AUTOVF-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP5]], <16 x i8> splat (i8 2), <16 x i8> splat (i8 1)
-; AUTOVF-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP3]], ptr align 1 [[TMP2]], <16 x i1> [[TMP0]])
-; AUTOVF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; AUTOVF-NEXT:    [[VEC_IND_NEXT]] = add nuw <16 x i8> [[VEC_IND]], splat (i8 16)
-; AUTOVF-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 208
-; AUTOVF-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; AUTOVF:       [[MIDDLE_BLOCK]]:
-; AUTOVF-NEXT:    br label %[[FOR_END:.*]]
+; AUTOVF-NEXT:    [[TMP0:%.*]] = load i8, ptr [[TMP2]], align 1
+; AUTOVF-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0
+; AUTOVF-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
+; AUTOVF-NEXT:    store i8 [[DOT]], ptr [[TMP2]], align 1
+; AUTOVF-NEXT:    [[INC]] = add nsw i32 [[INDEX]], 1
+; AUTOVF-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INDEX]], 202
+; AUTOVF-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]]
 ; AUTOVF:       [[FOR_END]]:
 ; AUTOVF-NEXT:    ret i32 0
 ;
@@ -196,7 +192,7 @@ define void @scev4stride1(ptr noalias nocapture %a, ptr noalias nocapture readon
 ; AUTOVF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; AUTOVF-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
 ; AUTOVF-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
-; AUTOVF-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; AUTOVF-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; AUTOVF:       [[MIDDLE_BLOCK]]:
 ; AUTOVF-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
 ; AUTOVF:       [[FOR_END_LOOPEXIT]]:
@@ -333,24 +329,19 @@ define void @tail_folded_store_avx512(ptr %start, ptr %end) #3 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], 1
 ; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP3]], 63
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 63
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP3]], 1
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <64 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <64 x i32> [[BROADCAST_SPLATINSERT3]], <64 x i32> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 0, i32 [[TMP3]])
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = phi <64 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IV:%.*]] = phi <64 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <64 x i32> <i32 0, i32 -72, i32 -144, i32 -216, i32 -288, i32 -360, i32 -432, i32 -504, i32 -576, i32 -648, i32 -720, i32 -792, i32 -864, i32 -936, i32 -1008, i32 -1080, i32 -1152, i32 -1224, i32 -1296, i32 -1368, i32 -1440, i32 -1512, i32 -1584, i32 -1656, i32 -1728, i32 -1800, i32 -1872, i32 -1944, i32 -2016, i32 -2088, i32 -2160, i32 -2232, i32 -2304, i32 -2376, i32 -2448, i32 -2520, i32 -2592, i32 -2664, i32 -2736, i32 -2808, i32 -2880, i32 -2952, i32 -3024, i32 -3096, i32 -3168, i32 -3240, i32 -3312, i32 -3384, i32 -3456, i32 -3528, i32 -3600, i32 -3672, i32 -3744, i32 -3816, i32 -3888, i32 -3960, i32 -4032, i32 -4104, i32 -4176, i32 -4248, i32 -4320, i32 -4392, i32 -4464, i32 -4536>
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule <64 x i32> [[VEC_IV]], [[BROADCAST_SPLAT4]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.v64p0.v64p0(<64 x ptr> splat (ptr null), <64 x ptr> align 8 [[TMP5]], <64 x i1> [[TMP6]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 64
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i32(i32 [[INDEX_NEXT]], i32 [[TMP3]])
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <64 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i1 [[TMP9]], true
 ; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 -4608
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <64 x i32> [[VEC_IV]], splat (i32 64)
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
@@ -368,25 +359,20 @@ define void @tail_folded_store_avx512(ptr %start, ptr %end) #3 {
 ; AUTOVF-NEXT:    [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], 1
 ; AUTOVF-NEXT:    br label %[[VECTOR_PH:.*]]
 ; AUTOVF:       [[VECTOR_PH]]:
-; AUTOVF-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP3]], 7
-; AUTOVF-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; AUTOVF-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; AUTOVF-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP3]], 1
-; AUTOVF-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; AUTOVF-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT3]], <8 x i32> poison, <8 x i32> zeroinitializer
+; AUTOVF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[TMP3]])
 ; AUTOVF-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; AUTOVF:       [[VECTOR_BODY]]:
 ; AUTOVF-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AUTOVF-NEXT:    [[TMP6:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; AUTOVF-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
-; AUTOVF-NEXT:    [[VEC_IV:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; AUTOVF-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <8 x i32> <i32 0, i32 -72, i32 -144, i32 -216, i32 -288, i32 -360, i32 -432, i32 -504>
-; AUTOVF-NEXT:    [[TMP6:%.*]] = icmp ule <8 x i32> [[VEC_IV]], [[BROADCAST_SPLAT4]]
 ; AUTOVF-NEXT:    call void @llvm.masked.scatter.v8p0.v8p0(<8 x ptr> splat (ptr null), <8 x ptr> align 8 [[TMP5]], <8 x i1> [[TMP6]])
-; AUTOVF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; AUTOVF-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; AUTOVF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[TMP3]])
+; AUTOVF-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; AUTOVF-NEXT:    [[TMP7:%.*]] = xor i1 [[TMP9]], true
 ; AUTOVF-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 -576
-; AUTOVF-NEXT:    [[VEC_IND_NEXT]] = add nuw <8 x i32> [[VEC_IV]], splat (i32 8)
-; AUTOVF-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; AUTOVF-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; AUTOVF-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; AUTOVF:       [[MIDDLE_BLOCK]]:
 ; AUTOVF-NEXT:    br label %[[EXIT:.*]]
 ; AUTOVF:       [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll b/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll
index 16e65d3f1b3526..e2c4996f51701c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll
@@ -22,12 +22,9 @@ define void @value_defined_in_loop1_used_for_trip_counts(i32 %start, i1 %c, ptr
 ; CHECK:       [[LOOP_3_PREHEADER]]:
 ; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[IV_1]], 1
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 [[IV_1]])
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> zeroinitializer, ptr align 1 [[DST]], <8 x i1> [[TMP0]])
 ; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
index ead2f8481e5398..c9a74ae42a8382 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
@@ -9,11 +9,11 @@ define void @test_pr59090(ptr %l_out, ptr noalias %b) #0 {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 10001)
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <8 x i16> [ <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <8 x i16> [[VEC_IND]], splat (i16 10000)
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[INDEX]], 6
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr [[B:%.*]], align 1, !llvm.access.group [[ACC_GRP0:![0-9]+]]
 ; CHECK-NEXT:    store i8 [[TMP3]], ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0]]
@@ -21,10 +21,11 @@ define void @test_pr59090(ptr %l_out, ptr noalias %b) #0 {
 ; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> poison, <48 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
 ; CHECK-NEXT:    [[TMP15:%.*]] = and <48 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false>
 ; CHECK-NEXT:    call void @llvm.masked.store.v48i8.p0(<48 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison>, ptr align 1 [[TMP13]], <48 x i1> [[TMP15]]), !llvm.access.group [[ACC_GRP0]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <8 x i16> [[VEC_IND]], splat (i16 8)
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10008
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT]], i64 10001)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       exit:
@@ -56,13 +57,14 @@ define void @test_pr59090_interleave(ptr %l_out, ptr noalias %b) #0 {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 10001)
+; CHECK-NEXT:    [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; CHECK-NEXT:    [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add nuw <8 x i64> [[VEC_IND]], splat (i64 8)
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IND]], splat (i64 10000)
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule <8 x i64> [[STEP_ADD]], splat (i64 10000)
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[INDEX]], 6
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP0]], 6
@@ -76,10 +78,13 @@ define void @test_pr59090_interleave(ptr %l_out, ptr noalias %b) #0 {
 ; CHECK-NEXT:    [[INTERLEAVED_MASK1:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <48 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
 ; CHECK-NEXT:    [[TMP9:%.*]] = and <48 x i1> [[INTERLEAVED_MASK1]], <i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false>
 ; CHECK-NEXT:    call void @llvm.masked.store.v48i8.p0(<48 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison>, ptr align 1 [[TMP7]], <48 x i1> [[TMP9]]), !llvm.access.group [[ACC_GRP0]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <8 x i64> [[STEP_ADD]], splat (i64 8)
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10016
-; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[INDEX_NEXT]], i64 10001)
+; CHECK-NEXT:    [[EXTRACT_NEXT_ALM_PART]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; CHECK-NEXT:    [[EXTRACT_NEXT_ALM_PART4]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       exit:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/widen-canonical-iv-register-pressure.ll b/llvm/test/Transforms/LoopVectorize/X86/widen-canonical-iv-register-pressure.ll
index f147f9f9beca28..e4683219283432 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/widen-canonical-iv-register-pressure.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/widen-canonical-iv-register-pressure.ll
@@ -12,15 +12,14 @@ define i32 @many_reductions(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, i64
 ; UF2-NEXT:  [[ENTRY:.*:]]
 ; UF2-NEXT:    br label %[[VECTOR_PH:.*]]
 ; UF2:       [[VECTOR_PH]]:
-; UF2-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], 15
-; UF2-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 15
-; UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; UF2-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; UF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; UF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; UF2-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 [[N]])
+; UF2-NEXT:    [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; UF2-NEXT:    [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
 ; UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; UF2:       [[VECTOR_BODY]]:
 ; UF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UF2-NEXT:    [[TMP0:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], %[[VECTOR_BODY]] ]
+; UF2-NEXT:    [[TMP1:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART25:%.*]], %[[VECTOR_BODY]] ]
 ; UF2-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
 ; UF2-NEXT:    [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
 ; UF2-NEXT:    [[VEC_PHI2:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
@@ -33,10 +32,6 @@ define i32 @many_reductions(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, i64
 ; UF2-NEXT:    [[VEC_PHI9:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
 ; UF2-NEXT:    [[VEC_PHI10:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
 ; UF2-NEXT:    [[VEC_PHI11:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
-; UF2-NEXT:    [[VEC_IV:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; UF2-NEXT:    [[VEC_IV16:%.*]] = add nuw <8 x i64> [[VEC_IV]], splat (i64 8)
-; UF2-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; UF2-NEXT:    [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IV16]], [[BROADCAST_SPLAT]]
 ; UF2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; UF2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
 ; UF2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
@@ -74,9 +69,12 @@ define i32 @many_reductions(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, i64
 ; UF2-NEXT:    [[TMP24]] = add <8 x i32> [[VEC_PHI10]], [[WIDE_MASKED_LOAD26]]
 ; UF2-NEXT:    [[TMP25]] = add <8 x i32> [[VEC_PHI11]], [[WIDE_MASKED_LOAD27]]
 ; UF2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
-; UF2-NEXT:    [[VEC_IND_NEXT]] = add nuw <8 x i64> [[VEC_IV16]], splat (i64 8)
-; UF2-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UF2-NEXT:    br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; UF2-NEXT:    [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; UF2-NEXT:    [[EXTRACT_NEXT_ALM_PART]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; UF2-NEXT:    [[EXTRACT_NEXT_ALM_PART25]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
+; UF2-NEXT:    [[TMP26:%.*]] = extractelement <8 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; UF2-NEXT:    [[TMP45:%.*]] = xor i1 [[TMP26]], true
+; UF2-NEXT:    br i1 [[TMP45]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; UF2:       [[MIDDLE_BLOCK]]:
 ; UF2-NEXT:    [[TMP27:%.*]] = select <8 x i1> [[TMP0]], <8 x i32> [[TMP14]], <8 x i32> [[VEC_PHI]]
 ; UF2-NEXT:    [[TMP28:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> [[TMP15]], <8 x i32> [[VEC_PHI1]]
@@ -116,15 +114,18 @@ define i32 @many_reductions(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, i64
 ; UF4-NEXT:  [[ENTRY:.*:]]
 ; UF4-NEXT:    br label %[[VECTOR_PH:.*]]
 ; UF4:       [[VECTOR_PH]]:
-; UF4-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], 31
-; UF4-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 31
-; UF4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; UF4-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; UF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; UF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; UF4-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 [[N]])
+; UF4-NEXT:    [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; UF4-NEXT:    [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
+; UF4-NEXT:    [[EXTRACT_ENTRY_ALM_PART2:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 16)
+; UF4-NEXT:    [[EXTRACT_ENTRY_ALM_PART3:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 24)
 ; UF4-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; UF4:       [[VECTOR_BODY]]:
 ; UF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; UF4-NEXT:    [[TMP0:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], %[[VECTOR_BODY]] ]
+; UF4-NEXT:    [[TMP1:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART53:%.*]], %[[VECTOR_BODY]] ]
+; UF4-NEXT:    [[TMP2:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART2]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART54:%.*]], %[[VECTOR_BODY]] ]
+; UF4-NEXT:    [[TMP3:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART3]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART55:%.*]], %[[VECTOR_BODY]] ]
 ; UF4-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
 ; UF4-NEXT:    [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
 ; UF4-NEXT:    [[VEC_PHI2:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
@@ -149,16 +150,6 @@ define i32 @many_reductions(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, i64
 ; UF4-NEXT:    [[VEC_PHI21:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP49:%.*]], %[[VECTOR_BODY]] ]
 ; UF4-NEXT:    [[VEC_PHI22:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP50:%.*]], %[[VECTOR_BODY]] ]
 ; UF4-NEXT:    [[VEC_PHI23:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP51:%.*]], %[[VECTOR_BODY]] ]
-; UF4-NEXT:    [[BROADCAST_SPLATINSERT24:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0
-; UF4-NEXT:    [[BROADCAST_SPLAT25:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT24]], <8 x i64> poison, <8 x i32> zeroinitializer
-; UF4-NEXT:    [[VEC_IV:%.*]] = add nuw <8 x i64> [[BROADCAST_SPLAT25]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
-; UF4-NEXT:    [[VEC_IV28:%.*]] = add nuw <8 x i64> [[BROADCAST_SPLAT25]], <i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>
-; UF4-NEXT:    [[VEC_IV31:%.*]] = add nuw <8 x i64> [[BROADCAST_SPLAT25]], <i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23>
-; UF4-NEXT:    [[VEC_IV34:%.*]] = add nuw <8 x i64> [[BROADCAST_SPLAT25]], <i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>
-; UF4-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; UF4-NEXT:    [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IV28]], [[BROADCAST_SPLAT]]
-; UF4-NEXT:    [[TMP2:%.*]] = icmp ule <8 x i64> [[VEC_IV31]], [[BROADCAST_SPLAT]]
-; UF4-NEXT:    [[TMP3:%.*]] = icmp ule <8 x i64> [[VEC_IV34]], [[BROADCAST_SPLAT]]
 ; UF4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; UF4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
 ; UF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
@@ -232,8 +223,14 @@ define i32 @many_reductions(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, i64
 ; UF4-NEXT:    [[TMP50]] = add <8 x i32> [[VEC_PHI22]], [[WIDE_MASKED_LOAD56]]
 ; UF4-NEXT:    [[TMP51]] = add <8 x i32> [[VEC_PHI23]], [[WIDE_MASKED_LOAD57]]
 ; UF4-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
-; UF4-NEXT:    [[TMP52:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; UF4-NEXT:    br i1 [[TMP52]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; UF4-NEXT:    [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; UF4-NEXT:    [[EXTRACT_NEXT_ALM_PART]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
+; UF4-NEXT:    [[EXTRACT_NEXT_ALM_PART53]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
+; UF4-NEXT:    [[EXTRACT_NEXT_ALM_PART54]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 16)
+; UF4-NEXT:    [[EXTRACT_NEXT_ALM_PART55]] = call <8 x i1> @llvm.vector.extract.v8i1.v32i1(<32 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 24)
+; UF4-NEXT:    [[TMP52:%.*]] = extractelement <8 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
+; UF4-NEXT:    [[TMP83:%.*]] = xor i1 [[TMP52]], true
+; UF4-NEXT:    br i1 [[TMP83]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; UF4:       [[MIDDLE_BLOCK]]:
 ; UF4-NEXT:    [[TMP53:%.*]] = select <8 x i1> [[TMP0]], <8 x i32> [[TMP28]], <8 x i32> [[VEC_PHI]]
 ; UF4-NEXT:    [[TMP54:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> [[TMP29]], <8 x i32> [[VEC_PHI1]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
index 6f5b9d4403af12..123ee72235a7b8 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
@@ -413,19 +413,14 @@ define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %
 ; DISABLED_MASKED_STRIDED-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_PH:.*]]
 ; DISABLED_MASKED_STRIDED:       [[VECTOR_PH]]:
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
 ; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; DISABLED_MASKED_STRIDED:       [[VECTOR_BODY]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
@@ -511,9 +506,11 @@ define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], %[[PRED_LOAD_IF15]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP52:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
 ; DISABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP2]])
-; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP54:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = xor i1 [[TMP54]], true
+; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP53]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; DISABLED_MASKED_STRIDED:       [[MIDDLE_BLOCK]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
@@ -531,19 +528,14 @@ define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %
 ; ENABLED_MASKED_STRIDED-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_PH:.*]]
 ; ENABLED_MASKED_STRIDED:       [[VECTOR_PH]]:
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
 ; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
 ; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ENABLED_MASKED_STRIDED:       [[VECTOR_BODY]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = shl i32 [[INDEX]], 1
@@ -554,9 +546,11 @@ define void @masked_strided1_optsize_unknown_tc(ptr noalias nocapture readonly %
 ; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP6]], <8 x i1> [[TMP2]])
-; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = xor i1 [[TMP8]], true
+; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; ENABLED_MASKED_STRIDED:       [[MIDDLE_BLOCK]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
@@ -633,19 +627,14 @@ define void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture readonly %
 ; DISABLED_MASKED_STRIDED-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_PH:.*]]
 ; DISABLED_MASKED_STRIDED:       [[VECTOR_PH]]:
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
 ; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; DISABLED_MASKED_STRIDED:       [[VECTOR_BODY]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = mul nsw <8 x i32> [[VEC_IND]], splat (i32 3)
@@ -731,9 +720,11 @@ define void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture readonly %
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], %[[PRED_LOAD_IF15]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP52:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
 ; DISABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP2]])
-; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP54:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = xor i1 [[TMP54]], true
+; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP53]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; DISABLED_MASKED_STRIDED:       [[MIDDLE_BLOCK]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
@@ -751,19 +742,14 @@ define void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture readonly %
 ; ENABLED_MASKED_STRIDED-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_PH:.*]]
 ; ENABLED_MASKED_STRIDED:       [[VECTOR_PH]]:
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
 ; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[CONV]], i64 0
 ; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ENABLED_MASKED_STRIDED:       [[VECTOR_BODY]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16:.*]] ]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE16]] ]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = mul nsw <8 x i32> [[VEC_IND]], splat (i32 3)
@@ -849,9 +835,11 @@ define void @masked_strided3_optsize_unknown_tc(ptr noalias nocapture readonly %
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP51:%.*]] = phi <8 x i8> [ [[TMP45]], %[[PRED_LOAD_CONTINUE14]] ], [ [[TMP50]], %[[PRED_LOAD_IF15]] ]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP52:%.*]] = getelementptr i8, ptr [[Q]], i32 [[INDEX]]
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP51]], ptr align 1 [[TMP52]], <8 x i1> [[TMP2]])
-; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP54:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP53:%.*]] = xor i1 [[TMP54]], true
+; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP53]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; ENABLED_MASKED_STRIDED:       [[MIDDLE_BLOCK]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
@@ -1038,17 +1026,12 @@ define void @unconditional_strided1_optsize_unknown_tc(ptr noalias nocapture rea
 ; DISABLED_MASKED_STRIDED:       [[FOR_BODY_PREHEADER]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_PH:.*]]
 ; DISABLED_MASKED_STRIDED:       [[VECTOR_PH]]:
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; DISABLED_MASKED_STRIDED:       [[VECTOR_BODY]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE14:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_LOAD_CONTINUE14]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE14]] ]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
@@ -1132,9 +1115,11 @@ define void @unconditional_strided1_optsize_unknown_tc(ptr noalias nocapture rea
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP49:%.*]] = phi <8 x i8> [ [[TMP43]], %[[PRED_LOAD_CONTINUE12]] ], [ [[TMP48]], %[[PRED_LOAD_IF13]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[INDEX]]
 ; DISABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP49]], ptr align 1 [[TMP50]], <8 x i1> [[TMP0]])
-; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP51:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP52:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP51:%.*]] = xor i1 [[TMP52]], true
+; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP51]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; DISABLED_MASKED_STRIDED:       [[MIDDLE_BLOCK]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
@@ -1151,17 +1136,11 @@ define void @unconditional_strided1_optsize_unknown_tc(ptr noalias nocapture rea
 ; ENABLED_MASKED_STRIDED:       [[FOR_BODY_PREHEADER]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_PH:.*]]
 ; ENABLED_MASKED_STRIDED:       [[VECTOR_PH]]:
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ENABLED_MASKED_STRIDED:       [[VECTOR_BODY]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IV:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = shl nuw nsw i32 [[INDEX]], 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP1]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
@@ -1170,9 +1149,10 @@ define void @unconditional_strided1_optsize_unknown_tc(ptr noalias nocapture rea
 ; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[Q]], i32 [[INDEX]]
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[STRIDED_VEC]], ptr align 1 [[TMP4]], <8 x i1> [[TMP0]])
-; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add nuw <8 x i32> [[VEC_IV]], splat (i32 8)
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP6]], true
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; ENABLED_MASKED_STRIDED:       [[MIDDLE_BLOCK]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
@@ -2309,19 +2289,14 @@ define void @masked_strided2_unknown_tc(ptr noalias nocapture readonly %p, ptr n
 ; DISABLED_MASKED_STRIDED:       [[FOR_BODY_PREHEADER]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_PH:.*]]
 ; DISABLED_MASKED_STRIDED:       [[VECTOR_PH]]:
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
 ; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[GUARD]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; DISABLED_MASKED_STRIDED:       [[VECTOR_BODY]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE62:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE62]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE62]] ]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = icmp sgt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
@@ -2609,10 +2584,12 @@ define void @masked_strided2_unknown_tc(ptr noalias nocapture readonly %p, ptr n
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP158]], ptr [[TMP157]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[PRED_STORE_CONTINUE62]]
 ; DISABLED_MASKED_STRIDED:       [[PRED_STORE_CONTINUE62]]:
-; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP159:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP159]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP159:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP160:%.*]] = xor i1 [[TMP159]], true
+; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP160]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; DISABLED_MASKED_STRIDED:       [[MIDDLE_BLOCK]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
 ; DISABLED_MASKED_STRIDED:       [[FOR_END_LOOPEXIT]]:
@@ -2628,19 +2605,14 @@ define void @masked_strided2_unknown_tc(ptr noalias nocapture readonly %p, ptr n
 ; ENABLED_MASKED_STRIDED:       [[FOR_BODY_PREHEADER]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_PH:.*]]
 ; ENABLED_MASKED_STRIDED:       [[VECTOR_PH]]:
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
 ; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[GUARD]], i64 0
 ; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ENABLED_MASKED_STRIDED:       [[VECTOR_BODY]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = icmp sgt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP0]], <8 x i1> [[TMP1]], <8 x i1> zeroinitializer
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = shl i32 [[INDEX]], 1
@@ -2656,10 +2628,12 @@ define void @masked_strided2_unknown_tc(ptr noalias nocapture readonly %p, ptr n
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i8> [[TMP5]], <8 x i8> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i8> [[TMP9]], <16 x i8> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP6]], <16 x i1> [[INTERLEAVED_MASK]])
-; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = xor i1 [[TMP11]], true
+; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; ENABLED_MASKED_STRIDED:       [[MIDDLE_BLOCK]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
 ; ENABLED_MASKED_STRIDED:       [[FOR_END_LOOPEXIT]]:
@@ -2742,17 +2716,12 @@ define void @unconditional_masked_strided2_unknown_tc(ptr noalias nocapture read
 ; DISABLED_MASKED_STRIDED:       [[FOR_BODY_PREHEADER]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_PH:.*]]
 ; DISABLED_MASKED_STRIDED:       [[VECTOR_PH]]:
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; DISABLED_MASKED_STRIDED:       [[VECTOR_BODY]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE60:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE60]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE60]] ]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 1)
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
@@ -3038,10 +3007,12 @@ define void @unconditional_masked_strided2_unknown_tc(ptr noalias nocapture read
 ; DISABLED_MASKED_STRIDED-NEXT:    store i8 [[TMP156]], ptr [[TMP155]], align 1
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[PRED_STORE_CONTINUE60]]
 ; DISABLED_MASKED_STRIDED:       [[PRED_STORE_CONTINUE60]]:
-; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP157:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP157]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; DISABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP157:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP158:%.*]] = xor i1 [[TMP157]], true
+; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8)
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP158]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; DISABLED_MASKED_STRIDED:       [[MIDDLE_BLOCK]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
 ; DISABLED_MASKED_STRIDED:       [[FOR_END_LOOPEXIT]]:
@@ -3057,17 +3028,11 @@ define void @unconditional_masked_strided2_unknown_tc(ptr noalias nocapture read
 ; ENABLED_MASKED_STRIDED:       [[FOR_BODY_PREHEADER]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_PH:.*]]
 ; ENABLED_MASKED_STRIDED:       [[VECTOR_PH]]:
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 7
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[N]], 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 0, i32 [[N]])
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ENABLED_MASKED_STRIDED:       [[VECTOR_BODY]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = shl nuw nsw i32 [[INDEX]], 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 [[TMP1]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
@@ -3081,10 +3046,11 @@ define void @unconditional_masked_strided2_unknown_tc(ptr noalias nocapture read
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> [[TMP5]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP4]], <16 x i1> [[INTERLEAVED_MASK]])
-; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add nuw <8 x i32> [[VEC_IND]], splat (i32 8)
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX_NEXT]], i32 [[N]])
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; ENABLED_MASKED_STRIDED:       [[MIDDLE_BLOCK]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
 ; ENABLED_MASKED_STRIDED:       [[FOR_END_LOOPEXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
index ce1b579dba671a..986d4692f93821 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
@@ -137,19 +137,15 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
 ; DISABLED_MASKED_STRIDED-NEXT:    [[CMP15:%.*]] = icmp sgt i32 [[NUMPOINTS]], 0
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[CMP15]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
 ; DISABLED_MASKED_STRIDED:       [[FOR_BODY_PREHEADER]]:
-; DISABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_PH:.*]]
-; DISABLED_MASKED_STRIDED:       [[VECTOR_PH]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[NUMPOINTS]] to i64
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add nuw nsw i64 [[WIDE_TRIP_COUNT]], 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 4294967292
-; DISABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
-; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; DISABLED_MASKED_STRIDED:       [[VECTOR_BODY]]:
-; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE15:.*]] ]
-; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE15]] ]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[WIDE_TRIP_COUNT]])
+; DISABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_BODY1:.*]]
+; DISABLED_MASKED_STRIDED:       [[VECTOR_BODY1]]:
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE15:.*]] ]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_BODY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE15]] ]
+; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE15]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 [[INDEX]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP1]], <4 x i1> [[TMP0]], <4 x i16> poison)
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = shl nsw <4 x i64> [[VEC_IND]], splat (i64 2)
@@ -157,7 +153,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; DISABLED_MASKED_STRIDED:       [[PRED_STORE_IF]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP4]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP4]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP6]], ptr [[TMP5]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[PRED_STORE_CONTINUE]]
@@ -166,7 +162,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
 ; DISABLED_MASKED_STRIDED:       [[PRED_STORE_IF1]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP8]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP8]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP10]], ptr [[TMP9]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
@@ -175,7 +171,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
 ; DISABLED_MASKED_STRIDED:       [[PRED_STORE_IF3]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP12]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP12]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP14]], ptr [[TMP13]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[PRED_STORE_CONTINUE4]]
@@ -184,7 +180,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP15]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
 ; DISABLED_MASKED_STRIDED:       [[PRED_STORE_IF5]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP16]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP16]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP18]], ptr [[TMP17]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
@@ -195,7 +191,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP3]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
 ; DISABLED_MASKED_STRIDED:       [[PRED_STORE_IF8]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP21:%.*]] = extractelement <4 x i64> [[TMP20]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP21]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP21]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 0
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP23]], ptr [[TMP22]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[PRED_STORE_CONTINUE9]]
@@ -203,7 +199,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP7]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
 ; DISABLED_MASKED_STRIDED:       [[PRED_STORE_IF10]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = extractelement <4 x i64> [[TMP20]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP25:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP24]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP25:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP24]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP26:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP26]], ptr [[TMP25]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[PRED_STORE_CONTINUE11]]
@@ -211,7 +207,7 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP11]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]]
 ; DISABLED_MASKED_STRIDED:       [[PRED_STORE_IF12]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP27:%.*]] = extractelement <4 x i64> [[TMP20]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP27]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP27]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP29:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP29]], ptr [[TMP28]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[PRED_STORE_CONTINUE13]]
@@ -219,15 +215,16 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
 ; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP15]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15]]
 ; DISABLED_MASKED_STRIDED:       [[PRED_STORE_IF14]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = extractelement <4 x i64> [[TMP20]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP31:%.*]] = getelementptr inbounds [2 x i8], ptr [[POINTS]], i64 [[TMP30]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP31:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[POINTS]], i64 [[TMP30]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP32:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP32]], ptr [[TMP31]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[PRED_STORE_CONTINUE15]]
 ; DISABLED_MASKED_STRIDED:       [[PRED_STORE_CONTINUE15]]:
-; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; DISABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[WIDE_TRIP_COUNT]])
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP33:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP33]], label %[[VECTOR_BODY1]], label %[[MIDDLE_BLOCK:.*]], !llvm.loop [[LOOP3:![0-9]+]]
 ; DISABLED_MASKED_STRIDED:       [[MIDDLE_BLOCK]]:
 ; DISABLED_MASKED_STRIDED-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
 ; DISABLED_MASKED_STRIDED:       [[FOR_END_LOOPEXIT]]:
@@ -241,19 +238,14 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
 ; ENABLED_MASKED_STRIDED-NEXT:    [[CMP15:%.*]] = icmp sgt i32 [[NUMPOINTS]], 0
 ; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[CMP15]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
 ; ENABLED_MASKED_STRIDED:       [[FOR_BODY_PREHEADER]]:
+; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[NUMPOINTS]] to i64
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_PH:.*]]
 ; ENABLED_MASKED_STRIDED:       [[VECTOR_PH]]:
-; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[NUMPOINTS]] to i64
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add nuw nsw i64 [[WIDE_TRIP_COUNT]], 3
-; ENABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 4294967292
-; ENABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
-; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[WIDE_TRIP_COUNT]])
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ENABLED_MASKED_STRIDED:       [[VECTOR_BODY]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 [[INDEX]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP1]], <4 x i1> [[TMP0]], <4 x i16> poison)
 ; ENABLED_MASKED_STRIDED-NEXT:    [[DOTIDX:%.*]] = shl nsw i64 [[INDEX]], 3
@@ -264,10 +256,10 @@ define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias no
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <4 x i1> [[TMP0]], <4 x i1> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr align 2 [[TMP2]], <16 x i1> [[TMP4]])
-; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; ENABLED_MASKED_STRIDED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[WIDE_TRIP_COUNT]])
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP5]], label %[[VECTOR_BODY]], label %[[MIDDLE_BLOCK:.*]], !llvm.loop [[LOOP3:![0-9]+]]
 ; ENABLED_MASKED_STRIDED:       [[MIDDLE_BLOCK]]:
 ; ENABLED_MASKED_STRIDED-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
 ; ENABLED_MASKED_STRIDED:       [[FOR_END_LOOPEXIT]]:

>From 2fcd8842688c3fd9fdde894460d0151f66c4780a Mon Sep 17 00:00:00 2001
From: william <we3223 at gmail.com>
Date: Mon, 5 Oct 2026 23:05:38 +0800
Subject: [PATCH 4/5] [X86][TTI] Enable DataAndControlFlow tail folding style
 on AVX-512-1

---
 llvm/lib/Target/X86/X86TargetTransformInfo.h | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index f87cd2fee49271..47476fa2770c05 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -57,9 +57,8 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
 
   /// \name Vector TTI Implementations
   /// @{
-  
-  TailFoldingStyle
-  getPreferredTailFoldingStyle() const {
+
+  TailFoldingStyle getPreferredTailFoldingStyle() const override {
     return ST->hasAVX512() ? TailFoldingStyle::DataAndControlFlow
                            : TailFoldingStyle::DataWithoutLaneMask;
   }

>From 0632ecfb74092c5633200487656136ec6d2aa4cb Mon Sep 17 00:00:00 2001
From: william <we3223 at gmail.com>
Date: Fri, 9 Oct 2026 15:57:40 +0800
Subject: [PATCH 5/5] [Trigger CI] Re-run pre-merge tests

---
 .../X86/vectorize-interleaved-accesses-gap.ll | 70 +++++++++++--------
 1 file changed, 40 insertions(+), 30 deletions(-)

diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
index c9a74ae42a8382..296a1ad7435dc4 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
@@ -1,23 +1,24 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -passes=loop-vectorize -enable-masked-interleaved-mem-accesses -enable-interleaved-mem-accesses  -tail-folding-policy=must-fold-tail -force-vector-width=8 -S %s | FileCheck %s
 
 target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx13.0.0"
 
 define void @test_pr59090(ptr %l_out, ptr noalias %b) #0 {
-; CHECK-LABEL: @test_pr59090(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
+; CHECK-LABEL: define void @test_pr59090(
+; CHECK-SAME: ptr [[L_OUT:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 10001)
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[INDEX]], 6
-; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr [[B:%.*]], align 1, !llvm.access.group [[ACC_GRP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0:![0-9]+]]
 ; CHECK-NEXT:    store i8 [[TMP3]], ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0]]
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[L_OUT:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[L_OUT]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> poison, <48 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
 ; CHECK-NEXT:    [[TMP15:%.*]] = and <48 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false>
 ; CHECK-NEXT:    call void @llvm.masked.store.v48i8.p0(<48 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison>, ptr align 1 [[TMP13]], <48 x i1> [[TMP15]]), !llvm.access.group [[ACC_GRP0]]
@@ -25,10 +26,10 @@ define void @test_pr59090(ptr %l_out, ptr noalias %b) #0 {
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX_NEXT]], i64 10001)
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
-; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       exit:
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -53,24 +54,25 @@ exit:
 }
 
 define void @test_pr59090_interleave(ptr %l_out, ptr noalias %b) #0 {
-; CHECK-LABEL: @test_pr59090_interleave(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
+; CHECK-LABEL: define void @test_pr59090_interleave(
+; CHECK-SAME: ptr [[L_OUT:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 10001)
 ; CHECK-NEXT:    [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
 ; CHECK-NEXT:    [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 8)
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], [[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <8 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART4:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[INDEX]], 6
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP0]], 6
-; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[B:%.*]], align 1, !llvm.access.group [[ACC_GRP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0]]
 ; CHECK-NEXT:    store i8 [[TMP5]], ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[L_OUT:%.*]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[L_OUT]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[L_OUT]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> poison, <48 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
 ; CHECK-NEXT:    [[TMP8:%.*]] = and <48 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false>
@@ -84,10 +86,10 @@ define void @test_pr59090_interleave(ptr %l_out, ptr noalias %b) #0 {
 ; CHECK-NEXT:    [[EXTRACT_NEXT_ALM_PART4]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 8)
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP10]], true
-; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -118,3 +120,11 @@ attributes #0 = { "target-cpu"="skx" }
 !3 = !{!"llvm.loop.parallel_accesses", !0}
 !4 = distinct !{!4, !3, !5}
 !5 = !{!"llvm.loop.interleave.count", i32 2}
+;.
+; CHECK: [[ACC_GRP0]] = distinct !{}
+; CHECK: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]], [[META4:![0-9]+]]}
+; CHECK: [[META2]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP0]]}
+; CHECK: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META3]], [[META4]]}
+;.



More information about the llvm-commits mailing list