[llvm-branch-commits] [llvm] [LV] Widen `bitinsert` and `bitextract` (PR #242517)

Pedro Lobo via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Tue Nov 17 01:58:18 PST 2026


https://github.com/pedroclobo updated https://github.com/llvm/llvm-project/pull/242517

>From 107d228caef330b5f82e19342fdc3daf5df247e6 Mon Sep 17 00:00:00 2001
From: Pedro Lobo <pedro.lobo at tecnico.ulisboa.pt>
Date: Sun, 8 Nov 2026 10:07:57 +0000
Subject: [PATCH 1/2] [VPlan] Share the result-type helper of
 `VPReplicateRecipe` with `VPWidenRecipe` (NFC)

---
 llvm/lib/Transforms/Vectorize/VPlan.h         | 40 +++++++++++--------
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |  4 +-
 2 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 5d89141cde6f2e..6194bcbb6a5ade 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -598,6 +598,11 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
 LLVM_ABI Type *computeScalarTypeForInstruction(unsigned Opcode,
                                                ArrayRef<VPValue *> Operands);
 
+/// Compute the scalar result type for a recipe wrapping \p I with \p Operands
+/// (excluding any predicate mask).
+LLVM_ABI Type *computeScalarTypeForInstruction(const Instruction *I,
+                                               ArrayRef<VPValue *> Operands);
+
 /// VPSingleDefRecipe is a base class for recipes that model a sequence of one
 /// or more output IR that define a single result VPValue. Note that
 /// VPSingleDefRecipe must inherit from VPRecipeBase before VPSingleDefValue.
@@ -1832,26 +1837,33 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
                                         public VPIRMetadata {
   unsigned Opcode;
 
+  VPWidenRecipe(unsigned Opcode, Type *ResultTy, ArrayRef<VPValue *> Operands,
+                const VPIRFlags &Flags, const VPIRMetadata &Metadata,
+                DebugLoc DL)
+      : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, Operands, ResultTy, Flags,
+                            DL),
+        VPIRMetadata(Metadata), Opcode(Opcode) {
+    assert(flagsValidForOpcode(Opcode) &&
+           "Set flags not supported for the provided opcode");
+    assert(hasRequiredFlagsForOpcode(Opcode, getScalarType()) &&
+           "Opcode requires specific flags to be set");
+  }
+
 public:
   VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands,
                 const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
                 DebugLoc DL = {})
-      : VPWidenRecipe(I.getOpcode(), Operands, Flags, Metadata, DL) {
+      : VPWidenRecipe(I.getOpcode(),
+                      computeScalarTypeForInstruction(&I, Operands), Operands,
+                      Flags, Metadata, DL) {
     setUnderlyingValue(&I);
   }
 
   VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
                 const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
                 DebugLoc DL = {})
-      : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, Operands,
-                            computeScalarTypeForInstruction(Opcode, Operands),
-                            Flags, DL),
-        VPIRMetadata(Metadata), Opcode(Opcode) {
-    assert(flagsValidForOpcode(Opcode) &&
-           "Set flags not supported for the provided opcode");
-    assert(hasRequiredFlagsForOpcode(Opcode, getScalarType()) &&
-           "Opcode requires specific flags to be set");
-  }
+      : VPWidenRecipe(Opcode, computeScalarTypeForInstruction(Opcode, Operands),
+                      Operands, Flags, Metadata, DL) {}
 
   ~VPWidenRecipe() override = default;
 
@@ -3505,7 +3517,8 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
                     const VPIRFlags &Flags = {}, VPIRMetadata Metadata = {},
                     DebugLoc DL = DebugLoc::getUnknown())
       : VPRecipeWithIRFlags(VPRecipeBase::VPReplicateSC, Operands,
-                            computeScalarType(I, Operands), Flags, DL),
+                            computeScalarTypeForInstruction(I, Operands), Flags,
+                            DL),
         VPIRMetadata(Metadata), IsSingleScalar(IsSingleScalar),
         IsPredicated(Mask) {
     assert((!IsSingleScalar || !I->isCast()) &&
@@ -3517,11 +3530,6 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
 
   ~VPReplicateRecipe() override = default;
 
-  /// Compute the scalar result type for a VPReplicateRecipe wrapping \p I with
-  /// \p Operands (excluding any predicate mask).
-  static Type *computeScalarType(const Instruction *I,
-                                 ArrayRef<VPValue *> Operands);
-
   VPReplicateRecipe *clone() override { return cloneWithOperands(operands()); }
 
   VPReplicateRecipe *cloneWithOperands(ArrayRef<VPValue *> NewOperands) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 19af59ff4a2949..9f5fca8ea852d9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -633,8 +633,8 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
   return Op0Ty;
 }
 
-Type *VPReplicateRecipe::computeScalarType(const Instruction *I,
-                                           ArrayRef<VPValue *> Operands) {
+Type *llvm::computeScalarTypeForInstruction(const Instruction *I,
+                                            ArrayRef<VPValue *> Operands) {
   unsigned Opcode = I->getOpcode();
   if (Instruction::isCast(Opcode) ||
       is_contained(ArrayRef<unsigned>({Instruction::Load, Instruction::Alloca}),

>From 2c39d2d5c67c8f1944da9fb677aaad7e70f0c97d Mon Sep 17 00:00:00 2001
From: Pedro Lobo <pedro.lobo at tecnico.ulisboa.pt>
Date: Fri, 6 Nov 2026 13:46:52 +0000
Subject: [PATCH 2/2] [LV] Widen `bitinsert` and `bitextract`

---
 .../Transforms/Vectorize/LoopVectorize.cpp    |   4 +
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |  49 +-
 llvm/lib/Transforms/Vectorize/VPlanUtils.cpp  |   2 +
 .../X86/bitinsert-bitextract-cost.ll          | 337 ++++++++
 .../LoopVectorize/bitinsert-bitextract.ll     | 782 ++++++++++++++++++
 .../outer-loop-bitinsert-bitextract.ll        | 174 ++++
 6 files changed, 1347 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/bitinsert-bitextract-cost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/bitinsert-bitextract.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/outer-loop-bitinsert-bitextract.ll

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a2a2797b2038fc..cf2e436b72e718 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4885,6 +4885,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
   case Instruction::Call:
     return getVectorCallCost(cast<CallInst>(I), VF);
   case Instruction::ExtractValue:
+  case Instruction::BitInsert:
+  case Instruction::BitExtract:
     return TTI.getInstructionCost(I, Config.CostKind);
   case Instruction::Alloca:
     // We cannot easily widen alloca to a scalable alloca, as
@@ -5902,6 +5904,8 @@ VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
   case Instruction::Xor:
   case Instruction::Freeze:
   case Instruction::ExtractValue:
+  case Instruction::BitInsert:
+  case Instruction::BitExtract:
     return new VPWidenRecipe(*I, VPI->operandsWithoutMask(), *VPI, *VPI,
                              VPI->getDebugLoc());
   };
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 9f5fca8ea852d9..41639c089a22ef 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -605,6 +605,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
   case VPInstruction::IncomingAliasMask:
   case Instruction::Load:
   case Instruction::Alloca:
+  case Instruction::BitExtract:
     llvm_unreachable("type must be passed explicitly");
   case Instruction::Call:
     return getCalledFunction(Operands)->getReturnType();
@@ -637,7 +638,8 @@ Type *llvm::computeScalarTypeForInstruction(const Instruction *I,
                                             ArrayRef<VPValue *> Operands) {
   unsigned Opcode = I->getOpcode();
   if (Instruction::isCast(Opcode) ||
-      is_contained(ArrayRef<unsigned>({Instruction::Load, Instruction::Alloca}),
+      is_contained(ArrayRef<unsigned>({Instruction::Load, Instruction::Alloca,
+                                       Instruction::BitExtract}),
                    Opcode))
     return I->getType();
   return computeScalarTypeForInstruction(Opcode, Operands);
@@ -695,6 +697,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
   case Instruction::FCmp:
   case Instruction::ExtractElement:
   case Instruction::Store:
+  case Instruction::BitExtract:
   case VPInstruction::ActiveLaneMask:
   case VPInstruction::BranchOnCount:
   case VPInstruction::BranchOnTwoConds:
@@ -709,6 +712,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
     return 2;
   case Instruction::InsertElement:
   case Instruction::Select:
+  case Instruction::BitInsert:
   case VPInstruction::WideActiveLaneMask:
   case VPInstruction::ReductionStartVector:
   case VPInstruction::WideVectorLoad:
@@ -1310,6 +1314,29 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
   case Instruction::ExtractValue:
     return Ctx.TTI.getInsertExtractValueCost(Instruction::ExtractValue,
                                              Ctx.CostKind);
+  case Instruction::BitInsert:
+  case Instruction::BitExtract: {
+    bool IsInsert = Opcode == Instruction::BitInsert;
+    VPValue *Offset = getOperand(IsInsert ? 2 : 1);
+    TargetTransformInfo::OperandValueInfo OffsetInfo =
+        Ctx.getOperandInfo(Offset);
+
+    if (OffsetInfo.Kind == TargetTransformInfo::OK_AnyValue &&
+        Offset->isDefinedOutsideLoopRegions())
+      OffsetInfo.Kind = TargetTransformInfo::OK_UniformValue;
+
+    Type *ScalarOpTy = getOperand(IsInsert ? 1 : 0)->getScalarType();
+    Type *OpTy = VF.isVector() ? toVectorTy(ScalarOpTy, VF) : ScalarOpTy;
+    Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
+    SmallVector<const Value *, 3> Operands;
+    if (CtxI)
+      Operands.append(CtxI->value_op_begin(), CtxI->value_op_end());
+    if (IsInsert)
+      return Ctx.TTI.getBitInsertCost(ResultTy, OpTy, Ctx.CostKind, OffsetInfo,
+                                      Operands, CtxI);
+    return Ctx.TTI.getBitExtractCost(ResultTy, OpTy, Ctx.CostKind, OffsetInfo,
+                                     Operands, CtxI);
+  }
   case Instruction::ICmp:
   case Instruction::FCmp: {
     Type *ScalarOpTy = getOperand(0)->getScalarType();
@@ -3099,6 +3126,22 @@ void VPWidenRecipe::execute(VPTransformState &State) {
     State.set(this, Freeze);
     break;
   }
+  case Instruction::BitInsert: {
+    Value *Base = State.get(getOperand(0));
+    Value *Val = State.get(getOperand(1));
+    Value *Offset = State.get(getOperand(2));
+    Value *Insert = Builder.CreateBitInsert(Base, Val, Offset);
+    State.set(this, Insert);
+    break;
+  }
+  case Instruction::BitExtract: {
+    Value *Src = State.get(getOperand(0));
+    Value *Offset = State.get(getOperand(1));
+    Type *DestTy = VectorType::get(getScalarType(), State.VF);
+    Value *Extract = Builder.CreateBitExtract(DestTy, Src, Offset);
+    State.set(this, Extract);
+    break;
+  }
   case Instruction::ICmp:
   case Instruction::FCmp: {
     // Widen compares. Generate vector compares.
@@ -3179,6 +3222,8 @@ InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
   case Instruction::ICmp:
   case Instruction::FCmp:
   case Instruction::Select:
+  case Instruction::BitInsert:
+  case Instruction::BitExtract:
     return getCostForRecipeWithOpcode(getOpcode(), VF, Ctx);
   default:
     llvm_unreachable("Unsupported opcode for instruction");
@@ -4325,6 +4370,8 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
   case Instruction::ICmp:
   case Instruction::FCmp:
   case Instruction::Freeze:
+  case Instruction::BitInsert:
+  case Instruction::BitExtract:
     return getCostForRecipeWithOpcode(getOpcode(), ElementCount::getFixed(1),
                                       Ctx) *
            (isSingleScalar() ? 1 : VF.getFixedValue());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index f365e755a9d433..084247279dfc02 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -471,6 +471,8 @@ static bool preservesUniformity(unsigned Opcode) {
   case Instruction::ICmp:
   case Instruction::FCmp:
   case Instruction::Select:
+  case Instruction::BitInsert:
+  case Instruction::BitExtract:
   case VPInstruction::Not:
   case VPInstruction::Broadcast:
   case VPInstruction::MaskedCond:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/bitinsert-bitextract-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/bitinsert-bitextract-cost.ll
new file mode 100644
index 00000000000000..cacddac771effd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/bitinsert-bitextract-cost.ll
@@ -0,0 +1,337 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%x = bit" --filter "Cost of [0-9]+ for VF [0-9]+: (WIDEN|CLONE) ir<%x> = bit" --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=x86_64-- -mcpu=x86-64-v3 -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+define void @bitinsert_varying_base(ptr noalias %dst, ptr noalias %src, i8 %v, i64 %n) {
+; CHECK-LABEL: 'bitinsert_varying_base'
+; CHECK:  LV: Found an estimated cost of 4 for VF 1 For instruction: %x = bitinsert b16 %b, i8 %v, i32 8
+; CHECK:  Cost of 4 for VF 2: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+; CHECK:  Cost of 4 for VF 4: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+; CHECK:  Cost of 4 for VF 8: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+; CHECK:  Cost of 6 for VF 16: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+  %b = load b16, ptr %gep.src, align 2
+  %x = bitinsert b16 %b, i8 %v, i32 8
+  %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %iv
+  store b16 %x, ptr %gep.dst, align 2
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitinsert_uniform_base(ptr noalias %dst, ptr noalias %src, i8 %v, i64 %n) {
+; CHECK-LABEL: 'bitinsert_uniform_base'
+; CHECK:  LV: Found an estimated cost of 4 for VF 1 For instruction: %x = bitinsert b16 %b, i8 %v, i32 8
+; CHECK:  Cost of 4 for VF 2: CLONE ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+; CHECK:  Cost of 4 for VF 4: CLONE ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+; CHECK:  Cost of 4 for VF 8: CLONE ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+; CHECK:  Cost of 4 for VF 16: CLONE ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %b = load b16, ptr %src, align 2
+  %x = bitinsert b16 %b, i8 %v, i32 8
+  %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %iv
+  store b16 %x, ptr %gep.dst, align 2
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitinsert_invariant_offset(ptr noalias %dst, ptr noalias %src, i8 %v, i32 %off, i64 %n) {
+; CHECK-LABEL: 'bitinsert_invariant_offset'
+; CHECK:  LV: Found an estimated cost of 6 for VF 1 For instruction: %x = bitinsert b16 %b, i8 %v, i32 %off
+; CHECK:  Cost of 6 for VF 2: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+; CHECK:  Cost of 6 for VF 4: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+; CHECK:  Cost of 6 for VF 8: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+; CHECK:  Cost of 9 for VF 16: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+  %b = load b16, ptr %gep.src, align 2
+  %x = bitinsert b16 %b, i8 %v, i32 %off
+  %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %iv
+  store b16 %x, ptr %gep.dst, align 2
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitinsert_varying_offset(ptr noalias %dst, ptr noalias %src, ptr noalias %vals, ptr noalias %offs, i64 %n) {
+; CHECK-LABEL: 'bitinsert_varying_offset'
+; CHECK:  LV: Found an estimated cost of 6 for VF 1 For instruction: %x = bitinsert b32 %b, i8 %v, i32 %off
+; CHECK:  Cost of 8 for VF 2: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+; CHECK:  Cost of 8 for VF 4: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+; CHECK:  Cost of 13 for VF 8: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b32, ptr %src, i64 %iv
+  %b = load b32, ptr %gep.src, align 4
+  %gep.val = getelementptr inbounds i8, ptr %vals, i64 %iv
+  %v = load i8, ptr %gep.val, align 1
+  %gep.off = getelementptr inbounds i32, ptr %offs, i64 %iv
+  %off = load i32, ptr %gep.off, align 4
+  %x = bitinsert b32 %b, i8 %v, i32 %off
+  %gep.dst = getelementptr inbounds b32, ptr %dst, i64 %iv
+  store b32 %x, ptr %gep.dst, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitinsert_float_value(ptr noalias %dst, ptr noalias %src, float %v, i64 %n) {
+; CHECK-LABEL: 'bitinsert_float_value'
+; CHECK:  LV: Found an estimated cost of 3 for VF 1 For instruction: %x = bitinsert b64 %b, float %v, i32 32
+; CHECK:  Cost of 4 for VF 2: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<32>
+; CHECK:  Cost of 6 for VF 4: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<32>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+  %b = load b64, ptr %gep.src, align 8
+  %x = bitinsert b64 %b, float %v, i32 32
+  %gep.dst = getelementptr inbounds b64, ptr %dst, i64 %iv
+  store b64 %x, ptr %gep.dst, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitinsert_ptr_value(ptr noalias %dst, ptr noalias %src, ptr %v, i64 %n) {
+; CHECK-LABEL: 'bitinsert_ptr_value'
+; CHECK:  LV: Found an estimated cost of 3 for VF 1 For instruction: %x = bitinsert b64 %b, ptr %v, i32 0
+; CHECK:  Cost of 3 for VF 2: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<0>
+; CHECK:  Cost of 4 for VF 4: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<0>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+  %b = load b64, ptr %gep.src, align 8
+  %x = bitinsert b64 %b, ptr %v, i32 0
+  %gep.dst = getelementptr inbounds b64, ptr %dst, i64 %iv
+  store b64 %x, ptr %gep.dst, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitextract_varying_src(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: 'bitextract_varying_src'
+; CHECK:  LV: Found an estimated cost of 1 for VF 1 For instruction: %x = bitextract i8, b16 %b, i32 8
+; CHECK:  Cost of 2 for VF 2: WIDEN ir<%x> = bitextract ir<%b>, ir<8>
+; CHECK:  Cost of 2 for VF 4: WIDEN ir<%x> = bitextract ir<%b>, ir<8>
+; CHECK:  Cost of 2 for VF 8: WIDEN ir<%x> = bitextract ir<%b>, ir<8>
+; CHECK:  Cost of 4 for VF 16: WIDEN ir<%x> = bitextract ir<%b>, ir<8>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+  %b = load b16, ptr %gep.src, align 2
+  %x = bitextract i8, b16 %b, i32 8
+  %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+  store i8 %x, ptr %gep.dst, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitextract_uniform_src(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: 'bitextract_uniform_src'
+; CHECK:  LV: Found an estimated cost of 1 for VF 1 For instruction: %x = bitextract i8, b16 %b, i32 8
+; CHECK:  Cost of 1 for VF 2: CLONE ir<%x> = bitextract ir<%b>, ir<8>
+; CHECK:  Cost of 1 for VF 4: CLONE ir<%x> = bitextract ir<%b>, ir<8>
+; CHECK:  Cost of 1 for VF 8: CLONE ir<%x> = bitextract ir<%b>, ir<8>
+; CHECK:  Cost of 1 for VF 16: CLONE ir<%x> = bitextract ir<%b>, ir<8>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %b = load b16, ptr %src, align 2
+  %x = bitextract i8, b16 %b, i32 8
+  %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+  store i8 %x, ptr %gep.dst, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitextract_invariant_offset(ptr noalias %dst, ptr noalias %src, i32 %off, i64 %n) {
+; CHECK-LABEL: 'bitextract_invariant_offset'
+; CHECK:  LV: Found an estimated cost of 1 for VF 1 For instruction: %x = bitextract i8, b16 %b, i32 %off
+; CHECK:  Cost of 2 for VF 2: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+; CHECK:  Cost of 2 for VF 4: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+; CHECK:  Cost of 2 for VF 8: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+; CHECK:  Cost of 4 for VF 16: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+  %b = load b16, ptr %gep.src, align 2
+  %x = bitextract i8, b16 %b, i32 %off
+  %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+  store i8 %x, ptr %gep.dst, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitextract_varying_offset(ptr noalias %dst, ptr noalias %src, ptr noalias %offs, i64 %n) {
+; CHECK-LABEL: 'bitextract_varying_offset'
+; CHECK:  LV: Found an estimated cost of 1 for VF 1 For instruction: %x = bitextract i8, b32 %b, i32 %off
+; CHECK:  Cost of 3 for VF 2: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+; CHECK:  Cost of 3 for VF 4: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+; CHECK:  Cost of 8 for VF 8: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b32, ptr %src, i64 %iv
+  %b = load b32, ptr %gep.src, align 4
+  %gep.off = getelementptr inbounds i32, ptr %offs, i64 %iv
+  %off = load i32, ptr %gep.off, align 4
+  %x = bitextract i8, b32 %b, i32 %off
+  %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+  store i8 %x, ptr %gep.dst, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitextract_float_result(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: 'bitextract_float_result'
+; CHECK:  LV: Found an estimated cost of 1 for VF 1 For instruction: %x = bitextract float, b64 %b, i32 32
+; CHECK:  Cost of 2 for VF 2: WIDEN ir<%x> = bitextract ir<%b>, ir<32>
+; CHECK:  Cost of 3 for VF 4: WIDEN ir<%x> = bitextract ir<%b>, ir<32>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+  %b = load b64, ptr %gep.src, align 8
+  %x = bitextract float, b64 %b, i32 32
+  %gep.dst = getelementptr inbounds float, ptr %dst, i64 %iv
+  store float %x, ptr %gep.dst, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitextract_ptr_result(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: 'bitextract_ptr_result'
+; CHECK:  LV: Found an estimated cost of 1 for VF 1 For instruction: %x = bitextract ptr, b64 %b, i32 0
+; CHECK:  Cost of 1 for VF 2: WIDEN ir<%x> = bitextract ir<%b>, ir<0>
+; CHECK:  Cost of 2 for VF 4: WIDEN ir<%x> = bitextract ir<%b>, ir<0>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+  %b = load b64, ptr %gep.src, align 8
+  %x = bitextract ptr, b64 %b, i32 0
+  %gep.dst = getelementptr inbounds ptr, ptr %dst, i64 %iv
+  store ptr %x, ptr %gep.dst, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Test case for https://github.com/llvm/llvm-project/issues/241651.
+define void @bitextract_select_src(ptr noalias %dst, ptr noalias %conds, b128 %b1, b128 %b2, i64 %n) {
+; CHECK-LABEL: 'bitextract_select_src'
+; CHECK:  LV: Found an estimated cost of 2 for VF 1 For instruction: %x = bitextract i32, b128 %b, i32 16
+; CHECK:  Cost of 4 for VF 2: WIDEN ir<%x> = bitextract ir<%b>, ir<16>
+; CHECK:  Cost of 9 for VF 4: WIDEN ir<%x> = bitextract ir<%b>, ir<16>
+; CHECK:  Cost of 19 for VF 8: WIDEN ir<%x> = bitextract ir<%b>, ir<16>
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.cond = getelementptr inbounds i8, ptr %conds, i64 %iv
+  %c = load i8, ptr %gep.cond, align 1
+  %cmp = icmp eq i8 %c, 0
+  %b = select i1 %cmp, b128 %b1, b128 %b2
+  %x = bitextract i32, b128 %b, i32 16
+  %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 %x, ptr %gep.dst, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/bitinsert-bitextract.ll b/llvm/test/Transforms/LoopVectorize/bitinsert-bitextract.ll
new file mode 100644
index 00000000000000..ba468859fd8588
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/bitinsert-bitextract.ll
@@ -0,0 +1,782 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+define void @bitinsert_varying_base(ptr noalias %dst, ptr noalias %src, i8 %v, i64 %n) {
+; CHECK-LABEL: define void @bitinsert_varying_base(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i8 [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[V]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x b16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitinsert <4 x b16> [[WIDE_LOAD]], <4 x i8> [[BROADCAST_SPLAT]], <4 x i32> splat (i32 8)
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x b16> [[TMP2]], ptr [[TMP3]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[B:%.*]] = load b16, ptr [[GEP_SRC]], align 2
+; CHECK-NEXT:    [[X:%.*]] = bitinsert b16 [[B]], i8 [[V]], i32 8
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store b16 [[X]], ptr [[GEP_DST]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+  %b = load b16, ptr %gep.src, align 2
+  %x = bitinsert b16 %b, i8 %v, i32 8
+  %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %iv
+  store b16 %x, ptr %gep.dst, align 2
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitinsert_uniform_base(ptr noalias %dst, ptr noalias %src, i8 %v, i64 %n) {
+; CHECK-LABEL: define void @bitinsert_uniform_base(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i8 [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = load b16, ptr [[SRC]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitinsert b16 [[TMP1]], i8 [[V]], i32 8
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x b16> poison, b16 [[TMP2]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x b16> [[BROADCAST_SPLATINSERT]], <4 x b16> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x b16> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[B:%.*]] = load b16, ptr [[SRC]], align 2
+; CHECK-NEXT:    [[X:%.*]] = bitinsert b16 [[B]], i8 [[V]], i32 8
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store b16 [[X]], ptr [[GEP_DST]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %b = load b16, ptr %src, align 2
+  %x = bitinsert b16 %b, i8 %v, i32 8
+  %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %iv
+  store b16 %x, ptr %gep.dst, align 2
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitinsert_invariant_offset(ptr noalias %dst, ptr noalias %src, i8 %v, i32 %off, i64 %n) {
+; CHECK-LABEL: define void @bitinsert_invariant_offset(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i8 [[V:%.*]], i32 [[OFF:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[V]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[OFF]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x b16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitinsert <4 x b16> [[WIDE_LOAD]], <4 x i8> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x b16> [[TMP2]], ptr [[TMP3]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[B:%.*]] = load b16, ptr [[GEP_SRC]], align 2
+; CHECK-NEXT:    [[X:%.*]] = bitinsert b16 [[B]], i8 [[V]], i32 [[OFF]]
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store b16 [[X]], ptr [[GEP_DST]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+  %b = load b16, ptr %gep.src, align 2
+  %x = bitinsert b16 %b, i8 %v, i32 %off
+  %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %iv
+  store b16 %x, ptr %gep.dst, align 2
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitinsert_varying_offset(ptr noalias %dst, ptr noalias %src, ptr noalias %vals, ptr noalias %offs, i64 %n) {
+; CHECK-LABEL: define void @bitinsert_varying_offset(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[VALS:%.*]], ptr noalias [[OFFS:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds b32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x b32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[VALS]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OFFS]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = bitinsert <4 x b32> [[WIDE_LOAD]], <4 x i8> [[WIDE_LOAD1]], <4 x i32> [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds b32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x b32> [[TMP4]], ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds b32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[B:%.*]] = load b32, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[GEP_VAL:%.*]] = getelementptr inbounds i8, ptr [[VALS]], i64 [[IV]]
+; CHECK-NEXT:    [[V:%.*]] = load i8, ptr [[GEP_VAL]], align 1
+; CHECK-NEXT:    [[GEP_OFF:%.*]] = getelementptr inbounds i32, ptr [[OFFS]], i64 [[IV]]
+; CHECK-NEXT:    [[OFF:%.*]] = load i32, ptr [[GEP_OFF]], align 4
+; CHECK-NEXT:    [[X:%.*]] = bitinsert b32 [[B]], i8 [[V]], i32 [[OFF]]
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds b32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store b32 [[X]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b32, ptr %src, i64 %iv
+  %b = load b32, ptr %gep.src, align 4
+  %gep.val = getelementptr inbounds i8, ptr %vals, i64 %iv
+  %v = load i8, ptr %gep.val, align 1
+  %gep.off = getelementptr inbounds i32, ptr %offs, i64 %iv
+  %off = load i32, ptr %gep.off, align 4
+  %x = bitinsert b32 %b, i8 %v, i32 %off
+  %gep.dst = getelementptr inbounds b32, ptr %dst, i64 %iv
+  store b32 %x, ptr %gep.dst, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitinsert_float_value(ptr noalias %dst, ptr noalias %src, float %v, i64 %n) {
+; CHECK-LABEL: define void @bitinsert_float_value(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], float [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[V]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x b64>, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitinsert <4 x b64> [[WIDE_LOAD]], <4 x float> [[BROADCAST_SPLAT]], <4 x i32> splat (i32 32)
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds b64, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x b64> [[TMP2]], ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[B:%.*]] = load b64, ptr [[GEP_SRC]], align 8
+; CHECK-NEXT:    [[X:%.*]] = bitinsert b64 [[B]], float [[V]], i32 32
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds b64, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store b64 [[X]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+  %b = load b64, ptr %gep.src, align 8
+  %x = bitinsert b64 %b, float %v, i32 32
+  %gep.dst = getelementptr inbounds b64, ptr %dst, i64 %iv
+  store b64 %x, ptr %gep.dst, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitinsert_ptr_value(ptr noalias %dst, ptr noalias %src, ptr %v, i64 %n) {
+; CHECK-LABEL: define void @bitinsert_ptr_value(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x ptr> poison, ptr [[V]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x b64>, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitinsert <4 x b64> [[WIDE_LOAD]], <4 x ptr> [[BROADCAST_SPLAT]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds b64, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x b64> [[TMP2]], ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[B:%.*]] = load b64, ptr [[GEP_SRC]], align 8
+; CHECK-NEXT:    [[X:%.*]] = bitinsert b64 [[B]], ptr [[V]], i32 0
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds b64, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store b64 [[X]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+  %b = load b64, ptr %gep.src, align 8
+  %x = bitinsert b64 %b, ptr %v, i32 0
+  %gep.dst = getelementptr inbounds b64, ptr %dst, i64 %iv
+  store b64 %x, ptr %gep.dst, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitextract_varying_src(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: define void @bitextract_varying_src(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x b16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitextract <4 x i8>, <4 x b16> [[WIDE_LOAD]], <4 x i32> splat (i32 8)
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i8> [[TMP2]], ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[B:%.*]] = load b16, ptr [[GEP_SRC]], align 2
+; CHECK-NEXT:    [[X:%.*]] = bitextract i8, b16 [[B]], i32 8
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i8 [[X]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+  %b = load b16, ptr %gep.src, align 2
+  %x = bitextract i8, b16 %b, i32 8
+  %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+  store i8 %x, ptr %gep.dst, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitextract_uniform_src(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: define void @bitextract_uniform_src(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = load b16, ptr [[SRC]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitextract i8, b16 [[TMP1]], i32 8
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP2]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i8> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[B:%.*]] = load b16, ptr [[SRC]], align 2
+; CHECK-NEXT:    [[X:%.*]] = bitextract i8, b16 [[B]], i32 8
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i8 [[X]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %b = load b16, ptr %src, align 2
+  %x = bitextract i8, b16 %b, i32 8
+  %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+  store i8 %x, ptr %gep.dst, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitextract_invariant_offset(ptr noalias %dst, ptr noalias %src, i32 %off, i64 %n) {
+; CHECK-LABEL: define void @bitextract_invariant_offset(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i32 [[OFF:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[OFF]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x b16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitextract <4 x i8>, <4 x b16> [[WIDE_LOAD]], <4 x i32> [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i8> [[TMP2]], ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[B:%.*]] = load b16, ptr [[GEP_SRC]], align 2
+; CHECK-NEXT:    [[X:%.*]] = bitextract i8, b16 [[B]], i32 [[OFF]]
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i8 [[X]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+  %b = load b16, ptr %gep.src, align 2
+  %x = bitextract i8, b16 %b, i32 %off
+  %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+  store i8 %x, ptr %gep.dst, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitextract_varying_offset(ptr noalias %dst, ptr noalias %src, ptr noalias %offs, i64 %n) {
+; CHECK-LABEL: define void @bitextract_varying_offset(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[OFFS:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds b32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x b32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OFFS]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitextract <4 x i8>, <4 x b32> [[WIDE_LOAD]], <4 x i32> [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i8> [[TMP3]], ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds b32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[B:%.*]] = load b32, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[GEP_OFF:%.*]] = getelementptr inbounds i32, ptr [[OFFS]], i64 [[IV]]
+; CHECK-NEXT:    [[OFF:%.*]] = load i32, ptr [[GEP_OFF]], align 4
+; CHECK-NEXT:    [[X:%.*]] = bitextract i8, b32 [[B]], i32 [[OFF]]
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i8 [[X]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b32, ptr %src, i64 %iv
+  %b = load b32, ptr %gep.src, align 4
+  %gep.off = getelementptr inbounds i32, ptr %offs, i64 %iv
+  %off = load i32, ptr %gep.off, align 4
+  %x = bitextract i8, b32 %b, i32 %off
+  %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+  store i8 %x, ptr %gep.dst, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitextract_float_result(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: define void @bitextract_float_result(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x b64>, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitextract <4 x float>, <4 x b64> [[WIDE_LOAD]], <4 x i32> splat (i32 32)
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x float> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[B:%.*]] = load b64, ptr [[GEP_SRC]], align 8
+; CHECK-NEXT:    [[X:%.*]] = bitextract float, b64 [[B]], i32 32
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store float [[X]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+  %b = load b64, ptr %gep.src, align 8
+  %x = bitextract float, b64 %b, i32 32
+  %gep.dst = getelementptr inbounds float, ptr %dst, i64 %iv
+  store float %x, ptr %gep.dst, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @bitextract_ptr_result(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: define void @bitextract_ptr_result(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x b64>, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitextract <4 x ptr>, <4 x b64> [[WIDE_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x ptr> [[TMP2]], ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[B:%.*]] = load b64, ptr [[GEP_SRC]], align 8
+; CHECK-NEXT:    [[X:%.*]] = bitextract ptr, b64 [[B]], i32 0
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds ptr, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store ptr [[X]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+  %b = load b64, ptr %gep.src, align 8
+  %x = bitextract ptr, b64 %b, i32 0
+  %gep.dst = getelementptr inbounds ptr, ptr %dst, i64 %iv
+  store ptr %x, ptr %gep.dst, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Test case for https://github.com/llvm/llvm-project/issues/241651.
+define void @bitextract_select_src(ptr noalias %dst, ptr noalias %conds, b128 %b1, b128 %b2, i64 %n) {
+; CHECK-LABEL: define void @bitextract_select_src(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[CONDS:%.*]], b128 [[B1:%.*]], b128 [[B2:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x b128> poison, b128 [[B1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x b128> [[BROADCAST_SPLATINSERT]], <4 x b128> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x b128> poison, b128 [[B2]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x b128> [[BROADCAST_SPLATINSERT1]], <4 x b128> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[CONDS]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x b128> [[BROADCAST_SPLAT]], <4 x b128> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitextract <4 x i32>, <4 x b128> [[TMP3]], <4 x i32> splat (i32 16)
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_COND:%.*]] = getelementptr inbounds i8, ptr [[CONDS]], i64 [[IV]]
+; CHECK-NEXT:    [[C:%.*]] = load i8, ptr [[GEP_COND]], align 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-NEXT:    [[B:%.*]] = select i1 [[CMP]], b128 [[B1]], b128 [[B2]]
+; CHECK-NEXT:    [[X:%.*]] = bitextract i32, b128 [[B]], i32 16
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i32 [[X]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.cond = getelementptr inbounds i8, ptr %conds, i64 %iv
+  %c = load i8, ptr %gep.cond, align 1
+  %cmp = icmp eq i8 %c, 0
+  %b = select i1 %cmp, b128 %b1, b128 %b2
+  %x = bitextract i32, b128 %b, i32 16
+  %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 %x, ptr %gep.dst, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-bitinsert-bitextract.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-bitinsert-bitextract.ll
new file mode 100644
index 00000000000000..1c6b501acb66e8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/outer-loop-bitinsert-bitextract.ll
@@ -0,0 +1,174 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -force-vector-width=4 -S %s | FileCheck %s
+
+; The VPlan-native path widens bitinsert and bitextract in outer loops. The
+; result type of bitextract can't be computed from its operands, so the recipe
+; takes it from the instruction.
+
+define void @outer_loop_bitinsert(ptr noalias %dst, ptr noalias %src, i8 %v, i64 %n) {
+; CHECK-LABEL: define void @outer_loop_bitinsert(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i8 [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[V]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH3:.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x b16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitinsert <4 x b16> [[WIDE_LOAD]], <4 x i8> [[BROADCAST_SPLAT]], <4 x i32> splat (i32 8)
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    br label %[[INNER1:.*]]
+; CHECK:       [[INNER1]]:
+; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER1]] ]
+; CHECK-NEXT:    store <4 x b16> [[TMP2]], ptr [[TMP3]], align 2
+; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 8)
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[OUTER_LATCH3]], label %[[INNER1]]
+; CHECK:       [[OUTER_LATCH3]]:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[OUTER_IV]]
+; CHECK-NEXT:    [[B:%.*]] = load b16, ptr [[GEP_SRC]], align 2
+; CHECK-NEXT:    [[X:%.*]] = bitinsert b16 [[B]], i8 [[V]], i32 8
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[OUTER_IV]]
+; CHECK-NEXT:    br label %[[INNER:.*]]
+; CHECK:       [[INNER]]:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT:    store b16 [[X]], ptr [[GEP_DST]], align 2
+; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
+; CHECK-NEXT:    [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 8
+; CHECK-NEXT:    br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
+; CHECK-NEXT:    [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+  %gep.src = getelementptr inbounds b16, ptr %src, i64 %outer.iv
+  %b = load b16, ptr %gep.src, align 2
+  %x = bitinsert b16 %b, i8 %v, i32 8
+  %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %outer.iv
+  br label %inner
+
+inner:
+  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
+  store b16 %x, ptr %gep.dst, align 2
+  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
+  %inner.ec = icmp eq i64 %inner.iv.next, 8
+  br i1 %inner.ec, label %outer.latch, label %inner
+
+outer.latch:
+  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+  %outer.ec = icmp eq i64 %outer.iv.next, %n
+  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+  ret void
+}
+
+define void @outer_loop_bitextract(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: define void @outer_loop_bitextract(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH3:.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x b16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitextract <4 x i8>, <4 x b16> [[WIDE_LOAD]], <4 x i32> splat (i32 8)
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    br label %[[INNER1:.*]]
+; CHECK:       [[INNER1]]:
+; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER1]] ]
+; CHECK-NEXT:    store <4 x i8> [[TMP2]], ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 8)
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[OUTER_LATCH3]], label %[[INNER1]]
+; CHECK:       [[OUTER_LATCH3]]:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[OUTER_IV]]
+; CHECK-NEXT:    [[B:%.*]] = load b16, ptr [[GEP_SRC]], align 2
+; CHECK-NEXT:    [[X:%.*]] = bitextract i8, b16 [[B]], i32 8
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[OUTER_IV]]
+; CHECK-NEXT:    br label %[[INNER:.*]]
+; CHECK:       [[INNER]]:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT:    store i8 [[X]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
+; CHECK-NEXT:    [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 8
+; CHECK-NEXT:    br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
+; CHECK-NEXT:    [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+  %gep.src = getelementptr inbounds b16, ptr %src, i64 %outer.iv
+  %b = load b16, ptr %gep.src, align 2
+  %x = bitextract i8, b16 %b, i32 8
+  %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %outer.iv
+  br label %inner
+
+inner:
+  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
+  store i8 %x, ptr %gep.dst, align 1
+  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
+  %inner.ec = icmp eq i64 %inner.iv.next, 8
+  br i1 %inner.ec, label %outer.latch, label %inner
+
+outer.latch:
+  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+  %outer.ec = icmp eq i64 %outer.iv.next, %n
+  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !2
+
+exit:
+  ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.enable"}
+!2 = distinct !{!2, !1}



More information about the llvm-branch-commits mailing list