[llvm-branch-commits] [llvm] [LV] Widen `bitinsert` and `bitextract` (PR #242517)
Pedro Lobo via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Nov 17 01:58:17 PST 2026
https://github.com/pedroclobo updated https://github.com/llvm/llvm-project/pull/242517
>From 107d228caef330b5f82e19342fdc3daf5df247e6 Mon Sep 17 00:00:00 2001
From: Pedro Lobo <pedro.lobo at tecnico.ulisboa.pt>
Date: Sun, 8 Nov 2026 10:07:57 +0000
Subject: [PATCH 1/2] [VPlan] Share the result-type helper of
`VPReplicateRecipe` with `VPWidenRecipe` (NFC)
---
llvm/lib/Transforms/Vectorize/VPlan.h | 40 +++++++++++--------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 +-
2 files changed, 26 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 5d89141cde6f2e..6194bcbb6a5ade 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -598,6 +598,11 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
LLVM_ABI Type *computeScalarTypeForInstruction(unsigned Opcode,
ArrayRef<VPValue *> Operands);
+/// Compute the scalar result type for a recipe wrapping \p I with \p Operands
+/// (excluding any predicate mask).
+LLVM_ABI Type *computeScalarTypeForInstruction(const Instruction *I,
+ ArrayRef<VPValue *> Operands);
+
/// VPSingleDefRecipe is a base class for recipes that model a sequence of one
/// or more output IR that define a single result VPValue. Note that
/// VPSingleDefRecipe must inherit from VPRecipeBase before VPSingleDefValue.
@@ -1832,26 +1837,33 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
public VPIRMetadata {
unsigned Opcode;
+ VPWidenRecipe(unsigned Opcode, Type *ResultTy, ArrayRef<VPValue *> Operands,
+ const VPIRFlags &Flags, const VPIRMetadata &Metadata,
+ DebugLoc DL)
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, Operands, ResultTy, Flags,
+ DL),
+ VPIRMetadata(Metadata), Opcode(Opcode) {
+ assert(flagsValidForOpcode(Opcode) &&
+ "Set flags not supported for the provided opcode");
+ assert(hasRequiredFlagsForOpcode(Opcode, getScalarType()) &&
+ "Opcode requires specific flags to be set");
+ }
+
public:
VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
DebugLoc DL = {})
- : VPWidenRecipe(I.getOpcode(), Operands, Flags, Metadata, DL) {
+ : VPWidenRecipe(I.getOpcode(),
+ computeScalarTypeForInstruction(&I, Operands), Operands,
+ Flags, Metadata, DL) {
setUnderlyingValue(&I);
}
VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, Operands,
- computeScalarTypeForInstruction(Opcode, Operands),
- Flags, DL),
- VPIRMetadata(Metadata), Opcode(Opcode) {
- assert(flagsValidForOpcode(Opcode) &&
- "Set flags not supported for the provided opcode");
- assert(hasRequiredFlagsForOpcode(Opcode, getScalarType()) &&
- "Opcode requires specific flags to be set");
- }
+ : VPWidenRecipe(Opcode, computeScalarTypeForInstruction(Opcode, Operands),
+ Operands, Flags, Metadata, DL) {}
~VPWidenRecipe() override = default;
@@ -3505,7 +3517,8 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
const VPIRFlags &Flags = {}, VPIRMetadata Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeWithIRFlags(VPRecipeBase::VPReplicateSC, Operands,
- computeScalarType(I, Operands), Flags, DL),
+ computeScalarTypeForInstruction(I, Operands), Flags,
+ DL),
VPIRMetadata(Metadata), IsSingleScalar(IsSingleScalar),
IsPredicated(Mask) {
assert((!IsSingleScalar || !I->isCast()) &&
@@ -3517,11 +3530,6 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
~VPReplicateRecipe() override = default;
- /// Compute the scalar result type for a VPReplicateRecipe wrapping \p I with
- /// \p Operands (excluding any predicate mask).
- static Type *computeScalarType(const Instruction *I,
- ArrayRef<VPValue *> Operands);
-
VPReplicateRecipe *clone() override { return cloneWithOperands(operands()); }
VPReplicateRecipe *cloneWithOperands(ArrayRef<VPValue *> NewOperands) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 19af59ff4a2949..9f5fca8ea852d9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -633,8 +633,8 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
return Op0Ty;
}
-Type *VPReplicateRecipe::computeScalarType(const Instruction *I,
- ArrayRef<VPValue *> Operands) {
+Type *llvm::computeScalarTypeForInstruction(const Instruction *I,
+ ArrayRef<VPValue *> Operands) {
unsigned Opcode = I->getOpcode();
if (Instruction::isCast(Opcode) ||
is_contained(ArrayRef<unsigned>({Instruction::Load, Instruction::Alloca}),
>From 2c39d2d5c67c8f1944da9fb677aaad7e70f0c97d Mon Sep 17 00:00:00 2001
From: Pedro Lobo <pedro.lobo at tecnico.ulisboa.pt>
Date: Fri, 6 Nov 2026 13:46:52 +0000
Subject: [PATCH 2/2] [LV] Widen `bitinsert` and `bitextract`
---
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 49 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 2 +
.../X86/bitinsert-bitextract-cost.ll | 337 ++++++++
.../LoopVectorize/bitinsert-bitextract.ll | 782 ++++++++++++++++++
.../outer-loop-bitinsert-bitextract.ll | 174 ++++
6 files changed, 1347 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/bitinsert-bitextract-cost.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/bitinsert-bitextract.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/outer-loop-bitinsert-bitextract.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a2a2797b2038fc..cf2e436b72e718 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4885,6 +4885,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
case Instruction::Call:
return getVectorCallCost(cast<CallInst>(I), VF);
case Instruction::ExtractValue:
+ case Instruction::BitInsert:
+ case Instruction::BitExtract:
return TTI.getInstructionCost(I, Config.CostKind);
case Instruction::Alloca:
// We cannot easily widen alloca to a scalable alloca, as
@@ -5902,6 +5904,8 @@ VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
case Instruction::Xor:
case Instruction::Freeze:
case Instruction::ExtractValue:
+ case Instruction::BitInsert:
+ case Instruction::BitExtract:
return new VPWidenRecipe(*I, VPI->operandsWithoutMask(), *VPI, *VPI,
VPI->getDebugLoc());
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 9f5fca8ea852d9..41639c089a22ef 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -605,6 +605,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
case VPInstruction::IncomingAliasMask:
case Instruction::Load:
case Instruction::Alloca:
+ case Instruction::BitExtract:
llvm_unreachable("type must be passed explicitly");
case Instruction::Call:
return getCalledFunction(Operands)->getReturnType();
@@ -637,7 +638,8 @@ Type *llvm::computeScalarTypeForInstruction(const Instruction *I,
ArrayRef<VPValue *> Operands) {
unsigned Opcode = I->getOpcode();
if (Instruction::isCast(Opcode) ||
- is_contained(ArrayRef<unsigned>({Instruction::Load, Instruction::Alloca}),
+ is_contained(ArrayRef<unsigned>({Instruction::Load, Instruction::Alloca,
+ Instruction::BitExtract}),
Opcode))
return I->getType();
return computeScalarTypeForInstruction(Opcode, Operands);
@@ -695,6 +697,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::FCmp:
case Instruction::ExtractElement:
case Instruction::Store:
+ case Instruction::BitExtract:
case VPInstruction::ActiveLaneMask:
case VPInstruction::BranchOnCount:
case VPInstruction::BranchOnTwoConds:
@@ -709,6 +712,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
return 2;
case Instruction::InsertElement:
case Instruction::Select:
+ case Instruction::BitInsert:
case VPInstruction::WideActiveLaneMask:
case VPInstruction::ReductionStartVector:
case VPInstruction::WideVectorLoad:
@@ -1310,6 +1314,29 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
case Instruction::ExtractValue:
return Ctx.TTI.getInsertExtractValueCost(Instruction::ExtractValue,
Ctx.CostKind);
+ case Instruction::BitInsert:
+ case Instruction::BitExtract: {
+ bool IsInsert = Opcode == Instruction::BitInsert;
+ VPValue *Offset = getOperand(IsInsert ? 2 : 1);
+ TargetTransformInfo::OperandValueInfo OffsetInfo =
+ Ctx.getOperandInfo(Offset);
+
+ if (OffsetInfo.Kind == TargetTransformInfo::OK_AnyValue &&
+ Offset->isDefinedOutsideLoopRegions())
+ OffsetInfo.Kind = TargetTransformInfo::OK_UniformValue;
+
+ Type *ScalarOpTy = getOperand(IsInsert ? 1 : 0)->getScalarType();
+ Type *OpTy = VF.isVector() ? toVectorTy(ScalarOpTy, VF) : ScalarOpTy;
+ Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
+ SmallVector<const Value *, 3> Operands;
+ if (CtxI)
+ Operands.append(CtxI->value_op_begin(), CtxI->value_op_end());
+ if (IsInsert)
+ return Ctx.TTI.getBitInsertCost(ResultTy, OpTy, Ctx.CostKind, OffsetInfo,
+ Operands, CtxI);
+ return Ctx.TTI.getBitExtractCost(ResultTy, OpTy, Ctx.CostKind, OffsetInfo,
+ Operands, CtxI);
+ }
case Instruction::ICmp:
case Instruction::FCmp: {
Type *ScalarOpTy = getOperand(0)->getScalarType();
@@ -3099,6 +3126,22 @@ void VPWidenRecipe::execute(VPTransformState &State) {
State.set(this, Freeze);
break;
}
+ case Instruction::BitInsert: {
+ Value *Base = State.get(getOperand(0));
+ Value *Val = State.get(getOperand(1));
+ Value *Offset = State.get(getOperand(2));
+ Value *Insert = Builder.CreateBitInsert(Base, Val, Offset);
+ State.set(this, Insert);
+ break;
+ }
+ case Instruction::BitExtract: {
+ Value *Src = State.get(getOperand(0));
+ Value *Offset = State.get(getOperand(1));
+ Type *DestTy = VectorType::get(getScalarType(), State.VF);
+ Value *Extract = Builder.CreateBitExtract(DestTy, Src, Offset);
+ State.set(this, Extract);
+ break;
+ }
case Instruction::ICmp:
case Instruction::FCmp: {
// Widen compares. Generate vector compares.
@@ -3179,6 +3222,8 @@ InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
case Instruction::ICmp:
case Instruction::FCmp:
case Instruction::Select:
+ case Instruction::BitInsert:
+ case Instruction::BitExtract:
return getCostForRecipeWithOpcode(getOpcode(), VF, Ctx);
default:
llvm_unreachable("Unsupported opcode for instruction");
@@ -4325,6 +4370,8 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
case Instruction::ICmp:
case Instruction::FCmp:
case Instruction::Freeze:
+ case Instruction::BitInsert:
+ case Instruction::BitExtract:
return getCostForRecipeWithOpcode(getOpcode(), ElementCount::getFixed(1),
Ctx) *
(isSingleScalar() ? 1 : VF.getFixedValue());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index f365e755a9d433..084247279dfc02 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -471,6 +471,8 @@ static bool preservesUniformity(unsigned Opcode) {
case Instruction::ICmp:
case Instruction::FCmp:
case Instruction::Select:
+ case Instruction::BitInsert:
+ case Instruction::BitExtract:
case VPInstruction::Not:
case VPInstruction::Broadcast:
case VPInstruction::MaskedCond:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/bitinsert-bitextract-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/bitinsert-bitextract-cost.ll
new file mode 100644
index 00000000000000..cacddac771effd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/bitinsert-bitextract-cost.ll
@@ -0,0 +1,337 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%x = bit" --filter "Cost of [0-9]+ for VF [0-9]+: (WIDEN|CLONE) ir<%x> = bit" --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=x86_64-- -mcpu=x86-64-v3 -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+define void @bitinsert_varying_base(ptr noalias %dst, ptr noalias %src, i8 %v, i64 %n) {
+; CHECK-LABEL: 'bitinsert_varying_base'
+; CHECK: LV: Found an estimated cost of 4 for VF 1 For instruction: %x = bitinsert b16 %b, i8 %v, i32 8
+; CHECK: Cost of 4 for VF 2: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+; CHECK: Cost of 4 for VF 4: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+; CHECK: Cost of 4 for VF 8: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+; CHECK: Cost of 6 for VF 16: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+ %b = load b16, ptr %gep.src, align 2
+ %x = bitinsert b16 %b, i8 %v, i32 8
+ %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %iv
+ store b16 %x, ptr %gep.dst, align 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitinsert_uniform_base(ptr noalias %dst, ptr noalias %src, i8 %v, i64 %n) {
+; CHECK-LABEL: 'bitinsert_uniform_base'
+; CHECK: LV: Found an estimated cost of 4 for VF 1 For instruction: %x = bitinsert b16 %b, i8 %v, i32 8
+; CHECK: Cost of 4 for VF 2: CLONE ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+; CHECK: Cost of 4 for VF 4: CLONE ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+; CHECK: Cost of 4 for VF 8: CLONE ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+; CHECK: Cost of 4 for VF 16: CLONE ir<%x> = bitinsert ir<%b>, ir<%v>, ir<8>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %b = load b16, ptr %src, align 2
+ %x = bitinsert b16 %b, i8 %v, i32 8
+ %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %iv
+ store b16 %x, ptr %gep.dst, align 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitinsert_invariant_offset(ptr noalias %dst, ptr noalias %src, i8 %v, i32 %off, i64 %n) {
+; CHECK-LABEL: 'bitinsert_invariant_offset'
+; CHECK: LV: Found an estimated cost of 6 for VF 1 For instruction: %x = bitinsert b16 %b, i8 %v, i32 %off
+; CHECK: Cost of 6 for VF 2: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+; CHECK: Cost of 6 for VF 4: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+; CHECK: Cost of 6 for VF 8: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+; CHECK: Cost of 9 for VF 16: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+ %b = load b16, ptr %gep.src, align 2
+ %x = bitinsert b16 %b, i8 %v, i32 %off
+ %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %iv
+ store b16 %x, ptr %gep.dst, align 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitinsert_varying_offset(ptr noalias %dst, ptr noalias %src, ptr noalias %vals, ptr noalias %offs, i64 %n) {
+; CHECK-LABEL: 'bitinsert_varying_offset'
+; CHECK: LV: Found an estimated cost of 6 for VF 1 For instruction: %x = bitinsert b32 %b, i8 %v, i32 %off
+; CHECK: Cost of 8 for VF 2: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+; CHECK: Cost of 8 for VF 4: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+; CHECK: Cost of 13 for VF 8: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<%off>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b32, ptr %src, i64 %iv
+ %b = load b32, ptr %gep.src, align 4
+ %gep.val = getelementptr inbounds i8, ptr %vals, i64 %iv
+ %v = load i8, ptr %gep.val, align 1
+ %gep.off = getelementptr inbounds i32, ptr %offs, i64 %iv
+ %off = load i32, ptr %gep.off, align 4
+ %x = bitinsert b32 %b, i8 %v, i32 %off
+ %gep.dst = getelementptr inbounds b32, ptr %dst, i64 %iv
+ store b32 %x, ptr %gep.dst, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitinsert_float_value(ptr noalias %dst, ptr noalias %src, float %v, i64 %n) {
+; CHECK-LABEL: 'bitinsert_float_value'
+; CHECK: LV: Found an estimated cost of 3 for VF 1 For instruction: %x = bitinsert b64 %b, float %v, i32 32
+; CHECK: Cost of 4 for VF 2: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<32>
+; CHECK: Cost of 6 for VF 4: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<32>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+ %b = load b64, ptr %gep.src, align 8
+ %x = bitinsert b64 %b, float %v, i32 32
+ %gep.dst = getelementptr inbounds b64, ptr %dst, i64 %iv
+ store b64 %x, ptr %gep.dst, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitinsert_ptr_value(ptr noalias %dst, ptr noalias %src, ptr %v, i64 %n) {
+; CHECK-LABEL: 'bitinsert_ptr_value'
+; CHECK: LV: Found an estimated cost of 3 for VF 1 For instruction: %x = bitinsert b64 %b, ptr %v, i32 0
+; CHECK: Cost of 3 for VF 2: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<0>
+; CHECK: Cost of 4 for VF 4: WIDEN ir<%x> = bitinsert ir<%b>, ir<%v>, ir<0>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+ %b = load b64, ptr %gep.src, align 8
+ %x = bitinsert b64 %b, ptr %v, i32 0
+ %gep.dst = getelementptr inbounds b64, ptr %dst, i64 %iv
+ store b64 %x, ptr %gep.dst, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitextract_varying_src(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: 'bitextract_varying_src'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %x = bitextract i8, b16 %b, i32 8
+; CHECK: Cost of 2 for VF 2: WIDEN ir<%x> = bitextract ir<%b>, ir<8>
+; CHECK: Cost of 2 for VF 4: WIDEN ir<%x> = bitextract ir<%b>, ir<8>
+; CHECK: Cost of 2 for VF 8: WIDEN ir<%x> = bitextract ir<%b>, ir<8>
+; CHECK: Cost of 4 for VF 16: WIDEN ir<%x> = bitextract ir<%b>, ir<8>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+ %b = load b16, ptr %gep.src, align 2
+ %x = bitextract i8, b16 %b, i32 8
+ %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+ store i8 %x, ptr %gep.dst, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitextract_uniform_src(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: 'bitextract_uniform_src'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %x = bitextract i8, b16 %b, i32 8
+; CHECK: Cost of 1 for VF 2: CLONE ir<%x> = bitextract ir<%b>, ir<8>
+; CHECK: Cost of 1 for VF 4: CLONE ir<%x> = bitextract ir<%b>, ir<8>
+; CHECK: Cost of 1 for VF 8: CLONE ir<%x> = bitextract ir<%b>, ir<8>
+; CHECK: Cost of 1 for VF 16: CLONE ir<%x> = bitextract ir<%b>, ir<8>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %b = load b16, ptr %src, align 2
+ %x = bitextract i8, b16 %b, i32 8
+ %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+ store i8 %x, ptr %gep.dst, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitextract_invariant_offset(ptr noalias %dst, ptr noalias %src, i32 %off, i64 %n) {
+; CHECK-LABEL: 'bitextract_invariant_offset'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %x = bitextract i8, b16 %b, i32 %off
+; CHECK: Cost of 2 for VF 2: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+; CHECK: Cost of 2 for VF 4: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+; CHECK: Cost of 2 for VF 8: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+; CHECK: Cost of 4 for VF 16: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+ %b = load b16, ptr %gep.src, align 2
+ %x = bitextract i8, b16 %b, i32 %off
+ %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+ store i8 %x, ptr %gep.dst, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitextract_varying_offset(ptr noalias %dst, ptr noalias %src, ptr noalias %offs, i64 %n) {
+; CHECK-LABEL: 'bitextract_varying_offset'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %x = bitextract i8, b32 %b, i32 %off
+; CHECK: Cost of 3 for VF 2: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+; CHECK: Cost of 3 for VF 4: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+; CHECK: Cost of 8 for VF 8: WIDEN ir<%x> = bitextract ir<%b>, ir<%off>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b32, ptr %src, i64 %iv
+ %b = load b32, ptr %gep.src, align 4
+ %gep.off = getelementptr inbounds i32, ptr %offs, i64 %iv
+ %off = load i32, ptr %gep.off, align 4
+ %x = bitextract i8, b32 %b, i32 %off
+ %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+ store i8 %x, ptr %gep.dst, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitextract_float_result(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: 'bitextract_float_result'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %x = bitextract float, b64 %b, i32 32
+; CHECK: Cost of 2 for VF 2: WIDEN ir<%x> = bitextract ir<%b>, ir<32>
+; CHECK: Cost of 3 for VF 4: WIDEN ir<%x> = bitextract ir<%b>, ir<32>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+ %b = load b64, ptr %gep.src, align 8
+ %x = bitextract float, b64 %b, i32 32
+ %gep.dst = getelementptr inbounds float, ptr %dst, i64 %iv
+ store float %x, ptr %gep.dst, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitextract_ptr_result(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: 'bitextract_ptr_result'
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %x = bitextract ptr, b64 %b, i32 0
+; CHECK: Cost of 1 for VF 2: WIDEN ir<%x> = bitextract ir<%b>, ir<0>
+; CHECK: Cost of 2 for VF 4: WIDEN ir<%x> = bitextract ir<%b>, ir<0>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+ %b = load b64, ptr %gep.src, align 8
+ %x = bitextract ptr, b64 %b, i32 0
+ %gep.dst = getelementptr inbounds ptr, ptr %dst, i64 %iv
+ store ptr %x, ptr %gep.dst, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Test case for https://github.com/llvm/llvm-project/issues/241651.
+define void @bitextract_select_src(ptr noalias %dst, ptr noalias %conds, b128 %b1, b128 %b2, i64 %n) {
+; CHECK-LABEL: 'bitextract_select_src'
+; CHECK: LV: Found an estimated cost of 2 for VF 1 For instruction: %x = bitextract i32, b128 %b, i32 16
+; CHECK: Cost of 4 for VF 2: WIDEN ir<%x> = bitextract ir<%b>, ir<16>
+; CHECK: Cost of 9 for VF 4: WIDEN ir<%x> = bitextract ir<%b>, ir<16>
+; CHECK: Cost of 19 for VF 8: WIDEN ir<%x> = bitextract ir<%b>, ir<16>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.cond = getelementptr inbounds i8, ptr %conds, i64 %iv
+ %c = load i8, ptr %gep.cond, align 1
+ %cmp = icmp eq i8 %c, 0
+ %b = select i1 %cmp, b128 %b1, b128 %b2
+ %x = bitextract i32, b128 %b, i32 16
+ %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 %x, ptr %gep.dst, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/bitinsert-bitextract.ll b/llvm/test/Transforms/LoopVectorize/bitinsert-bitextract.ll
new file mode 100644
index 00000000000000..ba468859fd8588
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/bitinsert-bitextract.ll
@@ -0,0 +1,782 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+define void @bitinsert_varying_base(ptr noalias %dst, ptr noalias %src, i8 %v, i64 %n) {
+; CHECK-LABEL: define void @bitinsert_varying_base(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i8 [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[V]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x b16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = bitinsert <4 x b16> [[WIDE_LOAD]], <4 x i8> [[BROADCAST_SPLAT]], <4 x i32> splat (i32 8)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x b16> [[TMP2]], ptr [[TMP3]], align 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[B:%.*]] = load b16, ptr [[GEP_SRC]], align 2
+; CHECK-NEXT: [[X:%.*]] = bitinsert b16 [[B]], i8 [[V]], i32 8
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store b16 [[X]], ptr [[GEP_DST]], align 2
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+ %b = load b16, ptr %gep.src, align 2
+ %x = bitinsert b16 %b, i8 %v, i32 8
+ %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %iv
+ store b16 %x, ptr %gep.dst, align 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitinsert_uniform_base(ptr noalias %dst, ptr noalias %src, i8 %v, i64 %n) {
+; CHECK-LABEL: define void @bitinsert_uniform_base(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i8 [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = load b16, ptr [[SRC]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = bitinsert b16 [[TMP1]], i8 [[V]], i32 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x b16> poison, b16 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x b16> [[BROADCAST_SPLATINSERT]], <4 x b16> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x b16> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[B:%.*]] = load b16, ptr [[SRC]], align 2
+; CHECK-NEXT: [[X:%.*]] = bitinsert b16 [[B]], i8 [[V]], i32 8
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store b16 [[X]], ptr [[GEP_DST]], align 2
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %b = load b16, ptr %src, align 2
+ %x = bitinsert b16 %b, i8 %v, i32 8
+ %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %iv
+ store b16 %x, ptr %gep.dst, align 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitinsert_invariant_offset(ptr noalias %dst, ptr noalias %src, i8 %v, i32 %off, i64 %n) {
+; CHECK-LABEL: define void @bitinsert_invariant_offset(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i8 [[V:%.*]], i32 [[OFF:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[V]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[OFF]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x b16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = bitinsert <4 x b16> [[WIDE_LOAD]], <4 x i8> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x b16> [[TMP2]], ptr [[TMP3]], align 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[B:%.*]] = load b16, ptr [[GEP_SRC]], align 2
+; CHECK-NEXT: [[X:%.*]] = bitinsert b16 [[B]], i8 [[V]], i32 [[OFF]]
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store b16 [[X]], ptr [[GEP_DST]], align 2
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+ %b = load b16, ptr %gep.src, align 2
+ %x = bitinsert b16 %b, i8 %v, i32 %off
+ %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %iv
+ store b16 %x, ptr %gep.dst, align 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitinsert_varying_offset(ptr noalias %dst, ptr noalias %src, ptr noalias %vals, ptr noalias %offs, i64 %n) {
+; CHECK-LABEL: define void @bitinsert_varying_offset(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[VALS:%.*]], ptr noalias [[OFFS:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds b32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x b32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[VALS]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OFFS]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = bitinsert <4 x b32> [[WIDE_LOAD]], <4 x i8> [[WIDE_LOAD1]], <4 x i32> [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds b32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x b32> [[TMP4]], ptr [[TMP5]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds b32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[B:%.*]] = load b32, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[GEP_VAL:%.*]] = getelementptr inbounds i8, ptr [[VALS]], i64 [[IV]]
+; CHECK-NEXT: [[V:%.*]] = load i8, ptr [[GEP_VAL]], align 1
+; CHECK-NEXT: [[GEP_OFF:%.*]] = getelementptr inbounds i32, ptr [[OFFS]], i64 [[IV]]
+; CHECK-NEXT: [[OFF:%.*]] = load i32, ptr [[GEP_OFF]], align 4
+; CHECK-NEXT: [[X:%.*]] = bitinsert b32 [[B]], i8 [[V]], i32 [[OFF]]
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds b32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store b32 [[X]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b32, ptr %src, i64 %iv
+ %b = load b32, ptr %gep.src, align 4
+ %gep.val = getelementptr inbounds i8, ptr %vals, i64 %iv
+ %v = load i8, ptr %gep.val, align 1
+ %gep.off = getelementptr inbounds i32, ptr %offs, i64 %iv
+ %off = load i32, ptr %gep.off, align 4
+ %x = bitinsert b32 %b, i8 %v, i32 %off
+ %gep.dst = getelementptr inbounds b32, ptr %dst, i64 %iv
+ store b32 %x, ptr %gep.dst, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitinsert_float_value(ptr noalias %dst, ptr noalias %src, float %v, i64 %n) {
+; CHECK-LABEL: define void @bitinsert_float_value(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], float [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[V]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x b64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = bitinsert <4 x b64> [[WIDE_LOAD]], <4 x float> [[BROADCAST_SPLAT]], <4 x i32> splat (i32 32)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds b64, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x b64> [[TMP2]], ptr [[TMP3]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[B:%.*]] = load b64, ptr [[GEP_SRC]], align 8
+; CHECK-NEXT: [[X:%.*]] = bitinsert b64 [[B]], float [[V]], i32 32
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds b64, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store b64 [[X]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+ %b = load b64, ptr %gep.src, align 8
+ %x = bitinsert b64 %b, float %v, i32 32
+ %gep.dst = getelementptr inbounds b64, ptr %dst, i64 %iv
+ store b64 %x, ptr %gep.dst, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitinsert_ptr_value(ptr noalias %dst, ptr noalias %src, ptr %v, i64 %n) {
+; CHECK-LABEL: define void @bitinsert_ptr_value(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], ptr [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x ptr> poison, ptr [[V]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x b64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = bitinsert <4 x b64> [[WIDE_LOAD]], <4 x ptr> [[BROADCAST_SPLAT]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds b64, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x b64> [[TMP2]], ptr [[TMP3]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[B:%.*]] = load b64, ptr [[GEP_SRC]], align 8
+; CHECK-NEXT: [[X:%.*]] = bitinsert b64 [[B]], ptr [[V]], i32 0
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds b64, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store b64 [[X]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+ %b = load b64, ptr %gep.src, align 8
+ %x = bitinsert b64 %b, ptr %v, i32 0
+ %gep.dst = getelementptr inbounds b64, ptr %dst, i64 %iv
+ store b64 %x, ptr %gep.dst, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitextract_varying_src(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: define void @bitextract_varying_src(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x b16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = bitextract <4 x i8>, <4 x b16> [[WIDE_LOAD]], <4 x i32> splat (i32 8)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i8> [[TMP2]], ptr [[TMP3]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[B:%.*]] = load b16, ptr [[GEP_SRC]], align 2
+; CHECK-NEXT: [[X:%.*]] = bitextract i8, b16 [[B]], i32 8
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i8 [[X]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+ %b = load b16, ptr %gep.src, align 2
+ %x = bitextract i8, b16 %b, i32 8
+ %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+ store i8 %x, ptr %gep.dst, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitextract_uniform_src(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: define void @bitextract_uniform_src(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = load b16, ptr [[SRC]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = bitextract i8, b16 [[TMP1]], i32 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i8> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[B:%.*]] = load b16, ptr [[SRC]], align 2
+; CHECK-NEXT: [[X:%.*]] = bitextract i8, b16 [[B]], i32 8
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i8 [[X]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %b = load b16, ptr %src, align 2
+ %x = bitextract i8, b16 %b, i32 8
+ %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+ store i8 %x, ptr %gep.dst, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitextract_invariant_offset(ptr noalias %dst, ptr noalias %src, i32 %off, i64 %n) {
+; CHECK-LABEL: define void @bitextract_invariant_offset(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i32 [[OFF:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[OFF]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x b16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = bitextract <4 x i8>, <4 x b16> [[WIDE_LOAD]], <4 x i32> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i8> [[TMP2]], ptr [[TMP3]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[B:%.*]] = load b16, ptr [[GEP_SRC]], align 2
+; CHECK-NEXT: [[X:%.*]] = bitextract i8, b16 [[B]], i32 [[OFF]]
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i8 [[X]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b16, ptr %src, i64 %iv
+ %b = load b16, ptr %gep.src, align 2
+ %x = bitextract i8, b16 %b, i32 %off
+ %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+ store i8 %x, ptr %gep.dst, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitextract_varying_offset(ptr noalias %dst, ptr noalias %src, ptr noalias %offs, i64 %n) {
+; CHECK-LABEL: define void @bitextract_varying_offset(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[OFFS:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds b32, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x b32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OFFS]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = bitextract <4 x i8>, <4 x b32> [[WIDE_LOAD]], <4 x i32> [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i8> [[TMP3]], ptr [[TMP4]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds b32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[B:%.*]] = load b32, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[GEP_OFF:%.*]] = getelementptr inbounds i32, ptr [[OFFS]], i64 [[IV]]
+; CHECK-NEXT: [[OFF:%.*]] = load i32, ptr [[GEP_OFF]], align 4
+; CHECK-NEXT: [[X:%.*]] = bitextract i8, b32 [[B]], i32 [[OFF]]
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i8 [[X]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b32, ptr %src, i64 %iv
+ %b = load b32, ptr %gep.src, align 4
+ %gep.off = getelementptr inbounds i32, ptr %offs, i64 %iv
+ %off = load i32, ptr %gep.off, align 4
+ %x = bitextract i8, b32 %b, i32 %off
+ %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+ store i8 %x, ptr %gep.dst, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitextract_float_result(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: define void @bitextract_float_result(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x b64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = bitextract <4 x float>, <4 x b64> [[WIDE_LOAD]], <4 x i32> splat (i32 32)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[B:%.*]] = load b64, ptr [[GEP_SRC]], align 8
+; CHECK-NEXT: [[X:%.*]] = bitextract float, b64 [[B]], i32 32
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store float [[X]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+ %b = load b64, ptr %gep.src, align 8
+ %x = bitextract float, b64 %b, i32 32
+ %gep.dst = getelementptr inbounds float, ptr %dst, i64 %iv
+ store float %x, ptr %gep.dst, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @bitextract_ptr_result(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: define void @bitextract_ptr_result(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x b64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = bitextract <4 x ptr>, <4 x b64> [[WIDE_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x ptr> [[TMP2]], ptr [[TMP3]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds b64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[B:%.*]] = load b64, ptr [[GEP_SRC]], align 8
+; CHECK-NEXT: [[X:%.*]] = bitextract ptr, b64 [[B]], i32 0
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds ptr, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store ptr [[X]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.src = getelementptr inbounds b64, ptr %src, i64 %iv
+ %b = load b64, ptr %gep.src, align 8
+ %x = bitextract ptr, b64 %b, i32 0
+ %gep.dst = getelementptr inbounds ptr, ptr %dst, i64 %iv
+ store ptr %x, ptr %gep.dst, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Test case for https://github.com/llvm/llvm-project/issues/241651.
+define void @bitextract_select_src(ptr noalias %dst, ptr noalias %conds, b128 %b1, b128 %b2, i64 %n) {
+; CHECK-LABEL: define void @bitextract_select_src(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[CONDS:%.*]], b128 [[B1:%.*]], b128 [[B2:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x b128> poison, b128 [[B1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x b128> [[BROADCAST_SPLATINSERT]], <4 x b128> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x b128> poison, b128 [[B2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x b128> [[BROADCAST_SPLATINSERT1]], <4 x b128> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[CONDS]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x b128> [[BROADCAST_SPLAT]], <4 x b128> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitextract <4 x i32>, <4 x b128> [[TMP3]], <4 x i32> splat (i32 16)
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_COND:%.*]] = getelementptr inbounds i8, ptr [[CONDS]], i64 [[IV]]
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[GEP_COND]], align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-NEXT: [[B:%.*]] = select i1 [[CMP]], b128 [[B1]], b128 [[B2]]
+; CHECK-NEXT: [[X:%.*]] = bitextract i32, b128 [[B]], i32 16
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[X]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.cond = getelementptr inbounds i8, ptr %conds, i64 %iv
+ %c = load i8, ptr %gep.cond, align 1
+ %cmp = icmp eq i8 %c, 0
+ %b = select i1 %cmp, b128 %b1, b128 %b2
+ %x = bitextract i32, b128 %b, i32 16
+ %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 %x, ptr %gep.dst, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-bitinsert-bitextract.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-bitinsert-bitextract.ll
new file mode 100644
index 00000000000000..1c6b501acb66e8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/outer-loop-bitinsert-bitextract.ll
@@ -0,0 +1,174 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -force-vector-width=4 -S %s | FileCheck %s
+
+; The VPlan-native path widens bitinsert and bitextract in outer loops. The
+; result type of bitextract can't be computed from its operands, so the recipe
+; takes it from the instruction.
+
+define void @outer_loop_bitinsert(ptr noalias %dst, ptr noalias %src, i8 %v, i64 %n) {
+; CHECK-LABEL: define void @outer_loop_bitinsert(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i8 [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[V]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH3:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x b16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = bitinsert <4 x b16> [[WIDE_LOAD]], <4 x i8> [[BROADCAST_SPLAT]], <4 x i32> splat (i32 8)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: br label %[[INNER1:.*]]
+; CHECK: [[INNER1]]:
+; CHECK-NEXT: [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER1]] ]
+; CHECK-NEXT: store <4 x b16> [[TMP2]], ptr [[TMP3]], align 2
+; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 8)
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: br i1 [[TMP6]], label %[[OUTER_LATCH3]], label %[[INNER1]]
+; CHECK: [[OUTER_LATCH3]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[OUTER_IV]]
+; CHECK-NEXT: [[B:%.*]] = load b16, ptr [[GEP_SRC]], align 2
+; CHECK-NEXT: [[X:%.*]] = bitinsert b16 [[B]], i8 [[V]], i32 8
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds b16, ptr [[DST]], i64 [[OUTER_IV]]
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: store b16 [[X]], ptr [[GEP_DST]], align 2
+; CHECK-NEXT: [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
+; CHECK-NEXT: [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 8
+; CHECK-NEXT: br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
+; CHECK-NEXT: [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %gep.src = getelementptr inbounds b16, ptr %src, i64 %outer.iv
+ %b = load b16, ptr %gep.src, align 2
+ %x = bitinsert b16 %b, i8 %v, i32 8
+ %gep.dst = getelementptr inbounds b16, ptr %dst, i64 %outer.iv
+ br label %inner
+
+inner:
+ %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
+ store b16 %x, ptr %gep.dst, align 2
+ %inner.iv.next = add nuw nsw i64 %inner.iv, 1
+ %inner.ec = icmp eq i64 %inner.iv.next, 8
+ br i1 %inner.ec, label %outer.latch, label %inner
+
+outer.latch:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.ec = icmp eq i64 %outer.iv.next, %n
+ br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+define void @outer_loop_bitextract(ptr noalias %dst, ptr noalias %src, i64 %n) {
+; CHECK-LABEL: define void @outer_loop_bitextract(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH3:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x b16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = bitextract <4 x i8>, <4 x b16> [[WIDE_LOAD]], <4 x i32> splat (i32 8)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: br label %[[INNER1:.*]]
+; CHECK: [[INNER1]]:
+; CHECK-NEXT: [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER1]] ]
+; CHECK-NEXT: store <4 x i8> [[TMP2]], ptr [[TMP3]], align 1
+; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 8)
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: br i1 [[TMP6]], label %[[OUTER_LATCH3]], label %[[INNER1]]
+; CHECK: [[OUTER_LATCH3]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds b16, ptr [[SRC]], i64 [[OUTER_IV]]
+; CHECK-NEXT: [[B:%.*]] = load b16, ptr [[GEP_SRC]], align 2
+; CHECK-NEXT: [[X:%.*]] = bitextract i8, b16 [[B]], i32 8
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[OUTER_IV]]
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: store i8 [[X]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT: [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
+; CHECK-NEXT: [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 8
+; CHECK-NEXT: br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
+; CHECK-NEXT: [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+ %gep.src = getelementptr inbounds b16, ptr %src, i64 %outer.iv
+ %b = load b16, ptr %gep.src, align 2
+ %x = bitextract i8, b16 %b, i32 8
+ %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %outer.iv
+ br label %inner
+
+inner:
+ %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
+ store i8 %x, ptr %gep.dst, align 1
+ %inner.iv.next = add nuw nsw i64 %inner.iv, 1
+ %inner.ec = icmp eq i64 %inner.iv.next, 8
+ br i1 %inner.ec, label %outer.latch, label %inner
+
+outer.latch:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.ec = icmp eq i64 %outer.iv.next, %n
+ br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !2
+
+exit:
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.enable"}
+!2 = distinct !{!2, !1}
More information about the llvm-branch-commits
mailing list