[llvm] Use (de)interleaving intrinsics for fixed-length vectors (PR #202583)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 04:21:44 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Kamlesh Kumar (kamleshbhalui)
<details>
<summary>Changes</summary>
unifying LLVM’s loop vectorization by using
target-independent `interleave` and `deinterleave` intrinsics for
fixed-length vector for interleaved accesses, matching the representation already
used for scalable vectors.
---
Patch is 220.77 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/202583.diff
33 Files Affected:
- (modified) llvm/include/llvm/Analysis/TargetTransformInfo.h (+6)
- (modified) llvm/include/llvm/Analysis/TargetTransformInfoImpl.h (+5)
- (modified) llvm/include/llvm/CodeGen/BasicTTIImpl.h (+5)
- (modified) llvm/include/llvm/CodeGen/TargetLowering.h (+4-6)
- (modified) llvm/lib/Analysis/TargetTransformInfo.cpp (+5)
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp (+11)
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h (+3)
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+37-32)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll (+10-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll (+6-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll (+6-11)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/interleave-allocsize-not-equal-typesize.ll (+2-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll (+6-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll (+6-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/interleaved-store-of-first-order-recurrence.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll (+12-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll (+4-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll (+4-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll (+6-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll (+6-13)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll (+32-32)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll (+19-22)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll (+12-12)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll (+24-24)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll (+14-12)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-live-outs.ll (+14-11)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-and-casts.ll (+14-28)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll (+26-28)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll (+93-101)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll (+28-32)
- (modified) llvm/test/Transforms/PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll (+6-4)
- (modified) llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll (+86-15)
- (modified) llvm/test/Transforms/PhaseOrdering/AArch64/interleavevectorization.ll (+117-39)
``````````diff
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 7d58473b81265..5f2fb3dbe3205 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1497,6 +1497,12 @@ class TargetTransformInfo {
/// and the number of execution units in the CPU.
LLVM_ABI unsigned getMaxInterleaveFactor(ElementCount VF) const;
+ /// \return True if the target supports vector interleave/deinterleave
+ /// intrinsics for the \p Factor and \p VecTy.
+ LLVM_ABI bool
+ supportsVectorInterleaveDeinterleaveIntrinsics(unsigned Factor,
+ VectorType *VecTy) const;
+
/// Collect properties of V used in cost analysis, e.g. OP_PowerOf2.
LLVM_ABI static OperandValueInfo getOperandInfo(const Value *V);
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 89fd4a1e7628e..0790749888dde 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -721,6 +721,11 @@ class LLVM_ABI TargetTransformInfoImplBase {
}
virtual unsigned getMaxInterleaveFactor(ElementCount VF) const { return 1; }
+ virtual bool
+ supportsVectorInterleaveDeinterleaveIntrinsics(unsigned Factor,
+ VectorType *VecTy) const {
+ return VecTy->isScalableTy();
+ }
virtual InstructionCost getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index e4b6bf51c7a4e..4f1137f63e39a 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1045,6 +1045,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
unsigned getMaxInterleaveFactor(ElementCount VF) const override { return 1; }
+ bool supportsVectorInterleaveDeinterleaveIntrinsics(
+ unsigned Factor, VectorType *VecTy) const override {
+ return VecTy->isScalableTy();
+ }
+
InstructionCost getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Opd1Info = {TTI::OK_AnyValue, TTI::OP_None},
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 82c47cce0f522..395b0602b46a3 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -3304,7 +3304,7 @@ class LLVM_ABI TargetLoweringBase {
/// instruction or a vp.load intrinsic.
/// \p Mask is a per-segment (i.e. number of lanes equal to that of one
/// component being interwoven) mask. Can be nullptr, in which case the
- /// result is uncondiitional.
+ /// result is unconditional.
/// \p Shuffles is the shufflevector list to DE-interleave the loaded vector.
/// \p Indices is the corresponding indices for each shufflevector.
/// \p Factor is the interleave factor.
@@ -3336,8 +3336,7 @@ class LLVM_ABI TargetLoweringBase {
}
/// Lower a deinterleave intrinsic to a target specific load intrinsic.
- /// Return true on success. Currently only supports
- /// llvm.vector.deinterleave{2,3,5,7}
+ /// Return true on success for factors supported by the target.
///
/// \p Load is the accompanying load instruction. Can be either a plain load
/// instruction or a vp.load intrinsic.
@@ -3351,14 +3350,13 @@ class LLVM_ABI TargetLoweringBase {
}
/// Lower an interleave intrinsic to a target specific store intrinsic.
- /// Return true on success. Currently only supports
- /// llvm.vector.interleave{2,3,5,7}
+ /// Return true on success for factors supported by the target.
///
/// \p Store is the accompanying store instruction. Can be either a plain
/// store or a vp.store intrinsic.
/// \p Mask is a per-segment (i.e. number of lanes equal to that of one
/// component being interwoven) mask. Can be nullptr, in which case the
- /// result is uncondiitional.
+ /// result is unconditional.
/// \p InterleaveValues contains the interleaved values.
virtual bool
lowerInterleaveIntrinsicToStore(Instruction *Store, Value *Mask,
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 856950ccae595..6fd2a0485bfa3 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -926,6 +926,11 @@ unsigned TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF) const {
return TTIImpl->getMaxInterleaveFactor(VF);
}
+bool TargetTransformInfo::supportsVectorInterleaveDeinterleaveIntrinsics(
+ unsigned Factor, VectorType *VecTy) const {
+ return TTIImpl->supportsVectorInterleaveDeinterleaveIntrinsics(Factor, VecTy);
+}
+
TargetTransformInfo::OperandValueInfo
TargetTransformInfo::getOperandInfo(const Value *V) {
OperandValueKind OpInfo = OK_AnyValue;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 730bec428a38e..29587722214ad 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5192,6 +5192,17 @@ unsigned AArch64TTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
return ST->getMaxInterleaveFactor();
}
+bool AArch64TTIImpl::supportsVectorInterleaveDeinterleaveIntrinsics(
+ unsigned Factor, VectorType *VecTy) const {
+ if (VecTy->isScalableTy()) {
+ assert(Factor <= 8 && "Scalable vector interleave/deinterleave intrinsics "
+ "only support factors up to 8");
+ return Factor <= 8;
+ }
+ // For Fixed length vectors.
+ return Factor <= 4;
+}
+
// For Falkor, we want to avoid having too many strided loads in a loop since
// that can exhaust the HW prefetcher resources. We adjust the unroller
// MaxCount preference below to attempt to ensure unrolling doesn't create too
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 200b4c9fba196..85faafc3321e3 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -175,6 +175,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
unsigned getMaxInterleaveFactor(ElementCount VF) const override;
+ bool supportsVectorInterleaveDeinterleaveIntrinsics(
+ unsigned Factor, VectorType *VecTy) const override;
+
bool prefersVectorizedAddressing() const override;
/// Check whether Opcode1 has less throughput according to the scheduling
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d841f54fa58a8..0bc374244220e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -4134,7 +4134,10 @@ static Value *createBitOrPointerCast(IRBuilderBase &Builder, Value *V,
/// Return a vector containing interleaved elements from multiple
/// smaller input vectors.
-static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
+static Value *interleaveVectors(IRBuilderBase &Builder,
+ const TargetTransformInfo &TTI,
+ ArrayRef<Value *> Vals,
+
const Twine &Name) {
unsigned Factor = Vals.size();
assert(Factor > 1 && "Tried to interleave invalid number of vectors");
@@ -4145,14 +4148,11 @@ static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
assert(Val->getType() == VecTy && "Tried to interleave mismatched types");
#endif
- // Scalable vectors cannot use arbitrary shufflevectors (only splats), so
- // must use intrinsics to interleave.
- if (VecTy->isScalableTy()) {
- assert(Factor <= 8 && "Unsupported interleave factor for scalable vectors");
+ if (TTI.supportsVectorInterleaveDeinterleaveIntrinsics(Factor, VecTy))
return Builder.CreateVectorInterleave(Vals, Name);
- }
- // Fixed length. Start by concatenating all vectors into a wide vector.
+ // Unsupported Fixed length intrinsics. Start by concatenating all vectors
+ // into a wide vector.
Value *WideVec = concatenateVectors(Builder, Vals);
// Interleave the elements into the wide vector.
@@ -4172,9 +4172,10 @@ static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
// }
// To:
// %wide.vec = load <12 x i32> ; Read 4 tuples of R,G,B
-// %R.vec = shuffle %wide.vec, poison, <0, 3, 6, 9> ; R elements
-// %G.vec = shuffle %wide.vec, poison, <1, 4, 7, 10> ; G elements
-// %B.vec = shuffle %wide.vec, poison, <2, 5, 8, 11> ; B elements
+// %deint = call {<4 x i32>, <4 x i32>, <4 x i32>}
+// @llvm.vector.deinterleave3(<12 x i32> %wide.vec) %R.vec = extractvalue
+// %deint, 0 ; R elements %G.vec = extractvalue %deint, 1 ; G
+// elements %B.vec = extractvalue %deint, 2 ; B elements
//
// Or translate following interleaved store group (factor = 3):
// for (i = 0; i < N; i+=3) {
@@ -4184,11 +4185,9 @@ static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
// Pic[i+2] = B; // Member of index 2
// }
// To:
-// %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7>
-// %B_U.vec = shuffle %B.vec, poison, <0, 1, 2, 3, u, u, u, u>
-// %interleaved.vec = shuffle %R_G.vec, %B_U.vec,
-// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> ; Interleave R,G,B elements
-// store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B
+// %interleaved.vec = call <12 x i32> @llvm.vector.interleave3(<4 x i32>
+// %R.vec, <4 x i32> %G.vec, <4 x i32> %B.vec) store <12 x i32>
+// %interleaved.vec ; Write 4 tuples of R,G,B
void VPInterleaveRecipe::execute(VPTransformState &State) {
assert((!needsMaskForGaps() || !State.VF.isScalable()) &&
"Masking gaps for scalable vectors is not yet supported.");
@@ -4204,15 +4203,19 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
VPValue *Addr = getAddr();
Value *ResAddr = State.get(Addr, VPLane(0));
- auto CreateGroupMask = [&BlockInMask, &State,
- &InterleaveFactor](Value *MaskForGaps) -> Value * {
- if (State.VF.isScalable()) {
+ bool UseIntrinsics =
+ State.TTI->supportsVectorInterleaveDeinterleaveIntrinsics(
+ InterleaveFactor, VecTy);
+ auto CreateGroupMask = [&BlockInMask, &State, &InterleaveFactor,
+ &UseIntrinsics](Value *MaskForGaps) -> Value * {
+ if (UseIntrinsics) {
assert(!MaskForGaps && "Interleaved groups with gaps are not supported.");
assert(InterleaveFactor <= 8 &&
"Unsupported deinterleave factor for scalable vectors");
auto *ResBlockInMask = State.get(BlockInMask);
SmallVector<Value *> Ops(InterleaveFactor, ResBlockInMask);
- return interleaveVectors(State.Builder, Ops, "interleaved.mask");
+ return interleaveVectors(State.Builder, *State.TTI, Ops,
+ "interleaved.mask");
}
if (!BlockInMask)
@@ -4253,25 +4256,24 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
Group->addMetadata(NewLoad);
ArrayRef<VPRecipeValue *> VPDefs = definedValues();
- if (VecTy->isScalableTy()) {
- // Scalable vectors cannot use arbitrary shufflevectors (only splats),
- // so must use intrinsics to deinterleave.
- assert(InterleaveFactor <= 8 &&
- "Unsupported deinterleave factor for scalable vectors");
+ bool UseIntrinsics =
+ State.TTI->supportsVectorInterleaveDeinterleaveIntrinsics(
+ InterleaveFactor, VecTy);
+ if (UseIntrinsics) {
NewLoad = State.Builder.CreateIntrinsic(
Intrinsic::getDeinterleaveIntrinsicID(InterleaveFactor),
NewLoad->getType(), NewLoad,
/*FMFSource=*/nullptr, "strided.vec");
}
- auto CreateStridedVector = [&InterleaveFactor, &State,
- &NewLoad](unsigned Index) -> Value * {
+ auto CreateStridedVector = [&InterleaveFactor, &State, &NewLoad,
+ &UseIntrinsics](unsigned Index) -> Value * {
assert(Index < InterleaveFactor && "Illegal group index");
- if (State.VF.isScalable())
+ if (UseIntrinsics)
return State.Builder.CreateExtractValue(NewLoad, Index);
- // For fixed length VF, use shuffle to extract the sub-vectors from the
- // wide load.
+ // For fixed length VF with unsupported Interleave Factor,
+ // use shuffle to extract the sub-vectors from the wide load.
auto StrideMask =
createStrideMask(Index, InterleaveFactor, State.VF.getFixedValue());
return State.Builder.CreateShuffleVector(NewLoad, StrideMask,
@@ -4342,7 +4344,8 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
}
// Interleave all the smaller vectors into one wider vector.
- Value *IVec = interleaveVectors(State.Builder, StoredVecs, "interleaved.vec");
+ Value *IVec = interleaveVectors(State.Builder, *State.TTI, StoredVecs,
+ "interleaved.vec");
Instruction *NewStoreInstr;
if (BlockInMask || MaskForGaps) {
Value *GroupMask = CreateGroupMask(MaskForGaps);
@@ -4416,7 +4419,8 @@ void VPInterleaveEVLRecipe::execute(VPTransformState &State) {
Value *GroupMask = nullptr;
if (VPValue *BlockInMask = getMask()) {
SmallVector<Value *> Ops(InterleaveFactor, State.get(BlockInMask));
- GroupMask = interleaveVectors(State.Builder, Ops, "interleaved.mask");
+ GroupMask =
+ interleaveVectors(State.Builder, *State.TTI, Ops, "interleaved.mask");
} else {
GroupMask =
State.Builder.CreateVectorSplat(WideVF, State.Builder.getTrue());
@@ -4487,7 +4491,8 @@ void VPInterleaveEVLRecipe::execute(VPTransformState &State) {
}
// Interleave all the smaller vectors into one wider vector.
- Value *IVec = interleaveVectors(State.Builder, StoredVecs, "interleaved.vec");
+ Value *IVec = interleaveVectors(State.Builder, *State.TTI, StoredVecs,
+ "interleaved.vec");
CallInst *NewStore =
State.Builder.CreateIntrinsic(Type::getVoidTy(Ctx), Intrinsic::vp_store,
{IVec, ResAddr, GroupMask, InterleaveEVL});
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll b/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
index 66b800c850892..9894e91a485df 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S < %s -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=4 | FileCheck %s
; RUN: opt -S < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 | FileCheck %s --check-prefix=FORCE-VEC
@@ -103,11 +104,13 @@ for.end:
; CHECK-LABEL: @ptr_ind_plus2(
; CHECK: %[[V0:.*]] = load <8 x i32>
-; CHECK: shufflevector <8 x i32> %[[V0]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK: shufflevector <8 x i32> %[[V0]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: %[[DEINTV0:.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %[[V0]])
+; CHECK: %[[TMP2:.*]] = extractvalue { <4 x i32>, <4 x i32> } %[[DEINTV0]], 0
+; CHECK: %[[TMP3:.*]] = extractvalue { <4 x i32>, <4 x i32> } %[[DEINTV0]], 1
; CHECK: %[[V1:.*]] = load <8 x i32>
-; CHECK: shufflevector <8 x i32> %[[V1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK: shufflevector <8 x i32> %[[V1]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: %[[DEINTV1:.*]] = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %[[V1]])
+; CHECK: %[[TMP4:.*]] = extractvalue { <4 x i32>, <4 x i32> } %[[DEINTV1]], 0
+; CHECK: %[[TMP5:.*]] = extractvalue { <4 x i32>, <4 x i32> } %[[DEINTV1]], 1
; CHECK: mul nsw <4 x i32>
; CHECK: mul nsw <4 x i32>
; CHECK: add <4 x i32>
@@ -117,8 +120,9 @@ for.end:
; FORCE-VEC-LABEL: @ptr_ind_plus2(
; FORCE-VEC: %[[V:.*]] = load <4 x i32>
-; FORCE-VEC: shufflevector <4 x i32> %[[V]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
-; FORCE-VEC: shufflevector <4 x i32> %[[V]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+; FORCE-VEC: %[[DEINTV:.*]] = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> %[[V]])
+; FORCE-VEC: %[[TMP1:.*]] = extractvalue { <2 x i32>, <2 x i32> } %[[DEINTV]], 0
+; FORCE-VEC: %[[TMP2:.*]] = extractvalue { <2 x i32>, <2 x i32> } %[[DEINTV]], 1
; FORCE-VEC: mul nsw <2 x i32>
; FORCE-VEC: add <2 x i32>
; FORCE-VEC: %index.next = add nuw i64 %index, 2
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 31a1d95dedd3c..c5f4f131761ca 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -449,8 +449,10 @@ define void @interleave_group(ptr %dst) #1 {
; COST1-NEXT: [[TMP2:%.*]] = mul i64 [[TMP0]], 3
; COST1-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; COST1-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP3]], align 1
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP4]], align 1
+; COST1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <48 x i8> @llvm.vector.interleave3.v48i8(<16 x i8> zeroinitializer, <16 x i8> zeroinitializer, <16 x i8> zeroinitializer)
+; COST1-NEXT: store <48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 1
+; COST1-NEXT: [[INTERLEAVED_VEC1:%.*]] = call <48 x i8> @llvm.vector.interleave3.v48i8(<16 x i8> zeroinitializer, <16 x i8> zeroinitializer, <16 x i8> zeroinitializer)
+; COST1-NEXT: store <48 x i8> [[INTERLEAVED_VEC1]], ptr [[TMP4]], align 1
; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; COST1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; COST1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
@@ -516,7 +518,8 @@ define void @interleave_group(ptr %dst) #1 {
; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COST10-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 3
; COST10-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
-; COST10-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP1]], align 1
+; COST10-NEXT: [[INTERLEAVED_VEC:%.*]] = call <48 x i8> @llvm.vector.interleave3.v48i8(<16 x i8> zeroinitializer, <16 x i8> zeroinitializer, <16 x i8> zeroinitializer)
+; COST10-NEXT: store <48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 1
; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; COST10-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; COST10-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index 9e8a95b6b9a47..661adf4d9f5d6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -643,17 +643,13 @@ define void at sext_sub_nsw_for_address(ptr %base, i64 %n, ptr %src) #0 {
; CHECK-NEXT: [[TMP76:%.*]] = load double, ptr [[TMP62]], align 8, !alias.scope [[META17]]
; CHECK-NEXT: [[TMP77:%.*]] = insertelement <2 x double> poison, double [[TMP75]], i32 0
; CHECK-NEXT: [[TMP78:%.*]] = insertelement <2 x double> [[TMP77]], double [[TMP76]], i32 1
-; CHECK-NEXT: [[TMP79:%.*]] = shufflevector <2 x double> zeroinitializer, <2 x double> [[TMP66]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevect...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/202583
More information about the llvm-commits
mailing list