[llvm] Enable LoopVectorizer interleaving for vectorized loops. (PR #184306)
Jonas Paulsson via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 23 03:15:37 PDT 2026
https://github.com/JonPsson1 updated https://github.com/llvm/llvm-project/pull/184306
>From 7becfcc8b0dc79490fc7c04fd780d65817f07e03 Mon Sep 17 00:00:00 2001
From: Jonas Paulsson <Jonas.Paulsson2 at ibm.com>
Date: Tue, 3 Mar 2026 10:32:52 +0100
Subject: [PATCH] Enable LV interleaving for vectorized loops.
---
.../SystemZ/SystemZTargetTransformInfo.cpp | 4 +
.../SystemZ/SystemZTargetTransformInfo.h | 2 +
.../LoopVectorize/SystemZ/addressing.ll | 96 ++++++++++++++++++-
.../SystemZ/vectorized-epilogue-loop.ll | 30 ++++--
4 files changed, 121 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 7ac19761341de..676254ae92e5e 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -478,6 +478,10 @@ unsigned SystemZTTIImpl::getMinPrefetchStride(unsigned NumMemAccesses,
return ST->hasMiscellaneousExtensions3() ? 8192 : 2048;
}
+unsigned SystemZTTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
+ return VF.isVector() ? 8 : 1;
+}
+
bool SystemZTTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) const {
EVT VT = TLI->getValueType(DL, DataType);
return (VT.isScalarInteger() && TLI->isTypeLegal(VT));
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index b7b63d232f3f7..8baa371cf47a7 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -82,6 +82,8 @@ class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> {
bool HasCall) const override;
bool enableWritePrefetching() const override { return true; }
+ unsigned getMaxInterleaveFactor(ElementCount VF) const override;
+
bool hasDivRemOp(Type *DataType, bool IsSigned) const override;
bool prefersVectorizedAddressing() const override { return false; }
bool LSRWithInstrQueries() const override { return true; }
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/addressing.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/addressing.ll
index aa4ed92abec55..7c36a9cdd2ff3 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/addressing.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/addressing.ll
@@ -19,9 +19,65 @@ define void @foo(ptr nocapture %A) {
; CHECK-NEXT: [[DOTIDX1:%.*]] = shl i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX1]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i64 16
+; CHECK-NEXT: [[DOTIDX2:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX2]]
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[TMP31]], i64 32
+; CHECK-NEXT: [[DOTIDX3:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX3]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i64 48
+; CHECK-NEXT: [[DOTIDX4:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX4]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i64 64
+; CHECK-NEXT: [[DOTIDX5:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX5]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 80
+; CHECK-NEXT: [[DOTIDX6:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX6]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP11]], i64 96
+; CHECK-NEXT: [[DOTIDX7:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX7]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP13]], i64 112
+; CHECK-NEXT: [[DOTIDX8:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX8]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP15]], i64 128
+; CHECK-NEXT: [[DOTIDX9:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX9]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[TMP17]], i64 144
+; CHECK-NEXT: [[DOTIDX10:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX10]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[TMP19]], i64 160
+; CHECK-NEXT: [[DOTIDX11:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX11]]
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[TMP21]], i64 176
+; CHECK-NEXT: [[DOTIDX12:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX12]]
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP23]], i64 192
+; CHECK-NEXT: [[DOTIDX13:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX13]]
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[TMP25]], i64 208
+; CHECK-NEXT: [[DOTIDX14:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX14]]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[TMP27]], i64 224
+; CHECK-NEXT: [[DOTIDX15:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[A]], i64 [[DOTIDX15]]
+; CHECK-NEXT: [[TMP30:%.*]] = getelementptr i8, ptr [[TMP29]], i64 240
; CHECK-NEXT: store i32 4, ptr [[TMP2]], align 4
; CHECK-NEXT: store i32 4, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: store i32 4, ptr [[TMP32]], align 4
+; CHECK-NEXT: store i32 4, ptr [[TMP6]], align 4
+; CHECK-NEXT: store i32 4, ptr [[TMP8]], align 4
+; CHECK-NEXT: store i32 4, ptr [[TMP10]], align 4
+; CHECK-NEXT: store i32 4, ptr [[TMP12]], align 4
+; CHECK-NEXT: store i32 4, ptr [[TMP14]], align 4
+; CHECK-NEXT: store i32 4, ptr [[TMP16]], align 4
+; CHECK-NEXT: store i32 4, ptr [[TMP18]], align 4
+; CHECK-NEXT: store i32 4, ptr [[TMP20]], align 4
+; CHECK-NEXT: store i32 4, ptr [[TMP22]], align 4
+; CHECK-NEXT: store i32 4, ptr [[TMP24]], align 4
+; CHECK-NEXT: store i32 4, ptr [[TMP26]], align 4
+; CHECK-NEXT: store i32 4, ptr [[TMP28]], align 4
+; CHECK-NEXT: store i32 4, ptr [[TMP30]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
@@ -60,15 +116,51 @@ define void @foo1(ptr nocapture noalias %A, ptr nocapture %PtrPtr) {
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [8 x i8], ptr [[PTRPTR:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr [8 x i8], ptr [[PTRPTR]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP11]], i64 8
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr [8 x i8], ptr [[PTRPTR]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP15]], i64 16
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr [8 x i8], ptr [[PTRPTR]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP23]], i64 24
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr [8 x i8], ptr [[PTRPTR]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[TMP25]], i64 32
+; CHECK-NEXT: [[TMP39:%.*]] = getelementptr [8 x i8], ptr [[PTRPTR]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP43:%.*]] = getelementptr i8, ptr [[TMP39]], i64 40
+; CHECK-NEXT: [[TMP44:%.*]] = getelementptr [8 x i8], ptr [[PTRPTR]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP44]], i64 48
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr [8 x i8], ptr [[PTRPTR]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP13]], i64 56
; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP1]], align 8
; CHECK-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[TMP17:%.*]] = load ptr, ptr [[TMP16]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = load ptr, ptr [[TMP24]], align 8
+; CHECK-NEXT: [[TMP19:%.*]] = load ptr, ptr [[TMP26]], align 8
+; CHECK-NEXT: [[TMP20:%.*]] = load ptr, ptr [[TMP43]], align 8
+; CHECK-NEXT: [[TMP21:%.*]] = load ptr, ptr [[TMP12]], align 8
+; CHECK-NEXT: [[TMP22:%.*]] = load ptr, ptr [[TMP14]], align 8
; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4
; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i64 0
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1
+; CHECK-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP17]], align 4
+; CHECK-NEXT: [[TMP28:%.*]] = load i32, ptr [[TMP18]], align 4
+; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> poison, i32 [[TMP27]], i64 0
+; CHECK-NEXT: [[TMP30:%.*]] = insertelement <2 x i32> [[TMP29]], i32 [[TMP28]], i64 1
+; CHECK-NEXT: [[TMP31:%.*]] = load i32, ptr [[TMP19]], align 4
+; CHECK-NEXT: [[TMP32:%.*]] = load i32, ptr [[TMP20]], align 4
+; CHECK-NEXT: [[TMP33:%.*]] = insertelement <2 x i32> poison, i32 [[TMP31]], i64 0
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <2 x i32> [[TMP33]], i32 [[TMP32]], i64 1
+; CHECK-NEXT: [[TMP35:%.*]] = load i32, ptr [[TMP21]], align 4
+; CHECK-NEXT: [[TMP36:%.*]] = load i32, ptr [[TMP22]], align 4
+; CHECK-NEXT: [[TMP37:%.*]] = insertelement <2 x i32> poison, i32 [[TMP35]], i64 0
+; CHECK-NEXT: [[TMP38:%.*]] = insertelement <2 x i32> [[TMP37]], i32 [[TMP36]], i64 1
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 8
+; CHECK-NEXT: [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 16
+; CHECK-NEXT: [[TMP42:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 24
; CHECK-NEXT: store <2 x i32> [[TMP8]], ptr [[TMP9]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: store <2 x i32> [[TMP30]], ptr [[TMP40]], align 4
+; CHECK-NEXT: store <2 x i32> [[TMP34]], ptr [[TMP41]], align 4
+; CHECK-NEXT: store <2 x i32> [[TMP38]], ptr [[TMP42]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/vectorized-epilogue-loop.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/vectorized-epilogue-loop.ll
index dcccb417d70e6..01ba4e4b459c2 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/vectorized-epilogue-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/vectorized-epilogue-loop.ll
@@ -10,42 +10,54 @@ define void @fun(ptr noalias %Src, ptr noalias %Dst, i64 %N) {
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ITER_CHECK:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK3:%.*]] = icmp ult i64 [[TMP0]], 16
+; CHECK-NEXT: [[MIN_ITERS_CHECK3:%.*]] = icmp ult i64 [[TMP0]], 64
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK3]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 16
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 64
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP2]], i64 16
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP2]], i64 32
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP2]], i64 48
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP3]], i64 16
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP3]], i64 32
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP3]], i64 48
; CHECK-NEXT: store <16 x i8> [[WIDE_LOAD]], ptr [[TMP3]], align 1
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: store <16 x i8> [[WIDE_LOAD2]], ptr [[TMP12]], align 1
+; CHECK-NEXT: store <16 x i8> [[WIDE_LOAD3]], ptr [[TMP13]], align 1
+; CHECK-NEXT: store <16 x i8> [[WIDE_LOAD4]], ptr [[TMP14]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], 4
+; CHECK-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], 8
; CHECK-NEXT: [[N_VEC5:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF4]]
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX6]]
-; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX6]]
-; CHECK-NEXT: store <4 x i8> [[WIDE_LOAD7]], ptr [[TMP6]], align 1
-; CHECK-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX6]], 4
+; CHECK-NEXT: store <8 x i8> [[WIDE_LOAD8]], ptr [[TMP6]], align 1
+; CHECK-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX6]], 8
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC5]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
More information about the llvm-commits
mailing list