[llvm] 114e208 - [LV] Fix sub-reduction PHI in vectorized epilogue (#182072)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 19 02:15:37 PST 2026
Author: Sander de Smalen
Date: 2026-02-19T10:15:32Z
New Revision: 114e20805f028821b41666cca563d6f81a075780
URL: https://github.com/llvm/llvm-project/commit/114e20805f028821b41666cca563d6f81a075780
DIFF: https://github.com/llvm/llvm-project/commit/114e20805f028821b41666cca563d6f81a075780.diff
LOG: [LV] Fix sub-reduction PHI in vectorized epilogue (#182072)
When the vectorized epilogue loop uses partial reductions, the PHI node
in the loop must start at 0 (because for partial sub-reductions the
sub is done in the middle block) and the compute-reduction-result must
subtract from the partial result (as calculated in the middle block of
the main vector loop), instead of subtracting from the original init
value.
This fixes the issue as reported on #178919 by @aeubanks.
Added:
llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2c2a4545b66db..9538f8cf94f47 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9234,7 +9234,23 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
if (auto *VPI = dyn_cast<VPInstruction>(PhiR->getStartValue())) {
assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&
"unexpected start value");
- VPI->setOperand(0, StartVal);
+ // Partial sub-reductions always start at 0 and account for the
+ // reduction start value in a final subtraction. Update it to use the
+ // resume value from the main vector loop.
+ if (PhiR->getVFScaleFactor() > 1 &&
+ PhiR->getRecurrenceKind() == RecurKind::Sub) {
+ auto *Sub = cast<VPInstruction>(RdxResult->getSingleUser());
+ assert(Sub->getOpcode() == Instruction::Sub && "Unexpected opcode");
+ assert(isa<VPIRValue>(Sub->getOperand(0)) &&
+ "Expected operand to match the original start value of the "
+ "reduction");
+ assert(VPlanPatternMatch::match(VPI->getOperand(0),
+ VPlanPatternMatch::m_ZeroInt()) &&
+ "Expected start value for partial sub-reduction to start at "
+ "zero");
+ Sub->setOperand(0, StartVal);
+ } else
+ VPI->setOperand(0, StartVal);
continue;
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
new file mode 100644
index 0000000000000..a876b4553bcf7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
@@ -0,0 +1,188 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=true -epilogue-vectorization-force-VF=4 -vectorizer-maximize-bandwidth -S < %s | FileCheck %s --check-prefix=CHECK-EPI
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=true -epilogue-vectorization-force-VF=8 -vectorizer-maximize-bandwidth -S < %s | FileCheck %s --check-prefix=CHECK-PARTIAL-RED-EPI
+
+target triple = "aarch64"
+
+define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
+; CHECK-EPI-LABEL: define i32 @sub_reduction(
+; CHECK-EPI-SAME: i32 [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-EPI-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-EPI-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-EPI-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
+; CHECK-EPI-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 39, [[TMP1]]
+; CHECK-EPI-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-EPI: [[VECTOR_PH]]:
+; CHECK-EPI-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-EPI-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
+; CHECK-EPI-NEXT: [[N_MOD_VF:%.*]] = urem i32 39, [[TMP3]]
+; CHECK-EPI-NEXT: [[N_VEC:%.*]] = sub i32 39, [[N_MOD_VF]]
+; CHECK-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-EPI: [[VECTOR_BODY]]:
+; CHECK-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
+; CHECK-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
+; CHECK-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-EPI-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
+; CHECK-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP5]], [[TMP6]]
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
+; CHECK-EPI-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-EPI-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-EPI: [[MIDDLE_BLOCK]]:
+; CHECK-EPI-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-EPI-NEXT: [[TMP10:%.*]] = sub i32 [[STARTVAL]], [[TMP9]]
+; CHECK-EPI-NEXT: [[CMP_N:%.*]] = icmp eq i32 39, [[N_VEC]]
+; CHECK-EPI-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-EPI: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-EPI-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[N_MOD_VF]], 4
+; CHECK-EPI-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK-EPI: [[VEC_EPILOG_PH]]:
+; CHECK-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-EPI-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ [[TMP11]], %[[VEC_EPILOG_PH]] ], [ [[TMP16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX2]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP12]], align 4
+; CHECK-EPI-NEXT: [[TMP13:%.*]] = sext <4 x i8> [[WIDE_LOAD4]] to <4 x i32>
+; CHECK-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP12]], align 4
+; CHECK-EPI-NEXT: [[TMP14:%.*]] = sext <4 x i8> [[WIDE_LOAD5]] to <4 x i32>
+; CHECK-EPI-NEXT: [[TMP15:%.*]] = mul <4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-EPI-NEXT: [[TMP16]] = sub <4 x i32> [[VEC_PHI3]], [[TMP15]]
+; CHECK-EPI-NEXT: [[INDEX_NEXT6]] = add nuw i32 [[INDEX2]], 4
+; CHECK-EPI-NEXT: [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT6]], 36
+; CHECK-EPI-NEXT: br i1 [[TMP17]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-EPI-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP16]])
+; CHECK-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-EPI: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX7:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: br label %[[LOOP:.*]]
+; CHECK-EPI: [[LOOP]]:
+; CHECK-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX7]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[IV]]
+; CHECK-EPI-NEXT: [[SRC1_LOAD:%.*]] = load i8, ptr [[SRC1_GEP]], align 4
+; CHECK-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = sext i8 [[SRC1_LOAD]] to i32
+; CHECK-EPI-NEXT: [[SRC2_GEP:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[IV]]
+; CHECK-EPI-NEXT: [[SRC2_LOAD:%.*]] = load i8, ptr [[SRC2_GEP]], align 4
+; CHECK-EPI-NEXT: [[SRC2_LOAD_EXT:%.*]] = sext i8 [[SRC2_LOAD]] to i32
+; CHECK-EPI-NEXT: [[MUL:%.*]] = mul i32 [[SRC1_LOAD_EXT]], [[SRC2_LOAD_EXT]]
+; CHECK-EPI-NEXT: [[SUB]] = sub i32 [[ACCUM]], [[MUL]]
+; CHECK-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 38
+; CHECK-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-EPI: [[EXIT]]:
+; CHECK-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi i32 [ [[SUB]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-EPI-NEXT: ret i32 [[SUB_LCSSA]]
+;
+; CHECK-PARTIAL-RED-EPI-LABEL: define i32 @sub_reduction(
+; CHECK-PARTIAL-RED-EPI-SAME: i32 [[STARTVAL:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 39, [[TMP1]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[N_MOD_VF:%.*]] = urem i32 39, [[TMP3]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[N_VEC:%.*]] = sub i32 39, [[N_MOD_VF]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_BODY]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP5]], [[TMP6]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[MIDDLE_BLOCK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP10:%.*]] = sub i32 [[STARTVAL]], [[TMP9]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[CMP_N:%.*]] = icmp eq i32 39, [[N_VEC]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[N_MOD_VF]], 8
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX2]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP12]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x i8>, ptr [[TMP12]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = sext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP14:%.*]] = sext <8 x i8> [[WIDE_LOAD5]] to <8 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = mul <8 x i32> [[TMP13]], [[TMP14]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE6]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI3]], <8 x i32> [[TMP15]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT7]] = add nuw i32 [[INDEX2]], 8
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT7]], 32
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE6]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP18:%.*]] = sub i32 [[BC_MERGE_RDX]], [[TMP17]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 32, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX8:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[STARTVAL]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[LOOP:.*]]
+; CHECK-PARTIAL-RED-EPI: [[LOOP]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[IV]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD:%.*]] = load i8, ptr [[SRC1_GEP]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = sext i8 [[SRC1_LOAD]] to i32
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_GEP:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[IV]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_LOAD:%.*]] = load i8, ptr [[SRC2_GEP]], align 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC2_LOAD_EXT:%.*]] = sext i8 [[SRC2_LOAD]] to i32
+; CHECK-PARTIAL-RED-EPI-NEXT: [[MUL:%.*]] = mul i32 [[SRC1_LOAD_EXT]], [[SRC2_LOAD_EXT]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB]] = sub i32 [[ACCUM]], [[MUL]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-PARTIAL-RED-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV]], 38
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[EXIT]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SUB_LCSSA:%.*]] = phi i32 [ [[SUB]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: ret i32 [[SUB_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %accum = phi i32 [ %startval, %entry ], [ %sub, %loop ]
+ %src1.gep = getelementptr i8, ptr %src1, i32 %iv
+ %src1.load = load i8, ptr %src1.gep, align 4
+ %src1.load.ext = sext i8 %src1.load to i32
+ %src2.gep = getelementptr i8, ptr %src1, i32 %iv
+ %src2.load = load i8, ptr %src2.gep, align 4
+ %src2.load.ext = sext i8 %src2.load to i32
+ %mul = mul i32 %src1.load.ext, %src2.load.ext
+ %sub = sub i32 %accum, %mul
+ %iv.next = add i32 %iv, 1
+ %exitcond.not = icmp eq i32 %iv, 38
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ ret i32 %sub
+}
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
More information about the llvm-commits
mailing list