[llvm] 8a47e62 - [VPlan] Re-add interleave group members to plan.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 5 07:07:09 PST 2021
Author: Florian Hahn
Date: 2021-01-05T15:06:47Z
New Revision: 8a47e6252ad43a2eb3238f9b36394571ba13f4a9
URL: https://github.com/llvm/llvm-project/commit/8a47e6252ad43a2eb3238f9b36394571ba13f4a9
DIFF: https://github.com/llvm/llvm-project/commit/8a47e6252ad43a2eb3238f9b36394571ba13f4a9.diff
LOG: [VPlan] Re-add interleave group members to plan.
Creating in-loop reductions relies on IR references to map
IR values to VPValues after interleave group creation.
Make sure we re-add the updated member to the plan, so the look-ups
still work as expected
This fixes a crash reported after D90562.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7f89fd9a1349..2bd35cca143e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8520,6 +8520,7 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
if (!Member->getType()->isVoidTy()) {
VPValue *OriginalV = Plan->getVPValue(Member);
Plan->removeVPValueFor(Member);
+ Plan->addVPValue(Member, VPIG->getVPValue(J));
OriginalV->replaceAllUsesWith(VPIG->getVPValue(J));
J++;
}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
index fad08b91259b..0628aa888a24 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
@@ -950,4 +950,85 @@ for.cond.cleanup: ; preds = %for.body, %entry
ret i8 %r.0.lcssa
}
+; Make sure interleave group members feeding in-loop reductions can be handled.
+define i32 @reduction_interleave_group(i32 %n, i32* %arr) #0 {
+; CHECK-LABEL: @reduction_interleave_group(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[GUARD:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[GUARD]], label [[FOR_BODY_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK: for.body.preheader:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw i32 [[TMP1]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 6
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[TMP2]], -4
+; CHECK-NEXT: [[IND_END:%.*]] = shl i32 [[N_VEC]], 1
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = or i32 [[OFFSET_IDX]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i32 -1
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <8 x i32>*
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP6]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC1]])
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STRIDED_VEC]])
+; CHECK-NEXT: [[TMP10]] = add i32 [[TMP9]], [[TMP8]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP13:!llvm.loop !.*]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT: [[RED_PHI:%.*]] = phi i32 [ [[RED_2:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT: [[ADD:%.*]] = or i32 [[IV]], 1
+; CHECK-NEXT: [[GEP_0:%.*]] = getelementptr inbounds i32, i32* [[ARR]], i32 [[ADD]]
+; CHECK-NEXT: [[L_0:%.*]] = load i32, i32* [[GEP_0]], align 4
+; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i32, i32* [[ARR]], i32 [[IV]]
+; CHECK-NEXT: [[L_1:%.*]] = load i32, i32* [[GEP_1]], align 4
+; CHECK-NEXT: [[RED_1:%.*]] = add i32 [[L_0]], [[RED_PHI]]
+; CHECK-NEXT: [[RED_2]] = add i32 [[RED_1]], [[L_1]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 2
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], [[LOOP14:!llvm.loop !.*]]
+; CHECK: exit:
+; CHECK-NEXT: [[RET_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[RED_2]], [[FOR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[RET_LCSSA]]
+;
+entry:
+ %guard = icmp sgt i32 %n, 0
+ br i1 %guard , label %for.body, label %exit
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
+ %red.phi = phi i32 [ %red.2, %for.body ], [ 0, %entry ]
+ %add = or i32 %iv, 1
+ %gep.0 = getelementptr inbounds i32, i32* %arr, i32 %add
+ %l.0 = load i32, i32* %gep.0, align 4
+ %gep.1 = getelementptr inbounds i32, i32* %arr, i32 %iv
+ %l.1 = load i32, i32* %gep.1, align 4
+ %red.1 = add i32 %l.0, %red.phi
+ %red.2 = add i32 %red.1, %l.1
+ %iv.next = add nuw nsw i32 %iv, 2
+ %cmp = icmp slt i32 %iv.next, %n
+ br i1 %cmp, label %for.body, label %exit
+
+exit:
+ %ret.lcssa = phi i32 [ 0, %entry ], [ %red.2, %for.body ]
+ ret i32 %ret.lcssa
+}
+
attributes #0 = { "target-features"="+mve" }
More information about the llvm-commits
mailing list