[llvm] 43f0f1a - [VPlan] Use onlyFirstLaneUsed in sinkScalarOperands.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 29 11:46:03 PDT 2022
Author: Florian Hahn
Date: 2022-10-29T19:45:19+01:00
New Revision: 43f0f1a66ff074628305f5d2fc11ce5dd4edef53
URL: https://github.com/llvm/llvm-project/commit/43f0f1a66ff074628305f5d2fc11ce5dd4edef53
DIFF: https://github.com/llvm/llvm-project/commit/43f0f1a66ff074628305f5d2fc11ce5dd4edef53.diff
LOG: [VPlan] Use onlyFirstLaneUsed in sinkScalarOperands.
Replace custom code to check if only the first lane is used by generic
helper `onlyFirstLaneUsed`. This enables VPlan-based sinking in a few
additional cases and was suggested in D133760.
Reviewed By: Ayal
Differential Revision: https://reviews.llvm.org/D136368
Added:
Modified:
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a2053aa732e2c..ca9b57e81b436 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -138,8 +138,7 @@ bool VPlanTransforms::sinkScalarOperands(VPlan &Plan) {
// All recipe users of the sink candidate must be in the same block SinkTo
// or all users outside of SinkTo must be uniform-after-vectorization (
// i.e., only first lane is used) . In the latter case, we need to duplicate
- // SinkCandidate. At the moment, we identify such UAV's by looking for the
- // address operands of widened memory recipes.
+ // SinkCandidate.
auto CanSinkWithUser = [SinkTo, &NeedsDuplicating,
SinkCandidate](VPUser *U) {
auto *UI = dyn_cast<VPRecipeBase>(U);
@@ -147,12 +146,8 @@ bool VPlanTransforms::sinkScalarOperands(VPlan &Plan) {
return false;
if (UI->getParent() == SinkTo)
return true;
- auto *WidenI = dyn_cast<VPWidenMemoryInstructionRecipe>(UI);
- if (WidenI && WidenI->getAddr() == SinkCandidate) {
- NeedsDuplicating = true;
- return true;
- }
- return false;
+ NeedsDuplicating = UI->onlyFirstLaneUsed(SinkCandidate);
+ return NeedsDuplicating;
};
if (!all_of(SinkCandidate->users(), CanSinkWithUser))
continue;
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
index fdec1d4ca4d53..4e309241336ec 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
@@ -38,8 +38,9 @@ define void @interleaved_with_cond_store_0(%pair *%p, i64 %x, i64 %n) {
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK: pred.store.if:
+; CHECK-NEXT: [[TMP2_1:%.*]] = getelementptr inbounds [[PAIR:%.*]], %pair* [[P:%.*]], i64 [[INDEX]], i32 1
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 0
-; CHECK-NEXT: store i64 [[TMP6]], i64* [[TMP2]], align 8
+; CHECK-NEXT: store i64 [[TMP6]], i64* [[TMP2_1]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]
; CHECK: pred.store.continue:
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1
@@ -136,8 +137,9 @@ define void @interleaved_with_cond_store_1(%pair *%p, i64 %x, i64 %n) {
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i64 0
; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK: pred.store.if:
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr inbounds [[PAIR:%.*]], %pair* [[P:%.*]], i64 [[INDEX]], i32 0
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 0
-; CHECK-NEXT: store i64 [[TMP9]], i64* [[TMP3]], align 8
+; CHECK-NEXT: store i64 [[TMP9]], i64* [[PTR0]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]
; CHECK: pred.store.continue:
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP7]], i64 1
@@ -246,8 +248,9 @@ define void @interleaved_with_cond_store_2(%pair *%p, i64 %x, i64 %n) {
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i64 0
; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK: pred.store.if:
+; CHECK-NEXT: [[PTR1:%.*]] = getelementptr inbounds [[PAIR]], %pair* [[P]], i64 [[INDEX]], i32 1
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 0
-; CHECK-NEXT: store i64 [[TMP9]], i64* [[TMP5]], align 8
+; CHECK-NEXT: store i64 [[TMP9]], i64* [[PTR1]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]
; CHECK: pred.store.continue:
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP7]], i64 1
More information about the llvm-commits
mailing list