[llvm] [VectorUtils][VPlan] Consolidate VPWidenIntrinsicRecipe::onlyFirstLaneUsed and isVectorIntrinsicWithScalarOpAtArg (PR #137497)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 30 03:18:10 PDT 2025
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/137497
>From ecc63e70ebb6edc26ef995015dd8aa0435d8a2e8 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 29 Apr 2025 23:16:44 +0800
Subject: [PATCH 1/5] Precommit test
---
.../LoopVectorize/widen-intrinsic.ll | 57 +++++++++++++++++++
1 file changed, 57 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
diff --git a/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll b/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
new file mode 100644
index 0000000000000..5b16051c836ce
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -S | FileCheck %s
+
+; Check that we don't unnecessarily broadcast %pow
+define void @powi(ptr noalias %p, i32 %pow) {
+; CHECK-LABEL: define void @powi(
+; CHECK-SAME: ptr noalias [[P:%.*]], i32 [[POW:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[POW]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[TMP0]], i32 0
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[BROADCAST_SPLAT]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[WIDE_LOAD]], i32 [[TMP2]])
+; CHECK-NEXT: store <4 x float> [[TMP3]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P_GEP:%.*]] = getelementptr float, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load float, ptr [[P_GEP]], align 4
+; CHECK-NEXT: [[Y:%.*]] = call float @llvm.powi.f32.i32(float [[X]], i32 [[POW]])
+; CHECK-NEXT: store float [[Y]], ptr [[P_GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i32 [[IV_NEXT]], 1024
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %p.gep = getelementptr float, ptr %p, i32 %iv
+ %x = load float, ptr %p.gep
+ %y = call float @llvm.powi(float %x, i32 %pow)
+ store float %y, ptr %p.gep
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 2cfab8b1248f8c0345c4d747d26d72a84d1527ab Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Sun, 27 Apr 2025 13:42:16 +0800
Subject: [PATCH 2/5] [VectorUtils][VPlan] Consolidate
VPWidenIntrinsicRecipe::onlyFirstLaneUsed and
isVectorIntrinsicWithScalarOpAtArg
We can reuse isVectorIntrinsicWithScalarOpAtArg in VectorUtils to determine if only the first lane will be used for a VPWidenIntrinsicRecipe, provided that we also move the VP EVL operand check into it.
This was needed by a local patch I was working on that created a VPWidenIntrinsicRecipe with a VP intrinsic, and prevents the need to update the scalar arguments in two places.
---
llvm/lib/Analysis/VectorUtils.cpp | 7 ++++++-
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 6 ++----
llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll | 5 +----
3 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 682066dc5af5f..907ec13a1b494 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -149,6 +149,11 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
if (TTI && Intrinsic::isTargetIntrinsic(ID))
return TTI->isTargetIntrinsicWithScalarOpAtArg(ID, ScalarOpdIdx);
+ // Vector predication intrinsics only demand the the first lane the last
+ // operand (the EVL operand).
+ if (VPIntrinsic::getVectorLengthParamPos(ID) == ScalarOpdIdx)
+ return true;
+
switch (ID) {
case Intrinsic::abs:
case Intrinsic::vp_abs:
@@ -166,7 +171,7 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
case Intrinsic::umul_fix_sat:
return (ScalarOpdIdx == 2);
case Intrinsic::experimental_vp_splice:
- return ScalarOpdIdx == 2 || ScalarOpdIdx == 4 || ScalarOpdIdx == 5;
+ return ScalarOpdIdx == 2 || ScalarOpdIdx == 4;
default:
return false;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 54990c7c806c4..f975f6628822a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1364,10 +1364,8 @@ StringRef VPWidenIntrinsicRecipe::getIntrinsicName() const {
bool VPWidenIntrinsicRecipe::onlyFirstLaneUsed(const VPValue *Op) const {
assert(is_contained(operands(), Op) && "Op must be an operand of the recipe");
- // Vector predication intrinsics only demand the the first lane the last
- // operand (the EVL operand).
- return VPIntrinsic::isVPIntrinsic(VectorIntrinsicID) &&
- Op == getOperand(getNumOperands() - 1);
+ unsigned Idx = std::distance(op_begin(), find(operands(), Op));
+ return isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, Idx, nullptr);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll b/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
index 5b16051c836ce..0ef0d806fe91a 100644
--- a/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
+++ b/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
@@ -8,16 +8,13 @@ define void @powi(ptr noalias %p, i32 %pow) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[POW]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i32 [[INDEX]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[TMP0]], i32 0
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[BROADCAST_SPLAT]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[WIDE_LOAD]], i32 [[TMP2]])
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[WIDE_LOAD]], i32 [[POW]])
; CHECK-NEXT: store <4 x float> [[TMP3]], ptr [[TMP1]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
>From 353adf097e492c73658ced05794d9c127c4d8b93 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 29 Apr 2025 23:17:50 +0800
Subject: [PATCH 3/5] Check all operands
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f975f6628822a..fcb27982e08e1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1364,8 +1364,13 @@ StringRef VPWidenIntrinsicRecipe::getIntrinsicName() const {
bool VPWidenIntrinsicRecipe::onlyFirstLaneUsed(const VPValue *Op) const {
assert(is_contained(operands(), Op) && "Op must be an operand of the recipe");
- unsigned Idx = std::distance(op_begin(), find(operands(), Op));
- return isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, Idx, nullptr);
+ for (auto [Idx, V] : enumerate(operands())) {
+ if (V != Op)
+ continue;
+ if (!isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, Idx, nullptr))
+ return false;
+ }
+ return true;
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
>From e525298faf52ce5bfb7460425c0025cbfdc65f0a Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 30 Apr 2025 10:00:10 +0800
Subject: [PATCH 4/5] Use all_of
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index fcb27982e08e1..281fe9a84b1f7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1364,13 +1364,11 @@ StringRef VPWidenIntrinsicRecipe::getIntrinsicName() const {
bool VPWidenIntrinsicRecipe::onlyFirstLaneUsed(const VPValue *Op) const {
assert(is_contained(operands(), Op) && "Op must be an operand of the recipe");
- for (auto [Idx, V] : enumerate(operands())) {
- if (V != Op)
- continue;
- if (!isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, Idx, nullptr))
- return false;
- }
- return true;
+ return all_of(enumerate(operands()), [this, &Op](auto &&X) {
+ auto [Idx, V] = X;
+ return V != Op || isVectorIntrinsicWithScalarOpAtArg(getVectorIntrinsicID(),
+ Idx, nullptr);
+ });
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
>From 162896ef88f026b56c4298c37225cd30e69c02f5 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 30 Apr 2025 18:17:46 +0800
Subject: [PATCH 5/5] Use const, update EVL comment, remove noalias, rename
test
---
llvm/lib/Analysis/VectorUtils.cpp | 3 +--
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +-
.../Transforms/LoopVectorize/widen-intrinsic.ll | 14 +++++++-------
3 files changed, 9 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 907ec13a1b494..968fd2f8c5d7f 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -149,8 +149,7 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
if (TTI && Intrinsic::isTargetIntrinsic(ID))
return TTI->isTargetIntrinsicWithScalarOpAtArg(ID, ScalarOpdIdx);
- // Vector predication intrinsics only demand the the first lane the last
- // operand (the EVL operand).
+ // Vector predication intrinsics have the EVL as the last operand.
if (VPIntrinsic::getVectorLengthParamPos(ID) == ScalarOpdIdx)
return true;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 281fe9a84b1f7..215aed1b0bc4e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1364,7 +1364,7 @@ StringRef VPWidenIntrinsicRecipe::getIntrinsicName() const {
bool VPWidenIntrinsicRecipe::onlyFirstLaneUsed(const VPValue *Op) const {
assert(is_contained(operands(), Op) && "Op must be an operand of the recipe");
- return all_of(enumerate(operands()), [this, &Op](auto &&X) {
+ return all_of(enumerate(operands()), [this, &Op](const auto &X) {
auto [Idx, V] = X;
return V != Op || isVectorIntrinsicWithScalarOpAtArg(getVectorIntrinsicID(),
Idx, nullptr);
diff --git a/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll b/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
index 0ef0d806fe91a..072ff23431e07 100644
--- a/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
+++ b/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
@@ -2,9 +2,9 @@
; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -S | FileCheck %s
; Check that we don't unnecessarily broadcast %pow
-define void @powi(ptr noalias %p, i32 %pow) {
-; CHECK-LABEL: define void @powi(
-; CHECK-SAME: ptr noalias [[P:%.*]], i32 [[POW:%.*]]) {
+define void @powi_only_first_lane_used_of_second_arg(ptr %p, i32 %pow) {
+; CHECK-LABEL: define void @powi_only_first_lane_used_of_second_arg(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[POW:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -14,11 +14,11 @@ define void @powi(ptr noalias %p, i32 %pow) {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i32 [[INDEX]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[TMP0]], i32 0
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[WIDE_LOAD]], i32 [[POW]])
-; CHECK-NEXT: store <4 x float> [[TMP3]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[WIDE_LOAD]], i32 [[POW]])
+; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[TMP1]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
More information about the llvm-commits
mailing list