[llvm] [LoopVectorize] Price vector library variants in getVectorCallCost (PR #202085)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 8 07:12:04 PDT 2026
https://github.com/seantalts updated https://github.com/llvm/llvm-project/pull/202085
>From 7344a61d09712e0bce85977897d9b2932a7a9c5a Mon Sep 17 00:00:00 2001
From: Sean Talts <sean.talts at gmail.com>
Date: Fri, 5 Jun 2026 18:32:33 +0000
Subject: [PATCH 1/2] [LoopVectorize] Price vector library variants in
getVectorCallCost instead of asserting
During loop vectorization, computePredInstDiscount queries the cost of instructions
at vector VF using getInstructionCost. If an instruction is a CallInst that has a
vector library variant, getInstructionCost delegates to getVectorCallCost, which
previously asserted that vector library variants should not reach it.
Instead of asserting, query the cost of the vector library variant using
TTI.getCallInstrCost and return it.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 36 +++++++++++++------
1 file changed, 25 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index fcfe11b6e9de6..d7335cbbda912 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2102,6 +2102,22 @@ static unsigned estimateElementCount(ElementCount VF,
return EstimatedVF;
}
+/// Returns the vector variant function of \p CI usable at \p VF, respecting \p
+/// MaskRequired. Returns nullptr if none is found.
+static Function *getVectorLibraryVariantFor(const CallInst &CI, ElementCount VF,
+ bool MaskRequired,
+ const TargetLibraryInfo *TLI) {
+ if (!TLI || CI.isNoBuiltin())
+ return nullptr;
+ for (const auto &Info : VFDatabase::getMappings(CI)) {
+ if (Info.Shape.VF == VF && (!MaskRequired || Info.isMasked())) {
+ if (auto *F = CI.getModule()->getFunction(Info.VectorName))
+ return F;
+ }
+ }
+ return nullptr;
+}
+
/// Returns true iff \p CI has a library vector variant usable at \p VF: a
/// mapping with matching VF, masked if required, whose vector function is
/// declared in the module. Such variants are priced by
@@ -2109,22 +2125,20 @@ static unsigned estimateElementCount(ElementCount VF,
static bool hasVectorLibraryVariantFor(const CallInst &CI, ElementCount VF,
bool MaskRequired,
const TargetLibraryInfo *TLI) {
- if (!TLI || CI.isNoBuiltin())
- return false;
- return any_of(VFDatabase::getMappings(CI), [&](const VFInfo &Info) {
- return Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()) &&
- CI.getModule()->getFunction(Info.VectorName);
- });
+ return getVectorLibraryVariantFor(CI, VF, MaskRequired, TLI) != nullptr;
}
InstructionCost
LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
ElementCount VF) const {
- // Vector library variants are priced by VPWidenCallRecipe::computeCost and
- // should not reach this function.
- assert((VF.isScalar() ||
- !hasVectorLibraryVariantFor(*CI, VF, isMaskRequired(CI), TLI)) &&
- "getVectorCallCost does not price vector library variants");
+ if (VF.isVector()) {
+ if (Function *Variant =
+ getVectorLibraryVariantFor(*CI, VF, isMaskRequired(CI), TLI)) {
+ return TTI.getCallInstrCost(nullptr, Variant->getReturnType(),
+ Variant->getFunctionType()->params(),
+ Config.CostKind);
+ }
+ }
Type *RetTy = CI->getType();
SmallVector<Type *, 4> Tys;
>From 9435303ffd0756297fd3672ce57b0d701c30309d Mon Sep 17 00:00:00 2001
From: Sean Talts <sean.talts at gmail.com>
Date: Mon, 8 Jun 2026 10:11:42 -0400
Subject: [PATCH 2/2] [LoopVectorize] Add test for pricing vector library
variants in predication discount
When a conditionally-executed call has a vector library variant matching the
VF, the call is a widen-with-mask candidate rather than scalar-with-predication.
If its result feeds an instruction that is scalar-with-predication (here a
scatter store), computePredInstDiscount walks the operand chain and queries the
call's cost at the vector VF via getVectorCallCost.
Previously this hit an assert ("getVectorCallCost does not price vector library
variants"); the variant is now priced via TTI.getCallInstrCost. The test
exercises that path and checks the loop vectorizes without crashing.
---
.../pred-inst-discount-vector-library-call.ll | 114 ++++++++++++++++++
1 file changed, 114 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/pred-inst-discount-vector-library-call.ll
diff --git a/llvm/test/Transforms/LoopVectorize/pred-inst-discount-vector-library-call.ll b/llvm/test/Transforms/LoopVectorize/pred-inst-discount-vector-library-call.ll
new file mode 100644
index 0000000000000..65adc1e0421f8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pred-inst-discount-vector-library-call.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -S 2>&1 | FileCheck %s
+
+; Regression test for the cost model. A conditionally-executed call has a vector
+; library variant matching the VF, so the call itself is a widen-with-mask
+; candidate (not scalar-with-predication). Its result feeds a scatter store that
+; *is* scalar-with-predication. While computing the predication discount for that
+; store, computePredInstDiscount walks the operand chain and queries the cost of
+; the call at the vector VF via getVectorCallCost. This used to hit an assert
+; ("getVectorCallCost does not price vector library variants"); the variant is
+; now priced via TTI.getCallInstrCost instead of crashing.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @pred_call_with_variant(ptr readonly %src, ptr noalias %dest, i64 %N) {
+; CHECK-LABEL: define void @pred_call_with_variant(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], ptr noalias [[DEST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ult <2 x i64> [[WIDE_LOAD]], splat (i64 5)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @foo(i64 [[TMP3]]) #[[ATTR0:[0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP3]]
+; CHECK-NEXT: store i64 [[TMP4]], ptr [[TMP5]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
+; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; CHECK: [[PRED_STORE_IF1]]:
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @foo(i64 [[TMP7]]) #[[ATTR0]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP7]]
+; CHECK-NEXT: store i64 [[TMP8]], ptr [[TMP9]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; CHECK: [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[END:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_LOOP:.*]] ]
+; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[IDX:%.*]] = load i64, ptr [[LD_ADDR]], align 8
+; CHECK-NEXT: [[IFCOND:%.*]] = icmp ult i64 [[IDX]], 5
+; CHECK-NEXT: br i1 [[IFCOND]], label %[[IF_THEN:.*]], label %[[FOR_LOOP]]
+; CHECK: [[IF_THEN]]:
+; CHECK-NEXT: [[FOO_RET:%.*]] = call i64 @foo(i64 [[IDX]]) #[[ATTR0]]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[IDX]]
+; CHECK-NEXT: store i64 [[FOO_RET]], ptr [[ST_ADDR]], align 8
+; CHECK-NEXT: br label %[[FOR_LOOP]]
+; CHECK: [[FOR_LOOP]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[LOOPCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[LOOPCOND]], label %[[END]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[END]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.loop ]
+ %ld.addr = getelementptr inbounds i64, ptr %src, i64 %iv
+ %idx = load i64, ptr %ld.addr, align 8
+ %ifcond = icmp ult i64 %idx, 5
+ br i1 %ifcond, label %if.then, label %for.loop
+
+if.then:
+ %foo.ret = call i64 @foo(i64 %idx) #0
+ ; A scatter to a loaded index: not consecutive, so the store is
+ ; scalar-with-predication and triggers the predication-discount analysis.
+ %st.addr = getelementptr inbounds i64, ptr %dest, i64 %idx
+ store i64 %foo.ret, ptr %st.addr, align 8
+ br label %for.loop
+
+for.loop:
+ %iv.next = add nsw nuw i64 %iv, 1
+ %loopcond = icmp eq i64 %iv.next, %N
+ br i1 %loopcond, label %end, label %for.body
+
+end:
+ ret void
+}
+
+declare i64 @foo(i64) #0
+declare <2 x i64> @vector_foo(<2 x i64>, <2 x i1>)
+
+; A masked vector variant matching the forced VF of 2, so the call is a valid
+; widen-with-mask candidate (not itself scalar-with-predication) while feeding
+; the scalar-with-predication scatter store.
+attributes #0 = { readonly nounwind "vector-function-abi-variant"="_ZGV_LLVM_M2v_foo(vector_foo)" }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
More information about the llvm-commits
mailing list