[llvm] [LoopVectorize] Don't assert in getVectorCallCost for vector library variants (PR #202085)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 05:16:16 PDT 2026
https://github.com/seantalts updated https://github.com/llvm/llvm-project/pull/202085
>From c33e3091de1413065e9b53c05077ad848cc58368 Mon Sep 17 00:00:00 2001
From: Sean Talts <sean.talts at gmail.com>
Date: Fri, 5 Jun 2026 18:32:33 +0000
Subject: [PATCH 1/2] [LoopVectorize] Don't assert in getVectorCallCost for
vector library variants
During loop vectorization, computePredInstDiscount queries the cost of
instructions at vector VF using getInstructionCost. If an instruction is a
CallInst that has a vector library variant, getInstructionCost delegates to
getVectorCallCost, which asserted that vector library variants should not reach
it.
Such a call can reach getVectorCallCost via computePredInstDiscount, before the
call's widening decision is made, when a predicated user (e.g. a scatter store)
is being considered for scalarization. Remove the assert and fall through to the
existing scalarization cost, which is the cost relevant to that analysis.
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7ad454d4a1797..5e7bbd71b5260 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2116,12 +2116,11 @@ static bool hasVectorLibraryVariantFor(const CallInst &CI, ElementCount VF,
InstructionCost
LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
ElementCount VF) const {
- // Vector library variants are priced by VPWidenCallRecipe::computeCost and
- // should not reach this function.
- assert((VF.isScalar() ||
- !hasVectorLibraryVariantFor(*CI, VF, isMaskRequired(CI), TLI)) &&
- "getVectorCallCost does not price vector library variants");
-
+ // A call with a vector library variant is normally priced by
+ // VPWidenCallRecipe::computeCost. It can still reach here via
+ // computePredInstDiscount, which queries the cost before the call's widening
+ // decision is made; in that case the predicated call is being considered for
+ // scalarization, so fall through to the scalarization cost below.
Type *RetTy = CI->getType();
SmallVector<Type *, 4> Tys;
for (auto &ArgOp : CI->args())
>From cde5808f0d7ead29d5ae11f3a34d6e26ff698a16 Mon Sep 17 00:00:00 2001
From: Sean Talts <sean.talts at gmail.com>
Date: Fri, 12 Jun 2026 08:07:20 -0400
Subject: [PATCH 2/2] [LoopVectorize] Add test for predication-discount cost of
a library-variant call
When a conditionally-executed call has a vector library variant matching the
VF, the call is a widen-with-mask candidate rather than scalar-with-predication.
If its result feeds an instruction that is scalar-with-predication (here a
scatter store), computePredInstDiscount walks the operand chain and queries the
call's cost at the vector VF via getVectorCallCost.
This used to hit an assert ("getVectorCallCost does not price vector library
variants"); getVectorCallCost now falls through to the scalarization cost. The
test exercises that path and checks the loop vectorizes without crashing.
---
.../pred-inst-discount-vector-library-call.ll | 114 ++++++++++++++++++
1 file changed, 114 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/pred-inst-discount-vector-library-call.ll
diff --git a/llvm/test/Transforms/LoopVectorize/pred-inst-discount-vector-library-call.ll b/llvm/test/Transforms/LoopVectorize/pred-inst-discount-vector-library-call.ll
new file mode 100644
index 0000000000000..5acf7a3a95102
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pred-inst-discount-vector-library-call.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -S 2>&1 | FileCheck %s
+
+; Regression test for the cost model. A conditionally-executed call has a vector
+; library variant matching the VF, so the call itself is a widen-with-mask
+; candidate (not scalar-with-predication). Its result feeds a scatter store that
+; *is* scalar-with-predication. While computing the predication discount for that
+; store, computePredInstDiscount walks the operand chain and queries the cost of
+; the call at the vector VF via getVectorCallCost. This used to hit an assert
+; ("getVectorCallCost does not price vector library variants"); getVectorCallCost
+; now falls through to the scalarization cost instead of asserting.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @pred_call_with_variant(ptr readonly %src, ptr noalias %dest, i64 %N) {
+; CHECK-LABEL: define void @pred_call_with_variant(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], ptr noalias [[DEST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ult <2 x i64> [[WIDE_LOAD]], splat (i64 5)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @foo(i64 [[TMP3]]) #[[ATTR0:[0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP3]]
+; CHECK-NEXT: store i64 [[TMP4]], ptr [[TMP5]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
+; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; CHECK: [[PRED_STORE_IF1]]:
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @foo(i64 [[TMP7]]) #[[ATTR0]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP7]]
+; CHECK-NEXT: store i64 [[TMP8]], ptr [[TMP9]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; CHECK: [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[END:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_LOOP:.*]] ]
+; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[IDX:%.*]] = load i64, ptr [[LD_ADDR]], align 8
+; CHECK-NEXT: [[IFCOND:%.*]] = icmp ult i64 [[IDX]], 5
+; CHECK-NEXT: br i1 [[IFCOND]], label %[[IF_THEN:.*]], label %[[FOR_LOOP]]
+; CHECK: [[IF_THEN]]:
+; CHECK-NEXT: [[FOO_RET:%.*]] = call i64 @foo(i64 [[IDX]]) #[[ATTR0]]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[IDX]]
+; CHECK-NEXT: store i64 [[FOO_RET]], ptr [[ST_ADDR]], align 8
+; CHECK-NEXT: br label %[[FOR_LOOP]]
+; CHECK: [[FOR_LOOP]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[LOOPCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[LOOPCOND]], label %[[END]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[END]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.loop ]
+ %ld.addr = getelementptr inbounds i64, ptr %src, i64 %iv
+ %idx = load i64, ptr %ld.addr, align 8
+ %ifcond = icmp ult i64 %idx, 5
+ br i1 %ifcond, label %if.then, label %for.loop
+
+if.then:
+ %foo.ret = call i64 @foo(i64 %idx) #0
+ ; A scatter to a loaded index: not consecutive, so the store is
+ ; scalar-with-predication and triggers the predication-discount analysis.
+ %st.addr = getelementptr inbounds i64, ptr %dest, i64 %idx
+ store i64 %foo.ret, ptr %st.addr, align 8
+ br label %for.loop
+
+for.loop:
+ %iv.next = add nsw nuw i64 %iv, 1
+ %loopcond = icmp eq i64 %iv.next, %N
+ br i1 %loopcond, label %end, label %for.body
+
+end:
+ ret void
+}
+
+declare i64 @foo(i64) #0
+declare <2 x i64> @vector_foo(<2 x i64>, <2 x i1>)
+
+; A masked vector variant matching the forced VF of 2, so the call is a valid
+; widen-with-mask candidate (not itself scalar-with-predication) while feeding
+; the scalar-with-predication scatter store.
+attributes #0 = { readonly nounwind "vector-function-abi-variant"="_ZGV_LLVM_M2v_foo(vector_foo)" }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
More information about the llvm-commits
mailing list