[llvm] 7b09a4e - [LV] Fix the cost model for freeze instructions (#197188)
via llvm-commits
llvm-commits at lists.llvm.org
Thu May 14 01:47:00 PDT 2026
Author: David Sherwood
Date: 2026-05-14T09:46:55+01:00
New Revision: 7b09a4efbff5002e9f81ad7e6f49d6e90b976d49
URL: https://github.com/llvm/llvm-project/commit/7b09a4efbff5002e9f81ad7e6f49d6e90b976d49
DIFF: https://github.com/llvm/llvm-project/commit/7b09a4efbff5002e9f81ad7e6f49d6e90b976d49.diff
LOG: [LV] Fix the cost model for freeze instructions (#197188)
While working on a PR to add a cost model for VPDerivedIV recipes I
noticed that a loop in or_reduction_with_freeze:
test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
stopped vectorising because the cost model decided it was no longer
worth it. However, the main cause of this was the incredibly high cost
(14) of freeze for VF=2. We were using the cost of a vector mul
instruction as a proxy for the freeze cost, which is incredibly bad for
an AArch64 target without SVE since the operation needs scalarising.
As far as I understand, the freeze instruction does not lead to any
actual code being generated and acts merely as a barrier to potentially
unsafe optimisations. As such, I've updated the cost model to return 0
instead.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 00d378adf9c5d..7213d4ae795ec 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5523,6 +5523,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
if (VF.isScalable())
return InstructionCost::getInvalid();
return TTI.getArithmeticInstrCost(Instruction::Mul, RetTy, Config.CostKind);
+ case Instruction::Freeze:
+ return TTI::TCC_Free;
default:
// This opcode is unknown. Assume that it is the same as 'mul'.
return TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 27569f9966de0..397366bae9af9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1000,9 +1000,12 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
RHSInfo, Operands, CtxI, &Ctx.TLI);
}
case Instruction::Freeze:
- // This opcode is unknown. Assume that it is the same as 'mul'.
- return Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, ResultTy,
- Ctx.CostKind);
+ // NOTE: The only way to ask for the cost is via getInstructionCost, which
+ // requires the actual vector instruction. Instead, both here and in the
+ // LoopVectorizationCostModel::getInstructionCost the costs mirror the
+ // current behaviour in llvm/Analysis/TargetTransformInfoImpl.h to keep
+ // them in sync.
+ return TTI::TCC_Free;
case Instruction::ExtractValue:
return Ctx.TTI.getInsertExtractValueCost(Instruction::ExtractValue,
Ctx.CostKind);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
index 8c8348cf5700f..4bb6bf39413d0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
@@ -60,8 +60,6 @@ exit:
ret i64 %res
}
-; The scalar cost of this loop must include the freeze's cost, otherwise VF=2
-; is incorrectly rejected as unprofitable.
define i32 @or_reduction_with_freeze(ptr %dst, ptr %src) {
; CHECK-LABEL: define i32 @or_reduction_with_freeze(
; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
@@ -75,7 +73,7 @@ define i32 @or_reduction_with_freeze(ptr %dst, ptr %src) {
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[DST6]], [[SRC7]]
; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 3
; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 18
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 10
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[DST1]] to i3
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
index 12d32872e1453..aeb222e677e63 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
@@ -104,7 +104,7 @@ define void @test_vpinstruction_freeze_cost(ptr %src, ptr noalias %dst) {
; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction: %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction: %g.src = getelementptr inbounds i64, ptr %src, i64 %iv
; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %l = load i64, ptr %g.src, align 8
-; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %fr = freeze i64 %l
+; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction: %fr = freeze i64 %l
; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction: %g.dst = getelementptr inbounds i64, ptr %dst, i64 %iv
; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %fr, ptr %g.dst, align 8
; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %iv.next = add nuw nsw i64 %iv, 1
@@ -117,7 +117,7 @@ define void @test_vpinstruction_freeze_cost(ptr %src, ptr noalias %dst) {
; CHECK: Cost of 0 for VF 2: CLONE ir<%g.src> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%g.src>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%l> = load vp<[[VP5]]>
-; CHECK: Cost of 2 for VF 2: WIDEN ir<%fr> = freeze ir<%l>
+; CHECK: Cost of 0 for VF 2: WIDEN ir<%fr> = freeze ir<%l>
; CHECK: Cost of 0 for VF 2: CLONE ir<%g.dst> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%g.dst>
; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP6]]>, ir<%fr>
@@ -142,7 +142,7 @@ define void @test_vpinstruction_freeze_cost(ptr %src, ptr noalias %dst) {
; CHECK: Cost of 0 for VF 4: CLONE ir<%g.src> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds ir<%g.src>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%l> = load vp<[[VP5]]>
-; CHECK: Cost of 2 for VF 4: WIDEN ir<%fr> = freeze ir<%l>
+; CHECK: Cost of 0 for VF 4: WIDEN ir<%fr> = freeze ir<%l>
; CHECK: Cost of 0 for VF 4: CLONE ir<%g.dst> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer inbounds ir<%g.dst>
; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP6]]>, ir<%fr>
More information about the llvm-commits
mailing list