[llvm] [LV] Set the underlying value of VPInstruction::Reverse (PR #173856)
Mel Chen via llvm-commits
llvm-commits at lists.llvm.org
Mon Dec 29 03:11:24 PST 2025
https://github.com/Mel-Chen created https://github.com/llvm/llvm-project/pull/173856
When vectorizing uncountable loops, a reverse widen load may serve as an exit-condition instruction. Its cost is first computed in LoopVectorizationPlanner::precomputeCosts and then ignored in VPlan::cost.
However, since the reverse operation and the reverse widen load are now separated into two recipes, the reverse recipe extracted from the reverse widen load ends up having its cost computed again, because VPlan::cost doesn’t know to ignore this reverse recipe.
This patch sets the underlying value of the reverse recipe so that VPlan::cost can ignore reverse recipes extracted from reverse accesses that serve as exit-condition instructions, thereby correcting the behavior of the VPlan-based cost model.
>From 4aef71dbfd9ea401cd2b7464f5fe07e2cd4664c4 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 29 Dec 2025 01:13:04 -0800
Subject: [PATCH 1/3] pre-commit test
---
.../LoopVectorize/X86/uncountable-reverse.ll | 30 +++++++++++++++++++
1 file changed, 30 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/uncountable-reverse.ll
diff --git a/llvm/test/Transforms/LoopVectorize/X86/uncountable-reverse.ll b/llvm/test/Transforms/LoopVectorize/X86/uncountable-reverse.ll
new file mode 100644
index 0000000000000..55af3493198ae
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/uncountable-reverse.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=loop-vectorize -S | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+ at gg = constant [4 x [80 x i8]] [
+ [80 x i8] c" ",
+ [80 x i8] c"22 ",
+ [80 x i8] c" ",
+ [80 x i8] c"4444 "
+]
+
+define i1 @test() #0 {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 79, %entry ], [ %iv.next, %loop ]
+ %iv.next = add i64 %iv, -1
+ %bound.check = icmp ne i64 %iv, 0
+ %gep = getelementptr [1 x i8], ptr getelementptr inbounds nuw (i8, ptr @gg, i64 80), i64 %iv
+ %load = load i8, ptr %gep, align 1
+ %cmp = icmp eq i8 %load, 32
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret i1 %cmp
+}
+
+attributes #0 = { "target-cpu"="znver3" }
>From 54698cd62391e911b0594495472715af144ded29 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 29 Dec 2025 01:35:08 -0800
Subject: [PATCH 2/3] simplest test case
---
.../Transforms/LoopVectorize/X86/uncountable-reverse.ll | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/uncountable-reverse.ll b/llvm/test/Transforms/LoopVectorize/X86/uncountable-reverse.ll
index 55af3493198ae..485086781f8de 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/uncountable-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/uncountable-reverse.ll
@@ -3,12 +3,7 @@
target triple = "x86_64-unknown-linux-gnu"
- at gg = constant [4 x [80 x i8]] [
- [80 x i8] c" ",
- [80 x i8] c"22 ",
- [80 x i8] c" ",
- [80 x i8] c"4444 "
-]
+ at gg = constant [80 x i8] c"22 "
define i1 @test() #0 {
entry:
@@ -18,7 +13,7 @@ loop:
%iv = phi i64 [ 79, %entry ], [ %iv.next, %loop ]
%iv.next = add i64 %iv, -1
%bound.check = icmp ne i64 %iv, 0
- %gep = getelementptr [1 x i8], ptr getelementptr inbounds nuw (i8, ptr @gg, i64 80), i64 %iv
+ %gep = getelementptr [1 x i8], ptr @gg, i64 %iv
%load = load i8, ptr %gep, align 1
%cmp = icmp eq i8 %load, 32
br i1 %cmp, label %loop, label %exit
>From 66e2633c8f7e5ba4eeeaf5162033591b61a9e89c Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 29 Dec 2025 02:43:01 -0800
Subject: [PATCH 3/3] fix
---
.../Transforms/Vectorize/LoopVectorize.cpp | 11 ++--
.../LoopVectorize/X86/uncountable-reverse.ll | 60 ++++++++++++++++++-
.../vplan-sink-scalars-and-merge.ll | 4 +-
3 files changed, 68 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 40c3efb644527..0edfc1aec5e3b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7705,8 +7705,10 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
*VPI, Load->getDebugLoc());
if (Reverse) {
Builder.insert(LoadR);
- return new VPInstruction(VPInstruction::Reverse, LoadR, {}, {},
- LoadR->getDebugLoc());
+ auto *Rev = new VPInstruction(VPInstruction::Reverse, LoadR, {}, {},
+ LoadR->getDebugLoc());
+ Rev->setUnderlyingValue(Load);
+ return Rev;
}
return LoadR;
}
@@ -7714,8 +7716,9 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
StoreInst *Store = cast<StoreInst>(I);
VPValue *StoredVal = VPI->getOperand(0);
if (Reverse)
- StoredVal = Builder.createNaryOp(VPInstruction::Reverse, StoredVal,
- Store->getDebugLoc());
+ StoredVal =
+ Builder.createNaryOp(VPInstruction::Reverse, StoredVal, Store,
+ /*Flags=*/{}, /*MD=*/{}, Store->getDebugLoc());
return new VPWidenStoreRecipe(*Store, Ptr, StoredVal, Mask, Consecutive,
Reverse, *VPI, Store->getDebugLoc());
}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/uncountable-reverse.ll b/llvm/test/Transforms/LoopVectorize/X86/uncountable-reverse.ll
index 485086781f8de..e597cd9e3ee25 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/uncountable-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/uncountable-reverse.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt < %s -passes=loop-vectorize -S | FileCheck %s
target triple = "x86_64-unknown-linux-gnu"
@@ -6,6 +6,64 @@ target triple = "x86_64-unknown-linux-gnu"
@gg = constant [80 x i8] c"22 "
define i1 @test() #0 {
+; CHECK-LABEL: define i1 @test(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = sub i64 79, [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [1 x i8], ptr @gg, i64 [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i64 -24
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -7
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <8 x i8> [[WIDE_LOAD]], <8 x i8> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <8 x i8> [[REVERSE]], splat (i8 32)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP4]], i32 7
+; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 64, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT4:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 79, [[INDEX1]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [1 x i8], ptr @gg, i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 -3
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP9]], align 1
+; CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD2]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq <4 x i8> [[REVERSE3]], splat (i8 32)
+; CHECK-NEXT: [[INDEX_NEXT4]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 76
+; CHECK-NEXT: br i1 [[TMP11]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP10]], i32 3
+; CHECK-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 15, %[[VEC_EPILOG_ITER_CHECK]] ], [ 79, %[[ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1
+; CHECK-NEXT: [[BOUND_CHECK:%.*]] = icmp ne i64 [[IV]], 0
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr [1 x i8], ptr @gg, i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[LOAD]], 32
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[CMP_LCSSA:%.*]] = phi i1 [ [[CMP]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[TMP12]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i1 [[CMP_LCSSA]]
+;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 9c4c9534157c7..5f5e48c4f1749 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -1159,8 +1159,8 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-NEXT: CLONE ir<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-end-pointer inbounds ir<%ptr.iv.next>, vp<[[VF]]>
; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>
-; CHECK-NEXT: EMIT vp<%9> = reverse ir<%l>
-; CHECK-NEXT: WIDEN ir<%c.1> = icmp ne vp<%9>, ir<0>
+; CHECK-NEXT: EMIT ir<%l>.1 = reverse ir<%l>
+; CHECK-NEXT: WIDEN ir<%c.1> = icmp ne ir<%l>.1, ir<0>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
More information about the llvm-commits
mailing list