[llvm] [VPlan] Consolidate VPIWithType and VPI (NFC) (PR #203019)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 20 04:04:06 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/203019
>From 27bd34b5e110bb5d63f27142b44c191ade0120f9 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Fri, 26 Jun 2026 18:41:13 +0100
Subject: [PATCH 1/6] [LV/test] Pre-commit RISC-V test
---
.../RISCV/scatter-type-mismatch.ll | 48 +++++++++++++++++++
1 file changed, 48 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/scatter-type-mismatch.ll
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scatter-type-mismatch.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scatter-type-mismatch.ll
new file mode 100644
index 0000000000000..1de9b39b49052
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/scatter-type-mismatch.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -p loop-vectorize -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s
+
+; This test shows the loop being vectorized with a scatter
+; when the GEP's type mismatches with the stored value's
+; type, necessitating the insertion of a cast.
+define void @scatter_gepty_mismatch_storety(ptr %p) {
+; CHECK-LABEL: define void @scatter_gepty_mismatch_storety(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 128, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr double, ptr [[P]], <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> zeroinitializer, <vscale x 4 x ptr> align 4 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i8.nxv4p0(<vscale x 4 x i8> zeroinitializer, <vscale x 4 x ptr> align 1 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP2]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr double, ptr %p, i64 %iv
+ store float 0.000000e+00, ptr %gep
+ store i8 0, ptr %gep
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From e3c3fd4fff0ce95e3e15bc0ba7dfa1b5b4419d61 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 2 Jul 2026 10:30:09 +0100
Subject: [PATCH 2/6] [VPlan] Pre-commit fpiv_wideivstep_flags test
---
.../VPlan/vplan-printing-flags.ll | 224 +++++++++++++++---
1 file changed, 186 insertions(+), 38 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll
index 30d916283d2be..a743dfc0583f2 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll
@@ -1,33 +1,78 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block:" --version 6
-; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
-; RUN: -vplan-print-after=printAfterInitialConstruction -disable-output %s 2>&1 \
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=2 \
+; RUN: -vplan-print-after=unrollByUF -disable-output %s 2>&1 \
; RUN: | FileCheck --strict-whitespace %s
define void @cast_flags_mixed(ptr noalias %A, ptr noalias %B) {
; CHECK-LABEL: VPlan for loop in 'cast_flags_mixed'
-; CHECK: VPlan ' for UF>=1' {
+; CHECK: VPlan 'Initial VPlan for VF={4},UF={2}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT branch-on-cond ir<false>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): loop
-; CHECK-EMPTY:
-; CHECK-NEXT: loop:
-; CHECK-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, loop ]
-; CHECK-NEXT: EMIT-SCALAR ir<%zext.nneg> = zext nneg ir<3> to i64
-; CHECK-NEXT: EMIT ir<%gep.zext> = getelementptr ir<%A>, ir<%zext.nneg>
-; CHECK-NEXT: EMIT-SCALAR ir<%sext.plain> = sext ir<%iv> to i64
-; CHECK-NEXT: EMIT ir<%gep.sext> = getelementptr ir<%A>, ir<%sext.plain>
-; CHECK-NEXT: EMIT-SCALAR ir<%trunc.flags> = trunc nuw nsw ir<3> to i8
-; CHECK-NEXT: EMIT ir<%gep.trunc> = getelementptr ir<%B>, ir<%trunc.flags>
-; CHECK-NEXT: EMIT store ir<%gep.sext>, ir<%gep.zext>
-; CHECK-NEXT: EMIT store ir<0>, ir<%gep.trunc>
-; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%ec> = icmp slt ir<%iv.next>, ir<4>
-; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = not ir<%ec>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
-; CHECK-NEXT: Successor(s): middle.block, loop
+; CHECK-NEXT: CLONE ir<%gep.zext> = getelementptr ir<%A>, ir<3>
+; CHECK-NEXT: CLONE ir<%gep.trunc> = getelementptr ir<%B>, ir<3>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = wide-iv-step vp<[[VP0]]>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = wide-iv-step vp<[[VP0]]>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>, vp<[[VP5]]>, vp<%step.add>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = WIDEN-INDUCTION nuw ir<0>, ir<1>, vp<[[VP0]]>, vp<[[VP6]]>, vp<%step.add>.1
+; CHECK-NEXT: EMIT vp<%step.add> = add nsw ir<%iv>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<%step.add>.1 = add nuw vp<[[VP8]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp ule vp<[[VP8]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp ule vp<%step.add>.1, vp<[[VP3]]>
+; CHECK-NEXT: WIDEN-CAST ir<%sext.plain> = sext ir<%iv> to i64
+; CHECK-NEXT: WIDEN-CAST ir<%sext.plain>.1 = sext vp<%step.add> to i64
+; CHECK-NEXT: Successor(s): pred.store
+; CHECK-EMPTY:
+; CHECK-NEXT: <xVFxUF> pred.store: {
+; CHECK-NEXT: pred.store.entry:
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
+; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.if:
+; CHECK-NEXT: REPLICATE ir<%gep.sext> = getelementptr ir<%A>, ir<%sext.plain>
+; CHECK-NEXT: REPLICATE store ir<%gep.sext>, ir<%gep.zext>
+; CHECK-NEXT: Successor(s): pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.continue:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): pred.store
+; CHECK-EMPTY:
+; CHECK-NEXT: <xVFxUF> pred.store: {
+; CHECK-NEXT: pred.store.entry:
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP10]]>
+; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.if:
+; CHECK-NEXT: REPLICATE ir<%gep.sext>.1 = getelementptr ir<%A>, ir<%sext.plain>.1
+; CHECK-NEXT: REPLICATE store ir<%gep.sext>.1, ir<%gep.zext>
+; CHECK-NEXT: Successor(s): pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.continue:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): loop.0
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.0:
+; CHECK-NEXT: CLONE store ir<0>, ir<%gep.trunc>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP7]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
;
@@ -54,29 +99,76 @@ exit:
define void @cast_flags_single(ptr noalias %A, ptr noalias %B) {
; CHECK-LABEL: VPlan for loop in 'cast_flags_single'
-; CHECK: VPlan ' for UF>=1' {
+; CHECK: VPlan 'Initial VPlan for VF={4},UF={2}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT branch-on-cond ir<false>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): loop
-; CHECK-EMPTY:
-; CHECK-NEXT: loop:
-; CHECK-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, loop ]
-; CHECK-NEXT: EMIT-SCALAR ir<%trunc.nuw.only> = trunc nuw ir<3> to i8
-; CHECK-NEXT: EMIT-SCALAR ir<%trunc.nsw.only> = trunc nsw ir<3> to i8
-; CHECK-NEXT: EMIT-SCALAR ir<%zext.plain> = zext ir<3> to i64
-; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr ir<%A>, ir<%iv>
-; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr ir<%B>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%trunc.nuw.only>, ir<%gep.a>
-; CHECK-NEXT: EMIT store ir<%trunc.nsw.only>, ir<%gep.a>
-; CHECK-NEXT: EMIT store ir<%zext.plain>, ir<%gep.b>
-; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%ec> = icmp slt ir<%iv.next>, ir<4>
-; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = not ir<%ec>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
-; CHECK-NEXT: Successor(s): middle.block, loop
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = wide-iv-step vp<[[VP0]]>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = WIDEN-INDUCTION nuw ir<0>, ir<1>, vp<[[VP0]]>, vp<[[VP5]]>, vp<%step.add>
+; CHECK-NEXT: EMIT vp<%step.add> = add nuw vp<[[VP7]]>, vp<[[VP5]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6]]> * ir<1>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp ule vp<%step.add>, vp<[[VP3]]>
+; CHECK-NEXT: Successor(s): pred.store
+; CHECK-EMPTY:
+; CHECK-NEXT: <xVFxUF> pred.store: {
+; CHECK-NEXT: pred.store.entry:
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
+; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.if:
+; CHECK-NEXT: vp<[[VP11:%[0-9]+]]> = SCALAR-STEPS vp<[[VP8]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<%A>, vp<[[VP11]]>
+; CHECK-NEXT: REPLICATE store ir<3>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE store ir<3>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<%B>, vp<[[VP11]]>
+; CHECK-NEXT: REPLICATE store ir<3>, ir<%gep.b>
+; CHECK-NEXT: Successor(s): pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.continue:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): pred.store
+; CHECK-EMPTY:
+; CHECK-NEXT: <xVFxUF> pred.store: {
+; CHECK-NEXT: pred.store.entry:
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP10]]>
+; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.if:
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP12:%[0-9]+]]> = trunc vp<[[VP0]]> to i16
+; CHECK-NEXT: vp<[[VP13:%[0-9]+]]> = SCALAR-STEPS vp<[[VP8]]>, ir<1>, vp<[[VP0]]>, vp<[[VP12]]>
+; CHECK-NEXT: REPLICATE ir<%gep.a>.1 = getelementptr ir<%A>, vp<[[VP13]]>
+; CHECK-NEXT: REPLICATE store ir<3>, ir<%gep.a>.1
+; CHECK-NEXT: REPLICATE store ir<3>, ir<%gep.a>.1
+; CHECK-NEXT: REPLICATE ir<%gep.b>.1 = getelementptr ir<%B>, vp<[[VP13]]>
+; CHECK-NEXT: REPLICATE store ir<3>, ir<%gep.b>.1
+; CHECK-NEXT: Successor(s): pred.store.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.store.continue:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): loop.2
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.2:
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
;
@@ -100,3 +192,59 @@ loop:
exit:
ret void
}
+
+define void @fpiv_wideivstep_flags(float %init, ptr %p, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'fpiv_wideivstep_flags'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF={2}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%n>, ir<8>
+; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%init> + vp<[[VP2]]> * ir<1.000000e+00>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = wide-iv-step vp<[[VP0]]>, ir<1.000000e+00>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%x> = WIDEN-INDUCTION fast ir<%init>, ir<1.000000e+00>, vp<[[VP0]]>, vp<[[VP5]]>, vp<%step.add>
+; CHECK-NEXT: EMIT vp<%step.add> = fadd fast ir<%x>, vp<[[VP5]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%p>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>, vp<[[VP8]]>
+; CHECK-NEXT: WIDEN store vp<[[VP9]]>, ir<%x>
+; CHECK-NEXT: WIDEN store vp<[[VP10]]>, vp<%step.add>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %x = phi float [ %init, %entry ], [ %add, %loop ]
+ %arrayidx = getelementptr inbounds float, ptr %p, i64 %iv
+ store float %x, ptr %arrayidx, align 4
+ %add = fadd fast float %x, 1.000000e+00
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 354980d2e8b62d8bf78934a31702d73d4dcfd5bd Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 10 Jun 2026 16:25:40 +0100
Subject: [PATCH 3/6] [VPlan] Consolidate VPIWithType and VPI (NFC)
The extra ResultType has been absorbed into VPInstruction, and the
recipe classes can now be consolidated.
---
.../Vectorize/LoopVectorizationPlanner.h | 31 ++--
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 87 +----------
.../Vectorize/VPlanConstruction.cpp | 2 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 147 ++++++------------
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 1 +
.../VPlan/vplan-printing-flags.ll | 36 ++---
.../Transforms/Vectorize/VPDomTreeTest.cpp | 12 +-
.../Transforms/Vectorize/VPlanTest.cpp | 40 ++---
10 files changed, 122 insertions(+), 242 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 89b16f9e27045..dca1a287c3f60 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -231,8 +231,8 @@ class VPBuilder {
Type *ResultTy, const VPIRFlags &Flags = {},
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
- return tryInsertInstruction(new VPInstructionWithType(
- Opcode, Operands, ResultTy, Flags, {}, DL, Name));
+ return tryInsertInstruction(
+ new VPInstruction(Opcode, Operands, Flags, {}, DL, Name, ResultTy));
}
VPInstruction *createFirstActiveLane(ArrayRef<VPValue *> Masks,
@@ -412,19 +412,18 @@ class VPBuilder {
new VPDerivedIVRecipe(Kind, FPBinOp, Start, Current, Step));
}
- VPInstructionWithType *createScalarLoad(Type *ResultTy, VPValue *Addr,
- DebugLoc DL,
- const VPIRMetadata &Metadata = {}) {
- return tryInsertInstruction(new VPInstructionWithType(
- Instruction::Load, Addr, ResultTy, {}, Metadata, DL));
+ VPInstruction *createScalarLoad(Type *ResultTy, VPValue *Addr, DebugLoc DL,
+ const VPIRMetadata &Metadata = {}) {
+ return tryInsertInstruction(new VPInstruction(Instruction::Load, Addr, {},
+ Metadata, DL, "", ResultTy));
}
VPInstruction *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,
Type *ResultTy, DebugLoc DL,
const VPIRMetadata &Metadata = {}) {
- return tryInsertInstruction(new VPInstructionWithType(
- Opcode, Op, ResultTy, VPIRFlags::getDefaultFlags(Opcode), Metadata,
- DL));
+ return tryInsertInstruction(
+ new VPInstruction(Opcode, Op, VPIRFlags::getDefaultFlags(Opcode),
+ Metadata, DL, "", ResultTy));
}
VPInstruction *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,
@@ -432,7 +431,7 @@ class VPBuilder {
const VPIRFlags &Flags,
const VPIRMetadata &Metadata = {}) {
return tryInsertInstruction(
- new VPInstructionWithType(Opcode, Op, ResultTy, Flags, Metadata, DL));
+ new VPInstruction(Opcode, Op, Flags, Metadata, DL, "", ResultTy));
}
/// Create a scalar call to the intrinsic \p IntrinsicID with \p Operands, and
@@ -443,8 +442,8 @@ class VPBuilder {
VPlan &Plan = getPlan();
SmallVector<VPValue *, 2> Ops(Operands);
Ops.push_back(Plan.getConstantInt(8 * sizeof(IntrinsicID), IntrinsicID));
- return tryInsertInstruction(new VPInstructionWithType(
- VPInstruction::Intrinsic, Ops, ResultTy, {}, {}, DL));
+ return tryInsertInstruction(new VPInstruction(VPInstruction::Intrinsic, Ops,
+ {}, {}, DL, "", ResultTy));
}
/// Create a scalar llvm.vscale call.
@@ -496,8 +495,10 @@ class VPBuilder {
DebugLoc DL, Instruction *UV) {
if (Instruction::isCast(Opcode)) {
assert(!Mask && "Cast cannot be predicated");
- return new VPInstructionWithType(Opcode, Operands, UV->getType(), Flags,
- Metadata, DL, UV->getName(), UV);
+ auto *VPI = new VPInstruction(Opcode, Operands, Flags, Metadata, DL,
+ UV->getName(), UV->getType());
+ VPI->setUnderlyingValue(UV);
+ return VPI;
}
return new VPReplicateRecipe(UV, Operands, /*IsSingleScalar=*/true, Mask,
Flags, Metadata, DL);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4cd6a55f01a90..92dea8f1d99af 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6482,9 +6482,8 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
if (Instruction::isCast(VPI->getOpcode())) {
auto *CI = cast<CastInst>(Instr);
- auto *CastR = cast<VPInstructionWithType>(VPI);
return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0),
- CastR->getResultType(), CI, *VPI, *VPI,
+ VPI->getScalarType(), CI, *VPI, *VPI,
VPI->getDebugLoc());
}
@@ -6750,8 +6749,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
VPReplicateRecipe, VPWidenLoadRecipe, VPWidenStoreRecipe,
VPWidenCallRecipe, VPWidenIntrinsicRecipe, VPVectorPointerRecipe,
VPVectorEndPointerRecipe, VPHistogramRecipe>(&R) ||
- (isa<VPInstructionWithType>(R) &&
- Instruction::isCast(cast<VPInstructionWithType>(R).getOpcode()) &&
+ (Instruction::isCast(cast<VPInstruction>(R).getOpcode()) &&
vputils::onlyFirstLaneUsed(R.getVPSingleValue())))
continue;
auto *VPI = cast<VPInstruction>(&R);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index edb24141f6073..8736617842b94 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1329,14 +1329,6 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// backedge value). Has the wide induction recipe as operand.
ExitingIVValue,
MaskedCond,
-
- // The opcodes below are used for VPInstructionWithType.
- // NOTE: VPInstructionWithType classes are also used for:
- // 1. All CastInst variants - see createVPInstructionsForVPBB, and other
- // cases where createScalarCast, createScalarZExtOrTrunc and
- // createScalarSExtOrTrunc are invoked.
- // 2. Scalar load instructions - see createVPInstructionsForVPBB.
-
/// Scale the first operand (vector step) by the second operand
/// (scalar-step). Casts both operands to the result type if needed.
WideIVStep,
@@ -1498,6 +1490,11 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Returns true if the recipe only uses the first lane of operand \p Op.
bool usesFirstLaneOnly(const VPValue *Op) const override;
+ /// Returns true if the recipe only uses scalars of operand \p Op.
+ bool usesScalars(const VPValue *Op) const override {
+ return Instruction::isCast(getOpcode()) || usesFirstLaneOnly(Op);
+ }
+
/// Returns true if the recipe only uses the first part of operand \p Op.
bool usesFirstPartOnly(const VPValue *Op) const override;
@@ -1522,78 +1519,6 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
#endif
};
-/// A specialization of VPInstruction augmenting it with a dedicated result
-/// type, to be used when the opcode and operands of the VPInstruction don't
-/// directly determine the result type. Note that there is no separate recipe ID
-/// for VPInstructionWithType; it shares the same ID as VPInstruction and is
-/// distinguished purely by the opcode.
-/// TODO: Merge with VPInstruction, now that VPRecipeValue provides the type.
-class VPInstructionWithType : public VPInstruction {
-public:
- VPInstructionWithType(unsigned Opcode, ArrayRef<VPValue *> Operands,
- Type *ResultTy, const VPIRFlags &Flags = {},
- const VPIRMetadata &Metadata = {},
- DebugLoc DL = DebugLoc::getUnknown(),
- const Twine &Name = "", Value *UV = nullptr)
- : VPInstruction(Opcode, Operands, Flags, Metadata, DL, Name, ResultTy) {
- setUnderlyingValue(UV);
- }
-
- static inline bool classof(const VPRecipeBase *R) {
- // VPInstructionWithType are VPInstructions with specific opcodes requiring
- // type information.
- auto *VPI = dyn_cast<VPInstruction>(R);
- if (!VPI)
- return false;
- unsigned Opc = VPI->getOpcode();
- if (Instruction::isCast(Opc))
- return true;
- switch (Opc) {
- case VPInstruction::WideIVStep:
- case VPInstruction::StepVector:
- case VPInstruction::Intrinsic:
- case Instruction::Load:
- return true;
- default:
- return false;
- }
- }
-
- static inline bool classof(const VPUser *R) {
- return isa<VPInstructionWithType>(cast<VPRecipeBase>(R));
- }
-
- VPInstruction *clone() override {
- auto *New =
- new VPInstructionWithType(getOpcode(), operands(), getResultType(),
- *this, *this, getDebugLoc(), getName());
- New->setUnderlyingValue(getUnderlyingValue());
- return New;
- }
-
- void execute(VPTransformState &State) override;
-
- /// Return the cost of this VPInstruction.
- InstructionCost computeCost(ElementCount VF,
- VPCostContext &Ctx) const override;
-
- Type *getResultType() const { return getScalarType(); }
-
- /// Cast recipes always use scalars of their operand.
- bool usesScalars(const VPValue *Op) const override {
- if (Instruction::isCast(getOpcode()))
- return true;
- return VPInstruction::usesScalars(Op);
- }
-
-protected:
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- /// Print the recipe.
- void printRecipe(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override;
-#endif
-};
-
/// Helper type to provide functions to access incoming values and blocks for
/// phi-like recipes.
class VPPhiAccessors {
@@ -3388,7 +3313,7 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
VPIRMetadata(Metadata), IsSingleScalar(IsSingleScalar),
IsPredicated(Mask) {
assert((!IsSingleScalar || !I->isCast()) &&
- "single-scalar casts should use VPInstructionWithType");
+ "Single-scalar casts should use VPInstruction");
setUnderlyingValue(I);
if (Mask)
addOperand(Mask);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 1a135b8549514..4f7f50b6f6fbc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1218,7 +1218,7 @@ static bool areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB, Loop *TheLoop,
const DataLayout &DL = TheLoop->getHeader()->getDataLayout();
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB)) {
for (VPRecipeBase &R : *VPBB) {
- auto *VPI = dyn_cast<VPInstructionWithType>(&R);
+ auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI || VPI->getOpcode() != Instruction::Load) {
assert(!R.mayReadFromMemory() && "unexpected recipe reading memory");
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f545c63d691d9..ab7d486fb2af5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -740,6 +740,16 @@ Value *VPInstruction::generate(VPTransformState &State) {
applyFlags(*I);
return Res;
}
+ if (Instruction::isCast(getOpcode())) {
+ Value *Op = State.get(getOperand(0), VPLane(0));
+ Value *Res = State.Builder.CreateCast(Instruction::CastOps(getOpcode()), Op,
+ getScalarType());
+ if (auto *CastOp = dyn_cast<Instruction>(Res)) {
+ applyFlags(*CastOp);
+ applyMetadata(*CastOp);
+ }
+ return Res;
+ }
switch (getOpcode()) {
case VPInstruction::Not: {
@@ -1105,6 +1115,17 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Result;
}
+ case VPInstruction::StepVector:
+ return State.Builder.CreateStepVector(
+ VectorType::get(getScalarType(), State.VF));
+ case VPInstruction::Intrinsic: {
+ SmallVector<Value *, 2> Args;
+ for (VPValue *Op : drop_end(operands()))
+ Args.push_back(State.get(Op, /*IsSingleScalar=*/true));
+ return State.Builder.CreateIntrinsic(getScalarType(),
+ vputils::getIntrinsicID(this), Args,
+ /*FMFSource=*/nullptr, getName());
+ }
default:
llvm_unreachable("Unsupported opcode for instruction");
}
@@ -1315,6 +1336,12 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
InstructionCost VPInstruction::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
+ // NOTE: At the moment it seems only possible to expose this path for
+ // the trunc, zext and sext opcodes.
+ if (Instruction::isCast(getOpcode()))
+ return getCostForRecipeWithOpcode(getOpcode(), ElementCount::getFixed(1),
+ Ctx);
+
if (Instruction::isBinaryOp(getOpcode())) {
if (!getUnderlyingValue() && getOpcode() != Instruction::FMul) {
// TODO: Compute cost for VPInstructions without underlying values once
@@ -1477,6 +1504,21 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
CmpInst::makeCmpResultType(ValTy),
CmpInst::ICMP_EQ, Ctx.CostKind);
}
+ case VPInstruction::Intrinsic: {
+ Type *Ty = getScalarType();
+ SmallVector<Type *, 2> ArgTys;
+ for (const VPValue *Op : drop_end(operands()))
+ ArgTys.push_back(Op->getScalarType());
+ IntrinsicCostAttributes Attrs(vputils::getIntrinsicID(this), Ty, ArgTys);
+ return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
+ }
+ case VPInstruction::StepVector:
+ // TODO: This isn't quite right since even if the step-vector is hoisted
+ // out of the loop it has a non-zero cost in the middle block, etc.
+ // Once the stepvector is correctly hoisted out of the vector loop by the
+ // licm transform we can add the cost here so that it doesn't incorrectly
+ // affect the choice of VF.
+ return 0;
case Instruction::FCmp:
case Instruction::ICmp:
return getCostForRecipeWithOpcode(
@@ -1845,103 +1887,14 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::NumActiveLanes:
O << "num-active-lanes";
break;
- default:
- O << Instruction::getOpcodeName(getOpcode());
- }
-
- printFlags(O);
- printOperands(O, SlotTracker);
-}
-#endif
-
-void VPInstructionWithType::execute(VPTransformState &State) {
- Type *ResultTy = getResultType();
- if (Instruction::isCast(getOpcode())) {
- Value *Op = State.get(getOperand(0), VPLane(0));
- Value *Cast = State.Builder.CreateCast(Instruction::CastOps(getOpcode()),
- Op, ResultTy);
- if (auto *CastOp = dyn_cast<Instruction>(Cast)) {
- applyFlags(*CastOp);
- applyMetadata(*CastOp);
- }
- State.set(this, Cast, VPLane(0));
- return;
- }
- switch (getOpcode()) {
- case VPInstruction::StepVector: {
- Value *StepVector =
- State.Builder.CreateStepVector(VectorType::get(ResultTy, State.VF));
- State.set(this, StepVector);
- break;
- }
- case VPInstruction::Intrinsic: {
- SmallVector<Value *, 2> Args;
- for (VPValue *Op : drop_end(operands()))
- Args.push_back(State.get(Op, /*IsSingleScalar=*/true));
- Value *Call =
- State.Builder.CreateIntrinsic(ResultTy, vputils::getIntrinsicID(this),
- Args, /*FMFSource=*/nullptr, getName());
- State.set(this, Call, true);
- break;
- }
-
- default:
- llvm_unreachable("opcode not implemented yet");
- }
-}
-
-InstructionCost VPInstructionWithType::computeCost(ElementCount VF,
- VPCostContext &Ctx) const {
- // NOTE: At the moment it seems only possible to expose this path for
- // the trunc, zext and sext opcodes. However, isScalarCast also covers
- // int<>fp conversions, bitcasts, ptr<>int conversions, etc.
- if (Instruction::isCast(getOpcode()))
- return getCostForRecipeWithOpcode(getOpcode(), ElementCount::getFixed(1),
- Ctx);
-
- switch (getOpcode()) {
- case VPInstruction::StepVector:
- // TODO: This isn't quite right since even if the step-vector is hoisted
- // out of the loop it has a non-zero cost in the middle block, etc.
- // Once the stepvector is correctly hoisted out of the vector loop by the
- // licm transform we can add the cost here so that it doesn't incorrectly
- // affect the choice of VF.
- return 0;
- case VPInstruction::Intrinsic: {
- Type *Ty = getScalarType();
- SmallVector<Type *, 2> ArgTys;
- for (const VPValue *Op : drop_end(operands()))
- ArgTys.push_back(Op->getScalarType());
- IntrinsicCostAttributes Attrs(vputils::getIntrinsicID(this), Ty, ArgTys);
- return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
- }
- default:
- // Although VPInstructionWithType is also used for
- // VPInstruction::WideIVStep it isn't currently possible to expose cases
- // where the cost is queried.
- llvm_unreachable("Unhandled opcode");
- }
- return 0;
-}
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void VPInstructionWithType::printRecipe(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const {
- O << Indent << "EMIT" << (isSingleScalar() ? "-SCALAR" : "") << " ";
- printAsOperand(O, SlotTracker);
- O << " = ";
-
- Type *ResultTy = getResultType();
- switch (getOpcode()) {
case VPInstruction::WideIVStep:
- O << "wide-iv-step ";
- printOperands(O, SlotTracker);
+ O << "wide-iv-step";
break;
case VPInstruction::StepVector:
- O << "step-vector " << *ResultTy;
+ O << "step-vector " << *getScalarType();
break;
case VPInstruction::Intrinsic: {
- O << "call " << *ResultTy << " @"
+ O << "call " << *getScalarType() << " @"
<< Intrinsic::getBaseName(vputils::getIntrinsicID(this)) << "(";
interleaveComma(drop_end(operands()), O, [&O, &SlotTracker](VPValue *Op) {
Op->printAsOperand(O, SlotTracker);
@@ -1950,16 +1903,18 @@ void VPInstructionWithType::printRecipe(raw_ostream &O, const Twine &Indent,
break;
}
case Instruction::Load:
- O << "load ";
- printOperands(O, SlotTracker);
+ O << "load";
break;
default:
- assert(Instruction::isCast(getOpcode()) && "unhandled opcode");
O << Instruction::getOpcodeName(getOpcode());
+ }
+
+ if (!operands_empty()) {
printFlags(O);
printOperands(O, SlotTracker);
- O << " to " << *ResultTy;
}
+ if (Instruction::isCast(getOpcode()))
+ O << " to " << *getScalarType();
}
#endif
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index a430de94cca14..b00a6567ae28f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -958,7 +958,7 @@ void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {
DefR->replaceUsesWithIf(LaneDefs[0], [DefR](VPUser &U, unsigned) {
if (U.usesFirstLaneOnly(DefR))
return true;
- auto *VPI = dyn_cast<VPInstructionWithType>(&U);
+ auto *VPI = dyn_cast<VPInstruction>(&U);
return VPI && Instruction::isCast(VPI->getOpcode());
});
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 738902519fe90..1c8284d30b010 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -473,6 +473,7 @@ class LLVM_ABI_FOR_TEST VPUser {
const_operand_range operands() const {
return const_operand_range(op_begin(), op_end());
}
+ bool operands_empty() const { return Operands.empty(); } // NOLINT
/// Returns true if the VPUser uses scalars of operand \p Op. Conservatively
/// returns if only first (scalar) lane is used, as default.
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll
index a743dfc0583f2..6237aab23dc55 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-flags.ll
@@ -27,18 +27,18 @@ define void @cast_flags_mixed(ptr noalias %A, ptr noalias %B) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>, vp<[[VP5]]>, vp<%step.add>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = WIDEN-INDUCTION nuw ir<0>, ir<1>, vp<[[VP0]]>, vp<[[VP6]]>, vp<%step.add>.1
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = WIDEN-INDUCTION nuw ir<0>, ir<1>, vp<[[VP0]]>, vp<[[VP6]]>, vp<%step.add>.1
; CHECK-NEXT: EMIT vp<%step.add> = add nsw ir<%iv>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT vp<%step.add>.1 = add nuw vp<[[VP8]]>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp ule vp<[[VP8]]>, vp<[[VP3]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp ule vp<%step.add>.1, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<%step.add>.1 = add nuw vp<[[VP9]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp ule vp<[[VP9]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp ule vp<%step.add>.1, vp<[[VP3]]>
; CHECK-NEXT: WIDEN-CAST ir<%sext.plain> = sext ir<%iv> to i64
; CHECK-NEXT: WIDEN-CAST ir<%sext.plain>.1 = sext vp<%step.add> to i64
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP10]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -53,7 +53,7 @@ define void @cast_flags_mixed(ptr noalias %A, ptr noalias %B) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP10]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP11]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -117,11 +117,11 @@ define void @cast_flags_single(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = WIDEN-INDUCTION nuw ir<0>, ir<1>, vp<[[VP0]]>, vp<[[VP5]]>, vp<%step.add>
-; CHECK-NEXT: EMIT vp<%step.add> = add nuw vp<[[VP7]]>, vp<[[VP5]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6]]> * ir<1>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = WIDEN-INDUCTION nuw ir<0>, ir<1>, vp<[[VP0]]>, vp<[[VP5]]>, vp<%step.add>
+; CHECK-NEXT: EMIT vp<%step.add> = add nuw vp<[[VP8]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp ule vp<[[VP8]]>, vp<[[VP3]]>
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp ule vp<%step.add>, vp<[[VP3]]>
+; CHECK-NEXT: vp<[[VP11:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6]]> * ir<1>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -130,11 +130,11 @@ define void @cast_flags_single(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: vp<[[VP11:%[0-9]+]]> = SCALAR-STEPS vp<[[VP8]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<%A>, vp<[[VP11]]>
+; CHECK-NEXT: vp<[[VP12:%[0-9]+]]> = SCALAR-STEPS vp<[[VP11]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<%A>, vp<[[VP12]]>
; CHECK-NEXT: REPLICATE store ir<3>, ir<%gep.a>
; CHECK-NEXT: REPLICATE store ir<3>, ir<%gep.a>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<%B>, vp<[[VP11]]>
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<%B>, vp<[[VP12]]>
; CHECK-NEXT: REPLICATE store ir<3>, ir<%gep.b>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
@@ -149,12 +149,12 @@ define void @cast_flags_single(ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP12:%[0-9]+]]> = trunc vp<[[VP0]]> to i16
-; CHECK-NEXT: vp<[[VP13:%[0-9]+]]> = SCALAR-STEPS vp<[[VP8]]>, ir<1>, vp<[[VP0]]>, vp<[[VP12]]>
-; CHECK-NEXT: REPLICATE ir<%gep.a>.1 = getelementptr ir<%A>, vp<[[VP13]]>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP13:%[0-9]+]]> = trunc vp<[[VP0]]> to i16
+; CHECK-NEXT: vp<[[VP14:%[0-9]+]]> = SCALAR-STEPS vp<[[VP11]]>, ir<1>, vp<[[VP0]]>, vp<[[VP13]]>
+; CHECK-NEXT: REPLICATE ir<%gep.a>.1 = getelementptr ir<%A>, vp<[[VP14]]>
; CHECK-NEXT: REPLICATE store ir<3>, ir<%gep.a>.1
; CHECK-NEXT: REPLICATE store ir<3>, ir<%gep.a>.1
-; CHECK-NEXT: REPLICATE ir<%gep.b>.1 = getelementptr ir<%B>, vp<[[VP13]]>
+; CHECK-NEXT: REPLICATE ir<%gep.b>.1 = getelementptr ir<%B>, vp<[[VP14]]>
; CHECK-NEXT: REPLICATE store ir<3>, ir<%gep.b>.1
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
@@ -208,7 +208,7 @@ define void @fpiv_wideivstep_flags(float %init, ptr %p, i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%init> + vp<[[VP2]]> * ir<1.000000e+00>
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = wide-iv-step vp<[[VP0]]>, ir<1.000000e+00>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = wide-iv-step fast vp<[[VP0]]>, ir<1.000000e+00>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
diff --git a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
index 321ea74f6d7b1..85cec18d8a4d8 100644
--- a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
@@ -239,29 +239,29 @@ TEST_F(VPDominatorTreeTest, DominanceRegionsTest) {
IntegerType *Int32 = IntegerType::get(C, 32);
VPBasicBlock *R1BB1 = Plan.createVPBasicBlock("R1BB1");
VPInstruction *R1BB1I =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
R1BB1->appendRecipe(R1BB1I);
VPBasicBlock *R1BB2 = Plan.createVPBasicBlock("R1BB2");
VPInstruction *R1BB2I =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
R1BB2->appendRecipe(R1BB2I);
VPBasicBlock *R1BB3 = Plan.createVPBasicBlock("R1BB3");
VPInstruction *R1BB3I =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
R1BB3->appendRecipe(R1BB3I);
VPRegionBlock *R1 = Plan.createReplicateRegion(R1BB1, R1BB3, "R1");
VPBasicBlock *R2BB1 = Plan.createVPBasicBlock("R2BB1");
VPInstruction *R2BB1I =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
R2BB1->appendRecipe(R2BB1I);
VPBasicBlock *R2BB2 = Plan.createVPBasicBlock("R2BB2");
VPInstruction *R2BB2I =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
R2BB2->appendRecipe(R2BB2I);
VPBasicBlock *R2BB3 = Plan.createVPBasicBlock("R2BB3");
VPInstruction *R2BB3I =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
R2BB3->appendRecipe(R2BB3I);
VPRegionBlock *R2 = Plan.createReplicateRegion(R2BB1, R2BB3, "R2");
R2BB2->setParent(R2);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index f10435dfb2da6..9191372dc3071 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -76,11 +76,11 @@ define void @f(i32 %x) {
TEST_F(VPInstructionTest, insertBefore) {
IntegerType *Int32 = IntegerType::get(C, 32);
VPInstruction *I1 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPInstruction *I2 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPInstruction *I3 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPBasicBlock &VPBB1 = *getPlan().createVPBasicBlock("");
VPBB1.appendRecipe(I1);
@@ -95,11 +95,11 @@ TEST_F(VPInstructionTest, insertBefore) {
TEST_F(VPInstructionTest, eraseFromParent) {
IntegerType *Int32 = IntegerType::get(C, 32);
VPInstruction *I1 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPInstruction *I2 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPInstruction *I3 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPBasicBlock &VPBB1 = *getPlan().createVPBasicBlock("");
VPBB1.appendRecipe(I1);
@@ -119,11 +119,11 @@ TEST_F(VPInstructionTest, eraseFromParent) {
TEST_F(VPInstructionTest, moveAfter) {
IntegerType *Int32 = IntegerType::get(C, 32);
VPInstruction *I1 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPInstruction *I2 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPInstruction *I3 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPBasicBlock &VPBB1 = *getPlan().createVPBasicBlock("");
VPBB1.appendRecipe(I1);
@@ -135,9 +135,9 @@ TEST_F(VPInstructionTest, moveAfter) {
CHECK_ITERATOR(VPBB1, I2, I1, I3);
VPInstruction *I4 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPInstruction *I5 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPBasicBlock &VPBB2 = *getPlan().createVPBasicBlock("");
VPBB2.appendRecipe(I4);
VPBB2.appendRecipe(I5);
@@ -152,11 +152,11 @@ TEST_F(VPInstructionTest, moveAfter) {
TEST_F(VPInstructionTest, moveBefore) {
IntegerType *Int32 = IntegerType::get(C, 32);
VPInstruction *I1 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPInstruction *I2 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPInstruction *I3 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPBasicBlock &VPBB1 = *getPlan().createVPBasicBlock("");
VPBB1.appendRecipe(I1);
@@ -168,9 +168,9 @@ TEST_F(VPInstructionTest, moveBefore) {
CHECK_ITERATOR(VPBB1, I2, I1, I3);
VPInstruction *I4 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPInstruction *I5 =
- new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
+ new VPInstruction(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPBasicBlock &VPBB2 = *getPlan().createVPBasicBlock("");
VPBB2.appendRecipe(I4);
VPBB2.appendRecipe(I5);
@@ -815,8 +815,8 @@ TEST_F(VPBasicBlockTest, reassociateBlocks) {
TEST_F(VPBasicBlockTest, splitAtEnd) {
VPlan &Plan = getPlan();
- VPInstruction *VPI = new VPInstructionWithType(VPInstruction::StepVector, {},
- IntegerType::get(C, 32));
+ VPInstruction *VPI = new VPInstruction(VPInstruction::StepVector, {}, {}, {},
+ {}, "", IntegerType::get(C, 32));
VPBasicBlock *VPBB = Plan.createVPBasicBlock("VPBB1", VPI);
VPBlockUtils::connectBlocks(Plan.getEntry(), VPBB);
VPBlockUtils::connectBlocks(VPBB, Plan.getScalarHeader());
@@ -1785,8 +1785,8 @@ TEST(VPDoubleValueDefTest, traverseUseLists) {
// Create a new VPRecipeBase which defines 2 values and has 2 operands.
LLVMContext C;
IntegerType *Int32 = IntegerType::get(C, 32);
- VPInstructionWithType Op0(VPInstruction::StepVector, {}, Int32);
- VPInstructionWithType Op1(VPInstruction::StepVector, {}, Int32);
+ VPInstruction Op0(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
+ VPInstruction Op1(VPInstruction::StepVector, {}, {}, {}, {}, "", Int32);
VPDoubleValueDef DoubleValueDef({&Op0, &Op1}, IntegerType::get(C, 32));
// Create a new users of the defined values.
>From 0caba9d642b4041396412538369f4a9cee3210b4 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Fri, 26 Jun 2026 18:45:15 +0100
Subject: [PATCH 4/6] [VPlan] To-revert bad change
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 ++--
.../RISCV/scatter-type-mismatch.ll | 27 ++++++-------------
2 files changed, 11 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ab7d486fb2af5..866ae7eceadce 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1339,8 +1339,9 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
// NOTE: At the moment it seems only possible to expose this path for
// the trunc, zext and sext opcodes.
if (Instruction::isCast(getOpcode()))
- return getCostForRecipeWithOpcode(getOpcode(), ElementCount::getFixed(1),
- Ctx);
+ return getCostForRecipeWithOpcode(
+ getOpcode(),
+ vputils::onlyFirstLaneUsed(this) ? ElementCount::getFixed(1) : VF, Ctx);
if (Instruction::isBinaryOp(getOpcode())) {
if (!getUnderlyingValue() && getOpcode() != Instruction::FMul) {
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scatter-type-mismatch.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scatter-type-mismatch.ll
index 1de9b39b49052..bead7ce2d91e7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/scatter-type-mismatch.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/scatter-type-mismatch.ll
@@ -7,27 +7,16 @@
define void @scatter_gepty_mismatch_storety(ptr %p) {
; CHECK-LABEL: define void @scatter_gepty_mismatch_storety(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT: [[VECTOR_PH:.*]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 128, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr double, ptr [[P]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> zeroinitializer, <vscale x 4 x ptr> align 4 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i8.nxv4p0(<vscale x 4 x i8> zeroinitializer, <vscale x 4 x ptr> align 1 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP2]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr double, ptr [[P]], i64 [[IV]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[GEP]], align 4
+; CHECK-NEXT: store i8 0, ptr [[GEP]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 128
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[VECTOR_BODY]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
>From 3a2dbbf87df6a74e3192d5d6a08ad01f93ee22d1 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Fri, 26 Jun 2026 18:45:39 +0100
Subject: [PATCH 5/6] [VPlan] Revert bad change
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 ++--
.../RISCV/scatter-type-mismatch.ll | 27 +++++++++++++------
2 files changed, 21 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 866ae7eceadce..ab7d486fb2af5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1339,9 +1339,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
// NOTE: At the moment it seems only possible to expose this path for
// the trunc, zext and sext opcodes.
if (Instruction::isCast(getOpcode()))
- return getCostForRecipeWithOpcode(
- getOpcode(),
- vputils::onlyFirstLaneUsed(this) ? ElementCount::getFixed(1) : VF, Ctx);
+ return getCostForRecipeWithOpcode(getOpcode(), ElementCount::getFixed(1),
+ Ctx);
if (Instruction::isBinaryOp(getOpcode())) {
if (!getUnderlyingValue() && getOpcode() != Instruction::FMul) {
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scatter-type-mismatch.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scatter-type-mismatch.ll
index bead7ce2d91e7..1de9b39b49052 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/scatter-type-mismatch.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/scatter-type-mismatch.ll
@@ -7,16 +7,27 @@
define void @scatter_gepty_mismatch_storety(ptr %p) {
; CHECK-LABEL: define void @scatter_gepty_mismatch_storety(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[VECTOR_PH:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr double, ptr [[P]], i64 [[IV]]
-; CHECK-NEXT: store float 0.000000e+00, ptr [[GEP]], align 4
-; CHECK-NEXT: store i8 0, ptr [[GEP]], align 1
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 128
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[VECTOR_BODY]]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 128, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr double, ptr [[P]], <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> zeroinitializer, <vscale x 4 x ptr> align 4 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i8.nxv4p0(<vscale x 4 x i8> zeroinitializer, <vscale x 4 x ptr> align 1 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP2]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
>From ac69df57d5840cd7197c0f0c55eaeec6000f1fcc Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 1 Jul 2026 10:44:12 +0100
Subject: [PATCH 6/6] [VPlan] Put back isSingleScalar-usesScalars change
---
llvm/lib/Transforms/Vectorize/VPlan.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 8736617842b94..ab844690ac308 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1492,7 +1492,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Returns true if the recipe only uses scalars of operand \p Op.
bool usesScalars(const VPValue *Op) const override {
- return Instruction::isCast(getOpcode()) || usesFirstLaneOnly(Op);
+ return isSingleScalar() || usesFirstLaneOnly(Op);
}
/// Returns true if the recipe only uses the first part of operand \p Op.
More information about the llvm-commits
mailing list