[llvm] [RISCV] Enable shouldMaximizeVectorBandwidth (PR #216005)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 13 03:39:27 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Pengcheng Wang (wangpc-pp)
<details>
<summary>Changes</summary>
In this PR, we size vector factors by the smallest type in the loop
rather than the widest, matching AArch64's behaviour. This lets
loops that mix narrow and wide types use the full vector register
bandwidth instead of being limited by the widest type.
Overly wide vector factors are pruned by register pressure estimation,
which RISC-V already opts into via shouldConsiderVectorizationRegPressure,
so this does not force excessively wide factors.
Assisted-by: TRAE CLI (Opus 4.8)
---
Patch is 174.92 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/216005.diff
16 Files Affected:
- (modified) llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp (+9)
- (modified) llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h (+3)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll (+8-8)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll (+59-36)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll (+55-15)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll (+90-52)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll (+11-11)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll (+14-14)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll (+15-15)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll (+15-15)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll (+240-280)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll (+16-16)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll (+48-48)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll (+23-23)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll (+7-7)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll (+11-11)
``````````diff
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 0f636d078324c..c263c978d2456 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -415,6 +415,15 @@ RISCVTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
llvm_unreachable("Unsupported register kind");
}
+bool RISCVTTIImpl::shouldMaximizeVectorBandwidth(
+ TargetTransformInfo::RegisterKind K) const {
+ // Size vector factors by the smallest type in the loop rather than the
+ // widest, so loops mixing narrow and wide types can use the full vector
+ // register bandwidth. Overly wide factors are pruned separately by register
+ // pressure estimation (shouldConsiderVectorizationRegPressure).
+ return K != TargetTransformInfo::RGK_Scalar && ST->hasVInstructions();
+}
+
InstructionCost RISCVTTIImpl::getStaticDataAddrGenerationCost(
const TTI::TargetCostKind CostKind) const {
switch (CostKind) {
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 40c6204ee0380..5f0dc2613925f 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -134,6 +134,9 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
TypeSize
getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override;
+ bool shouldMaximizeVectorBandwidth(
+ TargetTransformInfo::RegisterKind K) const override;
+
unsigned getRegUsageForType(Type *Ty) const override;
unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const override;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
index 9951165b77da6..c1c0acab2afaf 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
@@ -124,17 +124,17 @@ define void @vfwmaccbf16.vv(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64
; ZVFBFMIN: [[VECTOR_BODY]]:
; ZVFBFMIN-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; ZVFBFMIN-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; ZVFBFMIN-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; ZVFBFMIN-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
; ZVFBFMIN-NEXT: [[TMP1:%.*]] = getelementptr bfloat, ptr [[A]], i64 [[INDEX]]
; ZVFBFMIN-NEXT: [[TMP2:%.*]] = getelementptr bfloat, ptr [[B]], i64 [[INDEX]]
; ZVFBFMIN-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[C]], i64 [[INDEX]]
-; ZVFBFMIN-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x bfloat> @llvm.vp.load.nxv4bf16.p0(ptr align 2 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; ZVFBFMIN-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x bfloat> @llvm.vp.load.nxv4bf16.p0(ptr align 2 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; ZVFBFMIN-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; ZVFBFMIN-NEXT: [[TMP4:%.*]] = fpext <vscale x 4 x bfloat> [[VP_OP_LOAD]] to <vscale x 4 x float>
-; ZVFBFMIN-NEXT: [[TMP5:%.*]] = fpext <vscale x 4 x bfloat> [[VP_OP_LOAD1]] to <vscale x 4 x float>
-; ZVFBFMIN-NEXT: [[TMP6:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], <vscale x 4 x float> [[VP_OP_LOAD2]])
-; ZVFBFMIN-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP6]], ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVFBFMIN-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 8 x bfloat> @llvm.vp.load.nxv8bf16.p0(ptr align 2 [[TMP1]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVFBFMIN-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 8 x bfloat> @llvm.vp.load.nxv8bf16.p0(ptr align 2 [[TMP2]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVFBFMIN-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 8 x float> @llvm.vp.load.nxv8f32.p0(ptr align 4 [[TMP3]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVFBFMIN-NEXT: [[TMP4:%.*]] = fpext <vscale x 8 x bfloat> [[VP_OP_LOAD]] to <vscale x 8 x float>
+; ZVFBFMIN-NEXT: [[TMP5:%.*]] = fpext <vscale x 8 x bfloat> [[VP_OP_LOAD1]] to <vscale x 8 x float>
+; ZVFBFMIN-NEXT: [[TMP6:%.*]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[TMP4]], <vscale x 8 x float> [[TMP5]], <vscale x 8 x float> [[VP_OP_LOAD2]])
+; ZVFBFMIN-NEXT: call void @llvm.vp.store.nxv8f32.p0(<vscale x 8 x float> [[TMP6]], ptr align 4 [[TMP3]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
; ZVFBFMIN-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64
; ZVFBFMIN-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP7]], [[INDEX]]
; ZVFBFMIN-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
index 9f7ac7ac1771f..a16b284f3f4c8 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
@@ -8,27 +8,38 @@ target triple = "riscv64-unknown-linux-gnu"
define i32 @any_of_reduction_used_in_blend(ptr %src, i64 %N, i1 %c.0, i1 %c.1) #0 {
; CHECK-LABEL: define i32 @any_of_reduction_used_in_blend(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]], i1 [[C_0:%.*]], i1 [[C_1:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[ANY_OF_RED:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ANY_OF_RED_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH]] ]
-; CHECK-NEXT: br i1 [[C_0]], label %[[LOOP_LATCH]], label %[[ELSE_1:.*]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C_0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = select <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = select <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = or <vscale x 4 x i1> [[TMP3]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[SRC]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[ELSE_1:.*]]
; CHECK: [[ELSE_1]]:
-; CHECK-NEXT: br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[ELSE_2:.*]]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[LOOP_HEADER]] ], [ [[TMP8:%.*]], %[[ELSE_1]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[LOOP_HEADER]] ], [ [[AVL_NEXT:%.*]], %[[ELSE_1]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x ptr> @llvm.vp.gather.nxv4p0.nxv4p0(<vscale x 4 x ptr> align 8 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP5]])
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <vscale x 4 x ptr> [[WIDE_MASKED_GATHER]], splat (ptr null)
+; CHECK-NEXT: [[TMP7:%.*]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP6]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> [[VEC_PHI]], <vscale x 4 x i1> [[TMP7]]
+; CHECK-NEXT: [[TMP8]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[PREDPHI]], <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP5]])
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP10]], label %[[ELSE_2:.*]], label %[[ELSE_1]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[ELSE_2]]:
-; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[SRC]], align 8
-; CHECK-NEXT: [[C_2:%.*]] = icmp eq ptr [[L]], null
-; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C_2]], i32 0, i32 [[ANY_OF_RED]]
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK-NEXT: br label %[[LOOP_LATCH:.*]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[ANY_OF_RED_NEXT]] = phi i32 [ [[ANY_OF_RED]], %[[LOOP_HEADER]] ], [ [[ANY_OF_RED]], %[[ELSE_1]] ], [ [[SEL]], %[[ELSE_2]] ]
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[ANY_OF_RED_NEXT]], %[[LOOP_LATCH]] ]
-; CHECK-NEXT: ret i32 [[RES]]
+; CHECK-NEXT: ret i32 0
;
entry:
br label %loop.header
@@ -61,30 +72,36 @@ exit:
define i32 @any_of_reduction_used_in_blend_with_multiple_phis(ptr %src, i64 %N, i1 %c.0, i1 %c.1) #0 {
; CHECK-LABEL: define i32 @any_of_reduction_used_in_blend_with_multiple_phis(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]], i1 [[C_0:%.*]], i1 [[C_1:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[ANY_OF_RED:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ANY_OF_RED_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH]] ]
-; CHECK-NEXT: br i1 [[C_0]], label %[[X_1:.*]], label %[[ELSE_1:.*]]
-; CHECK: [[ELSE_1]]:
-; CHECK-NEXT: br i1 [[C_1]], label %[[X_1]], label %[[ELSE_2:.*]]
-; CHECK: [[ELSE_2]]:
-; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[SRC]], align 8
-; CHECK-NEXT: [[C_2:%.*]] = icmp eq ptr [[L]], null
-; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C_2]], i32 0, i32 [[ANY_OF_RED]]
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
-; CHECK: [[X_1]]:
-; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[ANY_OF_RED]], %[[LOOP_HEADER]] ], [ [[ANY_OF_RED]], %[[ELSE_1]] ]
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C_0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = select <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[SRC]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[LOOP_LATCH:.*]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[ANY_OF_RED_NEXT]] = phi i32 [ [[P]], %[[X_1]] ], [ [[SEL]], %[[ELSE_2]] ]
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[LOOP_HEADER]] ], [ [[TMP7:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[LOOP_HEADER]] ], [ [[AVL_NEXT:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP3]])
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x ptr> @llvm.vp.gather.nxv4p0.nxv4p0(<vscale x 4 x ptr> align 8 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <vscale x 4 x ptr> [[WIDE_MASKED_GATHER]], splat (ptr null)
+; CHECK-NEXT: [[TMP6:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP3]] to i64
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_LATCH]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[ANY_OF_RED_NEXT]], %[[LOOP_LATCH]] ]
-; CHECK-NEXT: ret i32 [[RES]]
+; CHECK-NEXT: ret i32 0
;
entry:
br label %loop.header
@@ -119,3 +136,9 @@ exit:
}
attributes #0 = { "target-cpu"="sifive-p670" }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 648bd292a1e2c..1a5e3472335b2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -157,15 +157,15 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP2]], 7
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP2]], 15
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i32 8, i32 [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i32 16, i32 [[N_MOD_VF]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[TMP9]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i1> poison, i1 [[C]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT]], <16 x i1> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE18:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE32:.*]] ]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[B]], i32 [[INDEX]]
; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
@@ -202,13 +202,53 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE16]]
; CHECK: [[PRED_STORE_CONTINUE16]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18]]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
; CHECK: [[PRED_STORE_IF17]]:
; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE18]]
; CHECK: [[PRED_STORE_CONTINUE18]]:
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[TMP11]], <8 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META13]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF18:.*]], label %[[PRED_STORE_CONTINUE19:.*]]
+; CHECK: [[PRED_STORE_IF18]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE19]]
+; CHECK: [[PRED_STORE_CONTINUE19]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
+; CHECK: [[PRED_STORE_IF19]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE20]]
+; CHECK: [[PRED_STORE_CONTINUE20]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]
+; CHECK: [[PRED_STORE_IF21]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE22]]
+; CHECK: [[PRED_STORE_CONTINUE22]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]
+; CHECK: [[PRED_STORE_IF23]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE24]]
+; CHECK: [[PRED_STORE_CONTINUE24]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]
+; CHECK: [[PRED_STORE_IF25]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE26]]
+; CHECK: [[PRED_STORE_CONTINUE26]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]
+; CHECK: [[PRED_STORE_IF27]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE28]]
+; CHECK: [[PRED_STORE_CONTINUE28]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]
+; CHECK: [[PRED_STORE_IF29]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE30]]
+; CHECK: [[PRED_STORE_CONTINUE30]]:
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF31:.*]], label %[[PRED_STORE_CONTINUE32]]
+; CHECK: [[PRED_STORE_IF31]]:
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE32]]
+; CHECK: [[PRED_STORE_CONTINUE32]]:
+; CHECK-NEXT: call void @llvm.masked.store.v16i32.p0(<16 x i32> zeroinitializer, ptr align 4 [[TMP11]], <16 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META13]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[T...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/216005
More information about the llvm-commits
mailing list