[llvm-branch-commits] [llvm] [VPlan] Fix find last reductions not using frozen condition in select (PR #223962)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Sep 16 03:20:18 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-risc-v
Author: Luke Lau (lukel97)
<details>
<summary>Changes</summary>
A find last reduction is something like:
(select c[n], x[n], (select c[n-1], x[n-1], (select ...)))
So even if previous iterations had poison for c[x], the top-level select blocks it.
MaskSelect uses the non-frozen Cond, and that in turn is fed into VPInstruction::ExtractLastActive, which in turn gets expanded to vector.reduce.umax, which returns poison if any of the lanes were poison.
So in the case e.g. c[n] = 1, c[n-1] = poison and VF=2, the scalar loop returns x[n] whilst vector loop returns poison.
We're also introducing multiple uses of Cond which isn't sound for undef, so for these reasons we need to use the frozen Cond.
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
---
Patch is 181.20 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/223962.diff
25 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp (+2-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment-fold-tail.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll (+35-35)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/findlast-epilogue-loop.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-select-cmp.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/conditional-scalar-assignment-fold-tail.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/conditional-scalar-assignment.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/conditional-scalar-assignment.ll (+28-28)
- (modified) llvm/test/Transforms/LoopVectorize/alias-mask.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/conditional-scalar-assignment-interleave-only.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll (+10-10)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll (+15-15)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-load.ll (+18-18)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-ptr-induction.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/find-last.ll (+6-6)
- (modified) llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll (+16-16)
- (modified) llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll (+87-87)
- (modified) llvm/test/Transforms/LoopVectorize/iv-select-cmp.ll (+9-9)
- (modified) llvm/test/Transforms/LoopVectorize/select-cmp.ll (+34-34)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 3adca452c117a..f69018b52a3f3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1876,9 +1876,8 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) {
if (HeaderMask)
Cond = Builder.createLogicalAnd(HeaderMask, Cond);
- VPValue *AnyOf =
- Builder.createNaryOp(VPInstruction::AnyOf, Builder.createFreeze(Cond));
- // FIXME: The Cond here needs to be frozen too.
+ Cond = Builder.createFreeze(Cond);
+ VPValue *AnyOf = Builder.createNaryOp(VPInstruction::AnyOf, Cond);
VPValue *MaskSelect = Builder.createSelect(AnyOf, Cond, MaskPHI);
MaskPHI->addIncoming(MaskSelect);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment-fold-tail.ll
index 8c98cd2e9663d..ca1c1080046d3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment-fold-tail.ll
@@ -19,14 +19,14 @@ define i32 @simple_find_last_reduction(i64 %N, ptr %data, i32 %a) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP7:%.*]] = icmp slt <vscale x 4 x i32> [[BROADCAST_SPLAT]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP9]])
-; CHECK-NEXT: [[TMP11]] = select i1 [[TMP10]], <vscale x 4 x i1> [[TMP8]], <vscale x 4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP11]] = select i1 [[TMP10]], <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP12]] = select i1 [[TMP10]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
@@ -94,7 +94,7 @@ define i32 @non_speculatable_find_last_reduction(ptr noalias %a, ptr noalias %b,
; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP9]], <vscale x 4 x i1> [[TMP8]], <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP11:%.*]] = freeze <vscale x 4 x i1> [[TMP8]]
; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP11]])
-; CHECK-NEXT: [[TMP13]] = select i1 [[TMP12]], <vscale x 4 x i1> [[TMP8]], <vscale x 4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP13]] = select i1 [[TMP12]], <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP14]] = select i1 [[TMP12]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD3]], <vscale x 4 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
index ee5b164eae8c3..b0f7bf5586b92 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
@@ -39,13 +39,13 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; SVE: [[VECTOR_BODY]]:
; SVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SVE-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; SVE-NEXT: [[TMP4:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; SVE-NEXT: [[TMP2:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
; SVE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[INDEX]]
; SVE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
; SVE-NEXT: [[TMP13:%.*]] = icmp slt <vscale x 4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
; SVE-NEXT: [[TMP7:%.*]] = freeze <vscale x 4 x i1> [[TMP13]]
; SVE-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP7]])
-; SVE-NEXT: [[TMP9]] = select i1 [[TMP8]], <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i1> [[TMP4]]
+; SVE-NEXT: [[TMP9]] = select i1 [[TMP8]], <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i1> [[TMP2]]
; SVE-NEXT: [[TMP10]] = select i1 [[TMP8]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
; SVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; SVE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -128,14 +128,14 @@ define ptr @simple_csa_ptr_select(i64 %N, ptr %data, i64 %a, ptr %init) {
; SVE: [[LOOP]]:
; SVE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
; SVE-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x ptr> [ [[BROADCAST_SPLAT2]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[LOOP]] ]
-; SVE-NEXT: [[TMP4:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[LOOP]] ]
+; SVE-NEXT: [[TMP2:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[LOOP]] ]
; SVE-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds ptr, ptr [[DATA]], i64 [[IV]]
; SVE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x ptr>, ptr [[LD_ADDR]], align 4
; SVE-NEXT: [[TMP6:%.*]] = ptrtoint <vscale x 2 x ptr> [[WIDE_LOAD]] to <vscale x 2 x i64>
; SVE-NEXT: [[TMP7:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP6]]
; SVE-NEXT: [[TMP8:%.*]] = freeze <vscale x 2 x i1> [[TMP7]]
; SVE-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP8]])
-; SVE-NEXT: [[TMP10]] = select i1 [[TMP9]], <vscale x 2 x i1> [[TMP7]], <vscale x 2 x i1> [[TMP4]]
+; SVE-NEXT: [[TMP10]] = select i1 [[TMP9]], <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i1> [[TMP2]]
; SVE-NEXT: [[TMP11]] = select i1 [[TMP9]], <vscale x 2 x ptr> [[WIDE_LOAD]], <vscale x 2 x ptr> [[VEC_PHI]]
; SVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; SVE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -198,13 +198,13 @@ define float @simple_csa_float_select(i64 %N, ptr %data, float %a) {
; NEON: [[LOOP]]:
; NEON-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
; NEON-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+00), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[LOOP]] ]
-; NEON-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[LOOP]] ]
+; NEON-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[LOOP]] ]
; NEON-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds float, ptr [[DATA]], i64 [[IV]]
; NEON-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[LD_ADDR]], align 4
; NEON-NEXT: [[TMP2:%.*]] = fcmp olt <4 x float> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
; NEON-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
; NEON-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
-; NEON-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP2]], <4 x i1> [[TMP0]]
+; NEON-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP3]], <4 x i1> [[TMP1]]
; NEON-NEXT: [[TMP6]] = select i1 [[TMP4]], <4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]]
; NEON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; NEON-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -247,13 +247,13 @@ define float @simple_csa_float_select(i64 %N, ptr %data, float %a) {
; SVE: [[LOOP]]:
; SVE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
; SVE-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ splat (float -1.000000e+00), %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[LOOP]] ]
-; SVE-NEXT: [[TMP4:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[LOOP]] ]
+; SVE-NEXT: [[TMP2:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[LOOP]] ]
; SVE-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds float, ptr [[DATA]], i64 [[IV]]
; SVE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[LD_ADDR]], align 4
; SVE-NEXT: [[TMP6:%.*]] = fcmp olt <vscale x 4 x float> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
; SVE-NEXT: [[TMP7:%.*]] = freeze <vscale x 4 x i1> [[TMP6]]
; SVE-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP7]])
-; SVE-NEXT: [[TMP9]] = select i1 [[TMP8]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> [[TMP4]]
+; SVE-NEXT: [[TMP9]] = select i1 [[TMP8]], <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i1> [[TMP2]]
; SVE-NEXT: [[TMP10]] = select i1 [[TMP8]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[VEC_PHI]]
; SVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; SVE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -404,14 +404,14 @@ define i32 @multi_use_cmp_for_csa_int_select(i64 %N, ptr %data, i32 %a) {
; SVE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
; SVE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
; SVE-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[LOOP]] ]
-; SVE-NEXT: [[TMP5:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[LOOP]] ]
+; SVE-NEXT: [[TMP3:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[LOOP]] ]
; SVE-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[LOOP]] ]
; SVE-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[IV]]
; SVE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[LD_ADDR]], align 4
; SVE-NEXT: [[TMP7:%.*]] = icmp slt <vscale x 4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
; SVE-NEXT: [[TMP8:%.*]] = freeze <vscale x 4 x i1> [[TMP7]]
; SVE-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP8]])
-; SVE-NEXT: [[TMP10]] = select i1 [[TMP9]], <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i1> [[TMP5]]
+; SVE-NEXT: [[TMP10]] = select i1 [[TMP9]], <vscale x 4 x i1> [[TMP8]], <vscale x 4 x i1> [[TMP3]]
; SVE-NEXT: [[TMP11]] = select i1 [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
; SVE-NEXT: [[TMP12]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i64> [[VEC_IND]], <vscale x 4 x i64> [[VEC_PHI3]]
; SVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
@@ -616,7 +616,7 @@ define i32 @int_select_with_extra_arith_payload(i64 %N, ptr readonly %A, ptr rea
; NEON: [[VECTOR_BODY]]:
; NEON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NEON-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; NEON-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; NEON-NEXT: [[TMP14:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
; NEON-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; NEON-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
; NEON-NEXT: [[TMP2:%.*]] = mul <4 x i32> [[WIDE_LOAD]], splat (i32 13)
@@ -629,7 +629,7 @@ define i32 @int_select_with_extra_arith_payload(i64 %N, ptr readonly %A, ptr rea
; NEON-NEXT: [[TMP7:%.*]] = icmp slt <4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
; NEON-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]]
; NEON-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
-; NEON-NEXT: [[TMP10]] = select i1 [[TMP9]], <4 x i1> [[TMP7]], <4 x i1> [[TMP0]]
+; NEON-NEXT: [[TMP10]] = select i1 [[TMP9]], <4 x i1> [[TMP8]], <4 x i1> [[TMP14]]
; NEON-NEXT: [[TMP11]] = select i1 [[TMP9]], <4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI]]
; NEON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; NEON-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -679,7 +679,7 @@ define i32 @int_select_with_extra_arith_payload(i64 %N, ptr readonly %A, ptr rea
; SVE: [[VECTOR_BODY]]:
; SVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SVE-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
-; SVE-NEXT: [[TMP4:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; SVE-NEXT: [[TMP2:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
; SVE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; SVE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
; SVE-NEXT: [[TMP6:%.*]] = mul <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 13)
@@ -692,7 +692,7 @@ define i32 @int_select_with_extra_arith_payload(i64 %N, ptr readonly %A, ptr rea
; SVE-NEXT: [[TMP11:%.*]] = icmp slt <vscale x 4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
; SVE-NEXT: [[TMP12:%.*]] = freeze <vscale x 4 x i1> [[TMP11]]
; SVE-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP12]])
-; SVE-NEXT: [[TMP14]] = select i1 [[TMP13]], <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> [[TMP4]]
+; SVE-NEXT: [[TMP14]] = select i1 [[TMP13]], <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> [[TMP2]]
; SVE-NEXT: [[TMP15]] = select i1 [[TMP13]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
; SVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; SVE-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -786,13 +786,13 @@ define i8 @simple_csa_byte_select(i64 %N, ptr %data, i8 %a) {
; SVE: [[VECTOR_BODY]]:
; SVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SVE-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i8> [ splat (i8 -1), %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; SVE-NEXT: [[TMP4:%.*]] = phi <vscale x 16 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; SVE-NEXT: [[TMP2:%.*]] = phi <vscale x 16 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
; SVE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[DATA]], i64 [[INDEX]]
; SVE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 4
; SVE-NEXT: [[TMP6:%.*]] = icmp slt <vscale x 16 x i8> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
; SVE-NEXT: [[TMP7:%.*]] = freeze <vscale x 16 x i1> [[TMP6]]
; SVE-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP7]])
-; SVE-NEXT: [[TMP9]] = select i1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP4]]
+; SVE-NEXT: [[TMP9]] = select i1 [[TMP8]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP2]]
; SVE-NEXT: [[TMP10]] = select i1 [[TMP8]], <vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> [[VEC_PHI]]
; SVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; SVE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -853,8 +853,8 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; NEON-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NEON-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
; NEON-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
-; NEON-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
-; NEON-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; NEON-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; NEON-NEXT: [[TMP15:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; NEON-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[IV]]
; NEON-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[LD_ADDR]], i64 4
; NEON-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[LD_ADDR]], align 4
@@ -865,8 +865,8 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; NEON-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP10]]
; NEON-NEXT: [[TMP14:%.*]] = or <4 x i1> [[TMP3]], [[TMP12]]
; NEON-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]])
-; NEON-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP2]], <4 x i1> [[TMP0]]
-; NEON-NEXT: [[TMP11]] = select i1 [[TMP4]], <4 x i1> [[TMP10]], <4 x i1> [[TMP1]]
+; NEON-NEXT: [[TMP5]] = select i1 [[TMP4]], <4 x i1> [[TMP3]], <4 x i1> [[TMP1]]
+; NEON-NEXT: [[TMP11]] = select i1 [[TMP4]], <4 x i1> [[TMP12]], <4 x i1> [[TMP15]]
; NEON-NEXT: [[TMP6]] = select i1 [[TMP4]], <4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI]]
; NEON-NEXT: [[TMP13]] = select i1 [[TMP4]], <4 x i32> [[WIDE_LOAD2]], <4 x i32> [[VEC_PHI1]]
; NEON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
@@ -913,8 +913,8 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; SVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; SVE-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
; SVE-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
-; SVE-NEXT: [[TMP4:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; SVE-NEXT: [[TMP15:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; SVE-NEXT: [[TMP14:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; SVE-NEXT: [[TMP4:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; SVE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[INDEX]]
; SVE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 [[TMP3]]
; SVE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
@@ -925,8 +925,8 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; SVE-NEXT: [[TMP20:%.*]] = freeze <vscale x 4 x i1> [[TMP19]]
; SVE-NEXT: [[TMP13:%.*]] = or <vscale x 4 x i1> [[TMP7]], [[TMP20]]
; SVE-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.r...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/223962
More information about the llvm-branch-commits
mailing list