[llvm] [LV] Apply urem folds to preheader (PR #213928)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 05:55:56 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-systemz
@llvm/pr-subscribers-llvm-transforms
Author: David Sherwood (david-arm)
<details>
<summary>Changes</summary>
Following on from PR #<!-- -->212198, this patch also applies the same urem vplan folds to the vector preheader, which leads to a lot of test changes that are mostly mechanical.
As a by-product it also fixes a compile-time regression introduced by #<!-- -->212198 due to the expensive `getVectorPreheader` function.
In future we can do further folds similar to instcombine, since `(X - (X & Y)) --> (X & ~Y)` and we know that `urem(X, PowerOf2Y) -> X & log2(PowerOf2Y)`. However, the root matching pattern in this case needs to be a subtract.
---
Patch is 1.29 MiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/213928.diff
353 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+1-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll (+5-5)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll (+17-17)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/clmul.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-live-outs.ll (+14-14)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll (+12-12)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll (+6-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll (+8-8)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/epilogue-vectorization-fix-scalar-resume-values.ll (+7-7)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/f128-fmuladd-reduction.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/find-last-iv-sinkable-expr-epilogue.ll (+5-5)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/fixed-order-recurrence.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll (+6-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/gather-cost.ll (+8-8)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll (+7-7)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll (+21-21)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/indvar-overflow-check-scalable-tc.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll (+11-11)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll (+5-5)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll (+4-5)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/load-cast-context.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll (+14-14)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll (+6-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll (+12-12)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-constant-ops.ll (+7-7)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll (+18-18)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll (+50-50)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-extends-shared-with-reduce.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-incomplete-chains.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-interleave.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-no-dotprod.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate-epilogue.ll (+6-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll (+6-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/reduction-small-size.ll (+5-5)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/scalable-fp-ext-trunc-illegal-type.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll (+16-16)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll (+9-9)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/streaming-compatible-sve-no-maximize-bandwidth.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-fixed-width-inorder-core.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-fold-tail.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-group-requires-scalar-epilogue.ll (+1-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll (+6-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll (+6-8)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/gather-cost.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll (+8-8)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll (+10-10)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-multiexit.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-qabs.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll (+13-13)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-reductions-interleave.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll (+27-27)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-selectandorcost.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/sphinx.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll (+14-14)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/tail-folding-not-allowed.ll (+5-5)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/tail-folding-scalar-epilogue-fallback.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll (+7-7)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/large-loop-rdx.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll (+8-8)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/pr41179.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/small-loop-rdx.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-bswap.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/vplan-scalarivsext-crash.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/interleaved-store-with-gap.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/ordered-reduction.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/reductions.ll (+6-6)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/riscv-unroll.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll (+8-8)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll (+8-8)
- (modified) llvm/test/Transforms/LoopVectorize/SystemZ/vectorized-epilogue-loop.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll (+51-51)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/vplan-predicate-switch.ll (+24-24)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/widen-canonical-iv-register-pressure.ll (+53-53)
- (modified) llvm/test/Transforms/LoopVectorize/WebAssembly/induction-branch-cost.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/WebAssembly/partial-reduce-accumulate.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/X86/cast-costs.ll (+5-5)
- (modified) llvm/test/Transforms/LoopVectorize/X86/clmul.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/X86/conditional-scalar-assignment.ll (+12-12)
- (modified) llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll (+5-5)
- (modified) llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll (+11-11)
- (modified) llvm/test/Transforms/LoopVectorize/X86/cost-model-i386.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/cost-model.ll (+7-7)
- (modified) llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/end-pointer-signed.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-ordered-reduction.ll (+37-37)
- (modified) llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll (+18-18)
- (modified) llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/gather-cost.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll (+18-18)
- (modified) llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/X86/induction-step.ll (+5-5)
- (modified) llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/interleave-opaque-pointers.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/intrinsiccost.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll (+10-10)
- (modified) llvm/test/Transforms/LoopVectorize/X86/iv-live-outs.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll (+60-60)
- (modified) llvm/test/Transforms/LoopVectorize/X86/multi-exit-cost.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/optsize.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/outer-loop-non-power-of-2-type.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr23997.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr35432.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr39160.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr47437.ll (+5-5)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr48340.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr54634.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr72969.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll (+12-12)
- (modified) llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll (+7-7)
- (modified) llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll (+14-14)
- (modified) llvm/test/Transforms/LoopVectorize/X86/small-size.ll (+14-14)
- (modified) llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-gaps.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-live-outs.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll (+9-9)
- (modified) llvm/test/Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/X86/widen-canonical-iv-register-pressure.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll (+10-10)
- (modified) llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/assume.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/bounded-load-multi-exit.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/bounded-load-predicated.ll (+6-6)
- (modified) llvm/test/Transforms/LoopVectorize/bounded-load-user-ic.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/bounded-load-vf-ranges.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/byte-type-function-variants.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/cast-induction.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/conditional-scalar-assignment-interleave-only.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll (+30-30)
- (modified) llvm/test/Transforms/LoopVectorize/cse-gep-source-element-type.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/dead_instructions.ll (+7-7)
- (modified) llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size-needs-loop-guards.ll (+13-7)
- (modified) llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll (+10-10)
- (modified) llvm/test/Transforms/LoopVectorize/div-exact.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/dont-fold-tail-for-divisible-TC.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/early-exit-minmax-trip-count.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll (+22-22)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll (+8-8)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll (+11-11)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll (+56-56)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-scev-expansion.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-trunc-induction-steps.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/expand-ptrtoaddr.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/expression-recipe-branch-weights.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/extract-last-veclane.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-iv-interleave.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-epilogue.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll (+12-12)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll (+15-15)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-load.ll (+9-9)
- (modified) llvm/test/Transforms/LoopVectorize/find-last.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll (+14-14)
- (modified) llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll (+9-9)
- (modified) llvm/test/Transforms/LoopVectorize/first-order-recurrence-with-uniform-ops.ll (+6-6)
- (modified) llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll (+51-51)
- (modified) llvm/test/Transforms/LoopVectorize/flags.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/float-induction.ll (+32-32)
- (modified) llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll (+5-5)
- (modified) llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/fpsat.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/histograms.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/hoist-and-sink-mem-ops-with-invariant-pointers.ll (+6-6)
- (modified) llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll (+6-6)
- (modified) llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/if-conversion.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/if-pred-stores.ll (+7-7)
- (modified) llvm/test/Transforms/LoopVectorize/if-reduction.ll (+15-15)
- (modified) llvm/test/Transforms/LoopVectorize/induction-ptrcasts.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/induction-step.ll (+9-9)
- (modified) llvm/test/Transforms/LoopVectorize/induction-unroll-novec.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/induction.ll (+104-104)
- (modified) llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll (+3-3)
- (modified) llvm/test/Transforms/LoopVectorize/interleaved-accesses-2.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll (+2-2)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 541a3da855992..8761361c78f75 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1366,10 +1366,7 @@ static void simplifyRecipe(VPSingleDefRecipe *Def) {
}
const APInt *APC;
- // TODO: Enable optimizations in the vector preheader in a follow-up PR.
- // This check currently means we only simplify before region dissolution.
- VPBasicBlock *Preheader = Plan->getVectorPreheader();
- if (CanCreateNewRecipe && Preheader && Def->getParent() != Preheader &&
+ if (CanCreateNewRecipe &&
match(Def, m_URem(m_VPValue(X), m_APInt(APC))) && APC->isPowerOf2()) {
return Def->replaceAllUsesWith(Builder.createAnd(
X, Plan->getConstantInt(*APC - 1), Def->getDebugLoc()));
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll b/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
index 23ffc98cbc283..9789e291dc1a9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
@@ -31,7 +31,7 @@ define void @test_interleave_reduction(ptr %arg, ptr %arg1) {
; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
; A320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; A320: [[VECTOR_PH]]:
-; A320-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
+; A320-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP3]], 1
; A320-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
; A320-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
; A320-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[TMP4]]
@@ -73,12 +73,12 @@ define void @test_interleave_reduction(ptr %arg, ptr %arg1) {
; A320-NEXT: br i1 [[CMP_N]], label %[[EXIT_INNER:.*]], label %[[SCALAR_PH]]
; A320: [[SCALAR_PH]]:
; A320-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[TPM27]], %[[OUTER]] ]
-; A320-NEXT: [[BC_RESUME_VAL7:%.*]] = phi ptr [ [[IND_END3]], %[[MIDDLE_BLOCK]] ], [ [[TPM32]], %[[OUTER]] ]
+; A320-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[IND_END3]], %[[MIDDLE_BLOCK]] ], [ [[TPM32]], %[[OUTER]] ]
; A320-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[OUTER]] ]
; A320-NEXT: br label %[[INNER:.*]]
; A320: [[INNER]]:
; A320-NEXT: [[PHI_PTR_I32:%.*]] = phi ptr [ [[NEXT_I32:%.*]], %[[INNER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; A320-NEXT: [[PHI_PTR_F64:%.*]] = phi ptr [ [[NEXT_F64:%.*]], %[[INNER]] ], [ [[BC_RESUME_VAL7]], %[[SCALAR_PH]] ]
+; A320-NEXT: [[PHI_PTR_F64:%.*]] = phi ptr [ [[NEXT_F64:%.*]], %[[INNER]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
; A320-NEXT: [[PHI_ACC:%.*]] = phi double [ [[TPM50:%.*]], %[[INNER]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; A320-NEXT: [[TPM44:%.*]] = load double, ptr [[PHI_PTR_F64]], align 8
; A320-NEXT: [[TPM45:%.*]] = load i32, ptr [[PHI_PTR_I32]], align 4
@@ -154,7 +154,7 @@ define double @sum_reduction(ptr nocapture readonly %a, i64 %n) {
; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; A320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; A320: [[VECTOR_PH]]:
-; A320-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; A320-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 7
; A320-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; A320-NEXT: br label %[[VECTOR_BODY:.*]]
; A320: [[VECTOR_BODY]]:
@@ -245,7 +245,7 @@ define double @dot_product(ptr nocapture readonly %a, ptr nocapture readonly %b,
; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; A320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; A320: [[VECTOR_PH]]:
-; A320-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; A320-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 7
; A320-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; A320-NEXT: br label %[[VECTOR_BODY:.*]]
; A320: [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
index 1a83a3ed4cdb8..c41e9d78de209 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
@@ -12,7 +12,7 @@ define void @test_blend_feeding_replicated_store_1(i64 %N, ptr noalias %src, ptr
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP43]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP43]], 16
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP43]], 15
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i64 16, i64 [[N_MOD_VF]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP43]], [[TMP2]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll b/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll
index bb1bcf18b0a41..bc16875a3fe3a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll
@@ -19,7 +19,7 @@ define i32 @bounded_load_reduction_bound2(ptr %A, i32 %N) {
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i32 [[N]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 16
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N]], 15
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -62,7 +62,7 @@ define i32 @bounded_load_reduction_bound2(ptr %A, i32 %N) {
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_MOD_VF13:%.*]] = urem i32 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF13:%.*]] = and i32 [[N]], 3
; CHECK-NEXT: [[N_VEC14:%.*]] = sub i32 [[N]], [[N_MOD_VF13]]
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -84,11 +84,11 @@ define i32 @bounded_load_reduction_bound2(ptr %A, i32 %N) {
; CHECK-NEXT: br i1 [[CMP_N20]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX21:%.*]] = phi i32 [ [[TMP22]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX20:%.*]] = phi i32 [ [[TMP22]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX21]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX20]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[BOUNDED:%.*]] = urem i32 [[IV]], 2
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[BOUNDED]]
; CHECK-NEXT: [[LV:%.*]] = load i32, ptr [[GEP]], align 4
@@ -133,7 +133,7 @@ define i32 @bounded_load_reduction_bound4(ptr %A, i32 %N) {
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i32 [[N]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 16
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N]], 15
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -171,7 +171,7 @@ define i32 @bounded_load_reduction_bound4(ptr %A, i32 %N) {
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_MOD_VF10:%.*]] = urem i32 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF10:%.*]] = and i32 [[N]], 3
; CHECK-NEXT: [[N_VEC11:%.*]] = sub i32 [[N]], [[N_MOD_VF10]]
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -191,11 +191,11 @@ define i32 @bounded_load_reduction_bound4(ptr %A, i32 %N) {
; CHECK-NEXT: br i1 [[CMP_N16]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX17:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX16:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX17]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX16]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[BOUNDED:%.*]] = urem i32 [[IV]], 4
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[BOUNDED]]
; CHECK-NEXT: [[LV:%.*]] = load i32, ptr [[GEP]], align 4
@@ -240,7 +240,7 @@ define i16 @bounded_load_reduction_bound4_i16(ptr %A, i32 %N) {
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i32 [[N]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 16
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N]], 15
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -268,7 +268,7 @@ define i16 @bounded_load_reduction_bound4_i16(ptr %A, i32 %N) {
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP8]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_MOD_VF4:%.*]] = urem i32 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF4:%.*]] = and i32 [[N]], 3
; CHECK-NEXT: [[N_VEC5:%.*]] = sub i32 [[N]], [[N_MOD_VF4]]
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i32 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -288,11 +288,11 @@ define i16 @bounded_load_reduction_bound4_i16(ptr %A, i32 %N) {
; CHECK-NEXT: br i1 [[CMP_N10]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi i16 [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP8]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX10:%.*]] = phi i16 [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP8]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[SUM:%.*]] = phi i16 [ [[BC_MERGE_RDX11]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[SUM:%.*]] = phi i16 [ [[BC_MERGE_RDX10]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[BOUNDED:%.*]] = urem i32 [[IV]], 4
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i16, ptr [[A]], i32 [[BOUNDED]]
; CHECK-NEXT: [[LV:%.*]] = load i16, ptr [[GEP]], align 2
@@ -335,7 +335,7 @@ define i32 @bounded_user_ic_exceeds_window(ptr %A, i32 %N) {
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i32 [[TMP0]], 3
; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 32
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N]], 31
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -535,7 +535,7 @@ define i32 @reverse_load_with_bounded(ptr %A, ptr %B, i32 %N) {
; CHECK-NEXT: [[MIN_ITERS_CHECK2:%.*]] = icmp ult i32 [[N]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK2]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 16
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N]], 15
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -591,7 +591,7 @@ define i32 @reverse_load_with_bounded(ptr %A, ptr %B, i32 %N) {
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP24]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_MOD_VF18:%.*]] = urem i32 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF18:%.*]] = and i32 [[N]], 3
; CHECK-NEXT: [[N_VEC19:%.*]] = sub i32 [[N]], [[N_MOD_VF18]]
; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -617,11 +617,11 @@ define i32 @reverse_load_with_bounded(ptr %A, ptr %B, i32 %N) {
; CHECK-NEXT: br i1 [[CMP_N26]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC19]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX27:%.*]] = phi i32 [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP24]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX26:%.*]] = phi i32 [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP24]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX27]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX26]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[BOUNDED:%.*]] = urem i32 [[IV]], 4
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[BOUNDED]]
; CHECK-NEXT: [[LV1:%.*]] = load i32, ptr [[GEP1]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
index 47ec13b653a18..2143cf45fc1bd 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
@@ -259,7 +259,7 @@ define void @widen_intrinsics_with_mixed_return_types(ptr noalias %src, ptr noal
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 7
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -277,7 +277,7 @@ define void @widen_intrinsics_with_mixed_return_types(ptr noalias %src, ptr noal
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -296,7 +296,7 @@ define void @widen_intrinsics_with_mixed_return_types(ptr noalias %src, ptr noal
; CHECK-NEXT: store i32 [[COND]], ptr [[DST_GEP]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/clmul.ll b/llvm/test/Transforms/LoopVectorize/AArch64/clmul.ll
index 8871a81b4c654..b866ccd9c6cf9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/clmul.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/clmul.ll
@@ -21,7 +21,7 @@ define void @clmul_loop(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 09852d7951c09..98bb0257a91f9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -56,7 +56,7 @@ define void @loop_dependent_cond(ptr %src, ptr noalias %dst, i64 %N) {
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
+; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 3
; DEFAULT...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/213928
More information about the llvm-commits
mailing list