[llvm] [LV] Add vplan folds for (X - (X & Y)) -> (X & ~Y) (PR #226466)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 05:15:01 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-powerpc
Author: David Sherwood (david-arm)
<details>
<summary>Changes</summary>
Now that the recipe simplification is more like instcombine, I decided to upload a patch I've had lying around for a while in case reviewers think it's worthwhile.
This simplifies the calculation of the number of iterations processed in the vector loop, using a recipe combine borrowed from instcombine for equivalent IR. Unfortunately it's a very invasive PR, since it changes over 350 tests!
---
Patch is 1.47 MiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/226466.diff
360 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+9)
- (modified) llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll (+15-22)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/clmul.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll (+3-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-live-outs.ll (+18-22)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll (+12-14)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll (+9-12)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll (+12-16)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/epilogue-vectorization-fix-scalar-resume-values.ll (+8-10)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/f128-fmuladd-reduction.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/find-last-iv-sinkable-expr-epilogue.ll (+6-8)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/fixed-order-recurrence.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll (+8-12)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/gather-cost.ll (+9-10)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll (+7-14)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll (+21-28)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/indvar-overflow-check-scalable-tc.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll (+13-20)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll (+9-12)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll (+6-8)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/intrinsiccost.ll (+6-12)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/load-cast-context.ll (+3-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll (+19-28)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll (+6-8)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll (+12-24)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-constant-ops.ll (+7-14)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll (+4-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll (+18-36)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll (+10-20)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-extends-shared-with-reduce.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-incomplete-chains.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-interleave.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-no-dotprod.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate-epilogue.ll (+6-8)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll (+8-12)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/reduction-small-size.ll (+7-10)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/scalable-fp-ext-trunc-illegal-type.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/select-index.ll (+17-26)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/streaming-compatible-sve-no-maximize-bandwidth.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-fixed-width-inorder-core.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-fold-tail.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll (+9-12)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll (+4-8)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll (+2-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/gather-cost.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll (+7-14)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-hoist-runtime-checks.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-qabs.ll (+9-12)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll (+13-26)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-reductions-interleave.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll (+27-54)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/mve-selectandorcost.ll (+3-4)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/sphinx.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll (+14-28)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/tail-folding-not-allowed.ll (+5-10)
- (modified) llvm/test/Transforms/LoopVectorize/ARM/tail-folding-scalar-epilogue-fallback.ll (+43-33)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll (+7-8)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/large-loop-rdx.ll (+5-6)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll (+12-16)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/partial-red.ll (+75-90)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/pr41179.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/small-loop-rdx.ll (+5-6)
- (modified) llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-bswap.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/interleaved-store-with-gap.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/ordered-reduction.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll (+6-12)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/reductions.ll (+6-12)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/riscv-unroll.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll (+8-16)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll (+8-16)
- (modified) llvm/test/Transforms/LoopVectorize/SystemZ/vectorized-epilogue-loop.ll (+3-4)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/cse-loads-tailfold.ll (+2-3)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/epilogue-vectorization-printing.ll (+43-47)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll (+6-8)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/memory-checks.ll (+4-8)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/vplan-predicate-switch.ll (+4-5)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/widen-canonical-iv-register-pressure.ll (+4-6)
- (modified) llvm/test/Transforms/LoopVectorize/WebAssembly/induction-branch-cost.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/WebAssembly/partial-reduce-accumulate.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/cast-costs.ll (+5-6)
- (modified) llvm/test/Transforms/LoopVectorize/X86/clmul.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/conditional-scalar-assignment.ll (+12-24)
- (modified) llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll (+22-24)
- (modified) llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/cost-model-i386.ll (+3-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/cost-model.ll (+7-10)
- (modified) llvm/test/Transforms/LoopVectorize/X86/divs-with-tail-folding.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/X86/end-pointer-signed.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll (+6-8)
- (modified) llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-ordered-reduction.ll (+22-26)
- (modified) llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll (+21-24)
- (modified) llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/gather-cost.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll (+4-6)
- (modified) llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll (+24-36)
- (modified) llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll (+19-21)
- (modified) llvm/test/Transforms/LoopVectorize/X86/induction-step.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/X86/interleave-opaque-pointers.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/intrinsiccost.ll (+96-100)
- (modified) llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll (+3-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll (+13-16)
- (modified) llvm/test/Transforms/LoopVectorize/X86/iv-live-outs.ll (+3-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll (+18-24)
- (modified) llvm/test/Transforms/LoopVectorize/X86/optsize.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/outer-loop-non-power-of-2-type.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr23997.ll (+5-6)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr35432.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr39160.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr47437.ll (+6-10)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr48340.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr54634.ll (+3-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr72969.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/predicate-switch.ll (+12-24)
- (modified) llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll (+16-18)
- (modified) llvm/test/Transforms/LoopVectorize/X86/small-size.ll (+14-17)
- (modified) llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-gaps.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-live-outs.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll (+9-14)
- (modified) llvm/test/Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll (+4-6)
- (modified) llvm/test/Transforms/LoopVectorize/X86/widen-canonical-iv-register-pressure.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll (+10-20)
- (modified) llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/alias-mask-negative-tests.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/assume.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/blend-i1.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/bounded-load-predicated.ll (+6-12)
- (modified) llvm/test/Transforms/LoopVectorize/bounded-load-user-ic.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/bounded-load-vf-ranges.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/byte-type-function-variants.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/cast-induction.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/conditional-scalar-assignment-interleave-only.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll (+19-38)
- (modified) llvm/test/Transforms/LoopVectorize/cse-gep-source-element-type.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/dead_instructions.ll (+3-4)
- (modified) llvm/test/Transforms/LoopVectorize/debugloc.ll (+7-3)
- (modified) llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size-needs-loop-guards.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll (+10-20)
- (modified) llvm/test/Transforms/LoopVectorize/div-exact.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/dont-fold-tail-for-divisible-TC.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/early-exit-minmax-trip-count.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/early-exit-trip-count-may-cause-ub.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll (+26-32)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll (+12-16)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-anyof-no-rdx-result.ll (+6-8)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll (+9-12)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll (+64-75)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-scev-expansion.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-trunc-induction-steps.ll (+3-4)
- (modified) llvm/test/Transforms/LoopVectorize/expand-ptrtoaddr.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/expand-scev-cse.ll (+4-6)
- (modified) llvm/test/Transforms/LoopVectorize/expression-recipe-branch-weights.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/extract-last-veclane.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-iv-interleave.ll (+4-8)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-epilogue.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll (+12-24)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll (+14-28)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-load.ll (+9-18)
- (modified) llvm/test/Transforms/LoopVectorize/find-last-ptr-induction.ll (+4-8)
- (modified) llvm/test/Transforms/LoopVectorize/find-last.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll (+14-28)
- (modified) llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll (+9-18)
- (modified) llvm/test/Transforms/LoopVectorize/first-order-recurrence-with-uniform-ops.ll (+6-12)
- (modified) llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll (+42-81)
- (modified) llvm/test/Transforms/LoopVectorize/flags.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/float-induction.ll (+32-64)
- (modified) llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll (+5-10)
- (modified) llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/fpsat.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/hoist-and-sink-mem-ops-with-invariant-pointers.ll (+4-8)
- (modified) llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll (+1-2)
- (modified) llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/if-conversion.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/if-pred-stores.ll (+3-6)
- (modified) llvm/test/Transforms/LoopVectorize/if-reduction.ll (+15-30)
- (modified) llvm/test/Transforms/LoopVectorize/induction-ptrcasts.ll (+2-4)
- (modified) llvm/test/Transforms/LoopVectorize/induction-step.ll (+8-14)
- (modified) llvm/test/Transforms/LoopVectorize/induction-unroll-novec.ll (+3-4)
- (modified) llvm/test/Transforms/LoopVectorize/induction-wrapflags.ll (+4-8)
- (modified) llvm/test/Transforms/LoopVectorize/induction.ll (+77-154)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 09fdaa63bab999..9a11fe608eac96 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1528,6 +1528,15 @@ static VPSingleDefRecipe *combineRecipe(VPlan &Plan, VPSingleDefRecipe *Def,
}
const APInt *APC;
+
+ // (X - (X & Y)) -> (X & ~Y)
+ if (CanCreateNewRecipe &&
+ match(Def,
+ m_Sub(m_VPValue(X), m_c_BinaryAnd(m_Deferred(X), m_VPValue(Y)))) &&
+ match(Y, m_APInt(APC)))
+ return Builder.createAnd(X, Plan.getConstantInt(~(*APC)),
+ Def->getDebugLoc());
+
if (CanCreateNewRecipe && match(Def, m_URem(m_VPValue(X), m_APInt(APC))) &&
APC->isPowerOf2())
return Builder.createAnd(X, Plan.getConstantInt(*APC - 1),
diff --git a/llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll b/llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll
index 30540e56c146cb..f984a2f34bcd24 100644
--- a/llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll
+++ b/llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll
@@ -56,8 +56,7 @@ define void @g(ptr %dst.1, ptr %start, i64 %N) {
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4
; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[N_VEC]], 3
; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_1]], i64 [[TMP2]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll b/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
index 8a6de3f6a7ba69..48a34f4152cd9a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/aggressive-interleaving.ll
@@ -31,8 +31,7 @@ define void @test_interleave_reduction(ptr %arg, ptr %arg1) {
; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
; A320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; A320: [[VECTOR_PH]]:
-; A320-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP3]], 1
-; A320-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; A320-NEXT: [[N_VEC:%.*]] = and i64 [[TMP3]], -2
; A320-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
; A320-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[TPM27]], i64 [[TMP4]]
; A320-NEXT: [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
@@ -154,8 +153,7 @@ define double @sum_reduction(ptr nocapture readonly %a, i64 %n) {
; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; A320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; A320: [[VECTOR_PH]]:
-; A320-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 7
-; A320-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; A320-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -8
; A320-NEXT: br label %[[VECTOR_BODY:.*]]
; A320: [[VECTOR_BODY]]:
; A320-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -245,8 +243,7 @@ define double @dot_product(ptr nocapture readonly %a, ptr nocapture readonly %b,
; A320-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; A320-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; A320: [[VECTOR_PH]]:
-; A320-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 7
-; A320-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; A320-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -8
; A320-NEXT: br label %[[VECTOR_BODY:.*]]
; A320: [[VECTOR_BODY]]:
; A320-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll b/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll
index 453ff606afedb0..2e3f57463c19ca 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll
@@ -19,8 +19,7 @@ define i32 @bounded_load_reduction_bound2(ptr %A, i32 %N) {
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i32 [[N]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[N]], 15
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[TMP4]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], -16
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -57,13 +56,13 @@ define i32 @bounded_load_reduction_bound2(ptr %A, i32 %N) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[N]], [[N_VEC]]
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP4]], 4
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP17:%.*]] = and i32 [[N]], 3
-; CHECK-NEXT: [[N_VEC13:%.*]] = sub i32 [[N]], [[TMP17]]
+; CHECK-NEXT: [[N_VEC13:%.*]] = and i32 [[N]], -4
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
@@ -117,8 +116,7 @@ define i32 @bounded_load_reduction_bound4(ptr %A, i32 %N) {
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i32 [[N]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[N]], 15
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[TMP2]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], -16
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -150,13 +148,13 @@ define i32 @bounded_load_reduction_bound4(ptr %A, i32 %N) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[N]], [[N_VEC]]
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP2]], 4
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = and i32 [[N]], 3
-; CHECK-NEXT: [[N_VEC10:%.*]] = sub i32 [[N]], [[TMP14]]
+; CHECK-NEXT: [[N_VEC10:%.*]] = and i32 [[N]], -4
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
@@ -208,8 +206,7 @@ define i16 @bounded_load_reduction_bound4_i16(ptr %A, i32 %N) {
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i32 [[N]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[N]], 15
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[TMP2]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], -16
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -231,13 +228,13 @@ define i16 @bounded_load_reduction_bound4_i16(ptr %A, i32 %N) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[N]], [[N_VEC]]
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP2]], 4
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i16 [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = and i32 [[N]], 3
-; CHECK-NEXT: [[N_VEC4:%.*]] = sub i32 [[N]], [[TMP10]]
+; CHECK-NEXT: [[N_VEC4:%.*]] = and i32 [[N]], -4
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i16> zeroinitializer, i16 [[BC_MERGE_RDX]], i64 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
@@ -287,8 +284,7 @@ define i32 @bounded_user_ic_exceeds_window(ptr %A, i32 %N) {
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i32 [[TMP0]], 3
; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[N]], 31
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[TMP2]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], -32
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -487,8 +483,7 @@ define i32 @bounded_load_bound64_scalable_vf(ptr %A, i64 %N) #0 {
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP18:%.*]] = and i64 [[N]], 3
-; CHECK-NEXT: [[N_VEC10:%.*]] = sub i64 [[N]], [[TMP18]]
+; CHECK-NEXT: [[N_VEC10:%.*]] = and i64 [[N]], -4
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
@@ -585,8 +580,7 @@ define i32 @bounded_load_bound4_fixed_vf(ptr %A, i64 %N) #0 {
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP18:%.*]] = and i64 [[N]], 3
-; CHECK-NEXT: [[N_VEC10:%.*]] = sub i64 [[N]], [[TMP18]]
+; CHECK-NEXT: [[N_VEC10:%.*]] = and i64 [[N]], -4
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
@@ -694,8 +688,7 @@ define i32 @reverse_load_with_bounded(ptr %A, ptr %B, i32 %N) {
; CHECK-NEXT: [[MIN_ITERS_CHECK2:%.*]] = icmp ult i32 [[N]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK2]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[N]], 15
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[TMP4]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], -16
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -745,13 +738,13 @@ define i32 @reverse_load_with_bounded(ptr %A, ptr %B, i32 %N) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[N]], [[N_VEC]]
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP4]], 4
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP26:%.*]] = and i32 [[N]], 3
-; CHECK-NEXT: [[N_VEC18:%.*]] = sub i32 [[N]], [[TMP26]]
+; CHECK-NEXT: [[N_VEC18:%.*]] = and i32 [[N]], -4
; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
index ad59b558fd850c..44b548be84f90d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
@@ -259,8 +259,7 @@ define void @widen_intrinsics_with_mixed_return_types(ptr noalias %src, ptr noal
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 7
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -8
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/clmul.ll b/llvm/test/Transforms/LoopVectorize/AArch64/clmul.ll
index b866ccd9c6cf92..89ff956976db03 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/clmul.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/clmul.ll
@@ -21,8 +21,7 @@ define void @clmul_loop(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 0c113484bb31ec..b6858539b31d69 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -56,8 +56,7 @@ define void @loop_dependent_cond(ptr %src, ptr noalias %dst, i64 %N) {
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 3
-; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; DEFAULT-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], -4
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
@@ -423,8 +422,7 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
; DEFAULT-NEXT: [[CONFLICT_RDX27:%.*]] = or i1 [[CONFLICT_RDX23]], [[FOUND_CONFLICT26]]
; DEFAULT-NEXT: br i1 [[CONFLICT_RDX27]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 3
-; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; DEFAULT-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], -4
; DEFAULT-NEXT: [[TMP3:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META8:![0-9]+]]
; DEFAULT-NEXT: [[TMP4:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META11:![0-9]+]]
; DEFAULT-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP3]]
@@ -715,8 +713,7 @@ define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias
; DEFAULT-NEXT: [[TMP14:%.*]] = or i1 [[TMP13]], [[TMP12]]
; DEFAULT-NEXT: br i1 [[TMP14]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 7
-; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; DEFAULT-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], -8
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE27:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
index ee5b164eae8c3e..9fea106b85dad5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
@@ -190,8 +190,7 @@ define float @simple_csa_float_select(i64 %N, ptr %data, float %a) {
; NEON-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 20
; NEON-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NEON: [[VECTOR_PH]]:
-; NEON-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
-; NEON-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; NEON-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4
; NEON-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[A]], i64 0
; NEON-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
; NEON-NEXT: br label %[[LOOP:.*]]
@@ -608,8 +607,7 @@ define i32 @int_select_with_extra_arith_payload(i64 %N, ptr readonly %A, ptr rea
; NEON-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; NEON-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NEON: [[VECTOR_PH]]:
-; NEON-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
-; NEON-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; NEON-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4
; NEON-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[THRESHOLD]], i64 0
; NEON-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; NEON-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -844,8 +842,7 @@ define i32 @simple_csa_int_select_use_interleave(i64 %N, ptr %data, i32 %a) {
; NEON-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; NEON-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NEON: [[VECTOR_...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/226466
More information about the llvm-commits
mailing list