[llvm-branch-commits] [llvm] [AArch64] Avoid materializing full masks for extractelement users of predicate-as-counter (PR #220961)
Benjamin Maxwell via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Sep 3 07:56:28 PDT 2026
https://github.com/MacDue created https://github.com/llvm/llvm-project/pull/220961
If the user of the original wide mask is an `extractelement` and the
index is known to be within the first segment of the
predicate-as-counter, replace it with `extractelement(pext(counter, 0))`.
This avoids materializing the vector mask and produces a form that can
be folded into a conditional branch when the predicate-as-counter is
produced by a `whilelo`.
Assisted-by: Codex
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
>From c74d1e415b0d37d74987ff693f2278b80df9eb8b Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 3 Sep 2026 13:36:55 +0000
Subject: [PATCH] [AArch64] Avoid materializing full masks for extractelement
users of predicate-as-counter
If the user of the original wide mask is an `extractelement` and the
index is known to be within the first segment of the
predicate-as-counter, replace it with `extractelement(pext(counter, 0))`.
This avoids materializing the vector mask and produces a form that can
be folded into a conditional branch when the predicate-as-counter is
produced by a `whilelo`.
Assisted-by: Codex
---
.../AArch64PredicateAsCounterLoopRewrites.cpp | 40 +++
.../predicate-as-counter-loop-rewrites.ll | 237 +++++++-----------
2 files changed, 128 insertions(+), 149 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp b/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp
index b701e6c176146..fa855d87793d3 100644
--- a/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp
@@ -228,6 +228,42 @@ static Value *createWhileLO(IRBuilder<> &Builder, unsigned ElementSizeInBits,
return Builder.CreateCall(
WhileLO, {Start, End, Builder.getInt32(VectorScale)}, "pac.mask");
}
+
+/// If \p UserI is an extractelement use of the original loop mask, attempt to
+/// rewrite it to `extractelement(pext(count, 0), idx)` if the extract index is
+/// known to be within the first mask section (which means pext index = 0). If
+/// the user of the extract is a branch and the source of the mask is a whilelo,
+/// this form can be optimized into checking the status flags.
+static bool tryRewriteExtractElement(Instruction &UserI,
+ const MaskRewriteCandidate &C,
+ Value *Count) {
+ auto *EEI = dyn_cast<ExtractElementInst>(&UserI);
+ if (!EEI)
+ return false;
+
+ ElementCount LegalEC = getSVEElementCount(C.ElementSizeInBits);
+ auto *Idx = dyn_cast<ConstantInt>(EEI->getIndexOperand());
+ if (!Idx || Idx->getValue().uge(LegalEC.getKnownMinValue()))
+ return false;
+
+ IRBuilder<> Builder(EEI);
+ Builder.SetCurrentDebugLocation(EEI->getDebugLoc());
+
+ Module *M = EEI->getModule();
+ FunctionCallee PExt = Intrinsic::getOrInsertDeclaration(
+ M, Intrinsic::aarch64_sve_pext,
+ {VectorType::get(Builder.getInt1Ty(), LegalEC)});
+ auto *ExtractMask =
+ Builder.CreateCall(PExt, {Count, Builder.getInt32(0)}, "pac.pext");
+
+ Value *Extracted = Builder.CreateExtractElement(
+ ExtractMask, EEI->getIndexOperand(), EEI->getName() + ".pac");
+
+ EEI->replaceAllUsesWith(Extracted);
+ EEI->eraseFromParent();
+ return true;
+}
+
class AArch64PredicateAsCounterLoopRewrites : public LoopPass {
public:
static char ID;
@@ -427,6 +463,10 @@ bool AArch64PredicateAsCounterLoopRewrites::rewriteCandidate(
Value *WideMask = nullptr;
for (Use *U : UsesToRewrite) {
+ auto *UserI = cast<Instruction>(U->getUser());
+ if (tryRewriteExtractElement(*UserI, C, Count))
+ continue;
+
if (!WideMask) {
BasicBlock::iterator InsertPt = OldMask->getIterator();
if (isa<PHINode>(OldMask))
diff --git a/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll b/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll
index b064c81107795..dca70b3555b7d 100644
--- a/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll
+++ b/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll
@@ -11,31 +11,24 @@ define void @rewrite_masked_load_store_i64_vlx4(ptr %x, i64 %n) #0 {
; CHECK-NEXT: whilelo pn8.d, xzr, x1, vlx4
; CHECK-NEXT: .LBB0_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1]
-; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0]
+; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1]
+; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0]
; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4
-; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0, #3, mul vl]
-; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0]
-; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #2, mul vl]
-; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s
-; CHECK-NEXT: pext { p5.d, p6.d }, pn8[1]
-; CHECK-NEXT: ld1d { z2.d }, p1/z, [x0, #1, mul vl]
-; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s
-; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl]
+; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0]
; CHECK-NEXT: inch x8
; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
-; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h
; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
-; CHECK-NEXT: st1d { z0.d }, p3, [x0, #3, mul vl]
-; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1
-; CHECK-NEXT: st1d { z1.d }, p2, [x0, #2, mul vl]
-; CHECK-NEXT: st1d { z2.d }, p1, [x0, #1, mul vl]
-; CHECK-NEXT: st1d { z3.d }, p0, [x0]
+; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl]
+; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl]
+; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl]
+; CHECK-NEXT: st1d { z3.d }, p2, [x0]
; CHECK-NEXT: incb x0, all, mul #4
-; CHECK-NEXT: fmov w9, s0
-; CHECK-NEXT: tbnz w9, #0, .LBB0_1
+; CHECK-NEXT: b.mi .LBB0_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret
entry:
@@ -71,31 +64,24 @@ define void @rewrite_masked_load_store_i64_multi_pred_header(ptr %x, i64 %n, i1
; CHECK-NEXT: whilelo pn8.d, xzr, x1, vlx4
; CHECK-NEXT: .LBB1_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1]
-; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0]
+; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1]
+; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0]
; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4
-; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0, #3, mul vl]
-; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0]
-; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #2, mul vl]
-; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s
-; CHECK-NEXT: pext { p5.d, p6.d }, pn8[1]
-; CHECK-NEXT: ld1d { z2.d }, p1/z, [x0, #1, mul vl]
-; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s
-; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl]
+; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0]
; CHECK-NEXT: inch x8
; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
-; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h
; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
-; CHECK-NEXT: st1d { z0.d }, p3, [x0, #3, mul vl]
-; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1
-; CHECK-NEXT: st1d { z1.d }, p2, [x0, #2, mul vl]
-; CHECK-NEXT: st1d { z2.d }, p1, [x0, #1, mul vl]
-; CHECK-NEXT: st1d { z3.d }, p0, [x0]
+; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl]
+; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl]
+; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl]
+; CHECK-NEXT: st1d { z3.d }, p2, [x0]
; CHECK-NEXT: incb x0, all, mul #4
-; CHECK-NEXT: fmov w9, s0
-; CHECK-NEXT: tbnz w9, #0, .LBB1_1
+; CHECK-NEXT: b.mi .LBB1_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret
entry:
@@ -136,54 +122,40 @@ define void @shared_start_mask_between_loops(ptr %x, ptr %y, i64 %n) #0 {
; CHECK-NEXT: mov x9, x8
; CHECK-NEXT: .LBB2_1: // %loop1
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: pext { p2.d, p3.d }, pn9[1]
-; CHECK-NEXT: pext { p0.d, p1.d }, pn9[0]
+; CHECK-NEXT: pext { p0.d, p1.d }, pn9[1]
+; CHECK-NEXT: pext { p2.d, p3.d }, pn9[0]
; CHECK-NEXT: whilelo pn9.d, x9, x2, vlx4
-; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0, #3, mul vl]
-; CHECK-NEXT: pext { p4.d, p5.d }, pn9[0]
-; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #2, mul vl]
-; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s
-; CHECK-NEXT: pext { p5.d, p6.d }, pn9[1]
-; CHECK-NEXT: ld1d { z2.d }, p1/z, [x0, #1, mul vl]
-; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s
-; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl]
+; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0]
; CHECK-NEXT: inch x9
; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
-; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h
; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
-; CHECK-NEXT: st1d { z0.d }, p3, [x0, #3, mul vl]
-; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1
-; CHECK-NEXT: st1d { z1.d }, p2, [x0, #2, mul vl]
-; CHECK-NEXT: st1d { z2.d }, p1, [x0, #1, mul vl]
-; CHECK-NEXT: st1d { z3.d }, p0, [x0]
+; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl]
+; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl]
+; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl]
+; CHECK-NEXT: st1d { z3.d }, p2, [x0]
; CHECK-NEXT: incb x0, all, mul #4
-; CHECK-NEXT: fmov w10, s0
-; CHECK-NEXT: tbnz w10, #0, .LBB2_1
+; CHECK-NEXT: b.mi .LBB2_1
; CHECK-NEXT: .LBB2_2: // %loop2
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0]
-; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1]
+; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1]
+; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0]
; CHECK-NEXT: whilelo pn8.d, x8, x2, vlx4
-; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0]
-; CHECK-NEXT: ld1d { z0.d }, p3/z, [x1, #3, mul vl]
-; CHECK-NEXT: ld1d { z1.d }, p2/z, [x1, #2, mul vl]
-; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s
-; CHECK-NEXT: pext { p5.d, p6.d }, pn8[1]
-; CHECK-NEXT: ld1d { z2.d }, p1/z, [x1, #1, mul vl]
-; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s
-; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1]
+; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1, #3, mul vl]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1, #2, mul vl]
+; CHECK-NEXT: ld1d { z2.d }, p3/z, [x1, #1, mul vl]
+; CHECK-NEXT: ld1d { z3.d }, p2/z, [x1]
; CHECK-NEXT: inch x8
-; CHECK-NEXT: st1d { z0.d }, p3, [x1, #3, mul vl]
-; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h
-; CHECK-NEXT: st1d { z1.d }, p2, [x1, #2, mul vl]
-; CHECK-NEXT: st1d { z2.d }, p1, [x1, #1, mul vl]
-; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1
-; CHECK-NEXT: st1d { z3.d }, p0, [x1]
+; CHECK-NEXT: st1d { z0.d }, p1, [x1, #3, mul vl]
+; CHECK-NEXT: st1d { z1.d }, p0, [x1, #2, mul vl]
+; CHECK-NEXT: st1d { z2.d }, p3, [x1, #1, mul vl]
+; CHECK-NEXT: st1d { z3.d }, p2, [x1]
; CHECK-NEXT: incb x1, all, mul #4
-; CHECK-NEXT: fmov w9, s0
-; CHECK-NEXT: tbnz w9, #0, .LBB2_2
+; CHECK-NEXT: b.mi .LBB2_2
; CHECK-NEXT: // %bb.3: // %exit
; CHECK-NEXT: ret
entry:
@@ -235,20 +207,17 @@ define void @rewrite_masked_load_store_i8_i32_induction(ptr %x, i32 %n) #0 {
; CHECK-NEXT: pext { p0.b, p1.b }, pn8[0]
; CHECK-NEXT: mov w12, w9
; CHECK-NEXT: decb x10, all, mul #2
-; CHECK-NEXT: whilelo pn8.b, x12, x8, vlx2
; CHECK-NEXT: incb x9, all, mul #2
-; CHECK-NEXT: pext { p2.b, p3.b }, pn8[0]
-; CHECK-NEXT: mov z2.b, p2/z, #1 // =0x1
+; CHECK-NEXT: whilelo pn8.b, x12, x8, vlx2
; CHECK-NEXT: sxtw x10, w10
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x10]
; CHECK-NEXT: add x11, x0, x10
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x10]
; CHECK-NEXT: ld1b { z1.b }, p1/z, [x11, #1, mul vl]
; CHECK-NEXT: add z0.b, z0.b, #1 // =0x1
; CHECK-NEXT: add z1.b, z1.b, #1 // =0x1
; CHECK-NEXT: st1b { z0.b }, p0, [x0, x10]
-; CHECK-NEXT: fmov w10, s2
; CHECK-NEXT: st1b { z1.b }, p1, [x11, #1, mul vl]
-; CHECK-NEXT: tbnz w10, #0, .LBB3_1
+; CHECK-NEXT: b.mi .LBB3_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret
entry:
@@ -289,16 +258,12 @@ define void @rewrite_masked_load_store_i32_vlx2(ptr %x, i64 %n) #0 {
; CHECK-NEXT: inch x8
; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT: pext { p2.s, p3.s }, pn8[0]
-; CHECK-NEXT: uzp1 p2.h, p2.h, p3.h
; CHECK-NEXT: add z0.s, z0.s, #1 // =0x1
; CHECK-NEXT: add z1.s, z1.s, #1 // =0x1
-; CHECK-NEXT: mov z2.h, p2/z, #1 // =0x1
; CHECK-NEXT: st1w { z0.s }, p1, [x0, #1, mul vl]
-; CHECK-NEXT: fmov w9, s2
; CHECK-NEXT: st1w { z1.s }, p0, [x0]
; CHECK-NEXT: incb x0, all, mul #2
-; CHECK-NEXT: tbnz w9, #0, .LBB4_1
+; CHECK-NEXT: b.mi .LBB4_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret
entry:
@@ -335,33 +300,26 @@ define void @mixed_data_vector_types(ptr %x, ptr %y, i64 %n) #0 {
; CHECK-NEXT: .LBB5_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0]
+; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1]
; CHECK-NEXT: add x9, x0, x8, lsl #3
+; CHECK-NEXT: uzp1 p4.s, p0.s, p1.s
; CHECK-NEXT: add x10, x1, x8, lsl #2
-; CHECK-NEXT: uzp1 p2.s, p0.s, p1.s
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
-; CHECK-NEXT: ld1d { z5.d }, p1/z, [x9, #1, mul vl]
-; CHECK-NEXT: ld1w { z1.s }, p2/z, [x1, x8, lsl #2]
+; CHECK-NEXT: uzp1 p5.s, p2.s, p3.s
+; CHECK-NEXT: ld1d { z1.d }, p3/z, [x9, #3, mul vl]
+; CHECK-NEXT: ld1d { z3.d }, p2/z, [x9, #2, mul vl]
+; CHECK-NEXT: ld1w { z2.s }, p4/z, [x1, x8, lsl #2]
; CHECK-NEXT: inch x8
-; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1]
-; CHECK-NEXT: ld1d { z3.d }, p3/z, [x9, #3, mul vl]
-; CHECK-NEXT: ld1d { z4.d }, p2/z, [x9, #2, mul vl]
+; CHECK-NEXT: ld1d { z4.d }, p1/z, [x9, #1, mul vl]
+; CHECK-NEXT: ld1w { z5.s }, p5/z, [x10, #1, mul vl]
; CHECK-NEXT: whilelo pn8.d, x8, x2, vlx4
-; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0]
-; CHECK-NEXT: uzp1 p0.s, p4.s, p5.s
-; CHECK-NEXT: pext { p4.d, p5.d }, pn8[1]
-; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s
-; CHECK-NEXT: uzp1 p0.h, p0.h, p4.h
-; CHECK-NEXT: uzp1 p4.s, p2.s, p3.s
-; CHECK-NEXT: mov z2.h, p0/z, #1 // =0x1
-; CHECK-NEXT: ld1w { z6.s }, p4/z, [x10, #1, mul vl]
-; CHECK-NEXT: fmov w9, s2
; CHECK-NEXT: // fake_use: $z0
-; CHECK-NEXT: // fake_use: $z5
; CHECK-NEXT: // fake_use: $z4
; CHECK-NEXT: // fake_use: $z3
; CHECK-NEXT: // fake_use: $z1
-; CHECK-NEXT: // fake_use: $z6
-; CHECK-NEXT: tbnz w9, #0, .LBB5_1
+; CHECK-NEXT: // fake_use: $z2
+; CHECK-NEXT: // fake_use: $z5
+; CHECK-NEXT: b.mi .LBB5_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret
entry:
@@ -402,28 +360,24 @@ define void @prefer_more_common_masked_access_size(ptr %x16, ptr %y32, i64 %n) #
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: pext { p0.h, p1.h }, pn8[0]
; CHECK-NEXT: add x9, x0, x8, lsl #1
+; CHECK-NEXT: add x10, x1, x8, lsl #2
; CHECK-NEXT: punpklo p2.h, p0.b
; CHECK-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
; CHECK-NEXT: st1h { z0.h }, p1, [x9, #1, mul vl]
-; CHECK-NEXT: add x9, x1, x8, lsl #2
-; CHECK-NEXT: punpkhi p0.h, p0.b
+; CHECK-NEXT: punpkhi p3.h, p1.b
+; CHECK-NEXT: punpklo p4.h, p1.b
; CHECK-NEXT: ld1w { z1.s }, p2/z, [x1, x8, lsl #2]
; CHECK-NEXT: incb x8
-; CHECK-NEXT: punpkhi p2.h, p1.b
-; CHECK-NEXT: punpklo p1.h, p1.b
-; CHECK-NEXT: ld1w { z5.s }, p0/z, [x9, #1, mul vl]
-; CHECK-NEXT: ld1w { z3.s }, p2/z, [x9, #3, mul vl]
+; CHECK-NEXT: punpkhi p0.h, p0.b
+; CHECK-NEXT: ld1w { z2.s }, p3/z, [x10, #3, mul vl]
+; CHECK-NEXT: ld1w { z3.s }, p4/z, [x10, #2, mul vl]
+; CHECK-NEXT: ld1w { z4.s }, p0/z, [x10, #1, mul vl]
; CHECK-NEXT: whilelo pn8.h, x8, x2, vlx2
-; CHECK-NEXT: ld1w { z4.s }, p1/z, [x9, #2, mul vl]
-; CHECK-NEXT: pext { p3.h, p4.h }, pn8[0]
-; CHECK-NEXT: uzp1 p3.b, p3.b, p4.b
-; CHECK-NEXT: mov z2.b, p3/z, #1 // =0x1
-; CHECK-NEXT: fmov w9, s2
; CHECK-NEXT: // fake_use: $z1
-; CHECK-NEXT: // fake_use: $z5
; CHECK-NEXT: // fake_use: $z4
; CHECK-NEXT: // fake_use: $z3
-; CHECK-NEXT: tbnz w9, #0, .LBB6_1
+; CHECK-NEXT: // fake_use: $z2
+; CHECK-NEXT: b.mi .LBB6_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret
entry:
@@ -466,28 +420,21 @@ define void @prefer_larger_access_size_on_tie(ptr %x16, ptr %y32, i64 %n) #0 {
; CHECK-NEXT: pext { p2.s, p3.s }, pn8[1]
; CHECK-NEXT: add x9, x0, x8, lsl #1
; CHECK-NEXT: uzp1 p4.h, p0.h, p1.h
+; CHECK-NEXT: add x10, x1, x8, lsl #2
; CHECK-NEXT: uzp1 p5.h, p2.h, p3.h
; CHECK-NEXT: st1h { z0.h }, p4, [x0, x8, lsl #1]
; CHECK-NEXT: st1h { z0.h }, p5, [x9, #1, mul vl]
-; CHECK-NEXT: add x9, x1, x8, lsl #2
; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
; CHECK-NEXT: incb x8
-; CHECK-NEXT: ld1w { z3.s }, p3/z, [x9, #3, mul vl]
-; CHECK-NEXT: ld1w { z4.s }, p2/z, [x9, #2, mul vl]
-; CHECK-NEXT: ld1w { z5.s }, p1/z, [x9, #1, mul vl]
+; CHECK-NEXT: ld1w { z2.s }, p3/z, [x10, #3, mul vl]
+; CHECK-NEXT: ld1w { z3.s }, p2/z, [x10, #2, mul vl]
+; CHECK-NEXT: ld1w { z4.s }, p1/z, [x10, #1, mul vl]
; CHECK-NEXT: whilelo pn8.s, x8, x2, vlx4
-; CHECK-NEXT: pext { p4.s, p5.s }, pn8[0]
-; CHECK-NEXT: uzp1 p0.h, p4.h, p5.h
-; CHECK-NEXT: pext { p4.s, p5.s }, pn8[1]
-; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h
-; CHECK-NEXT: uzp1 p0.b, p0.b, p4.b
-; CHECK-NEXT: mov z2.b, p0/z, #1 // =0x1
-; CHECK-NEXT: fmov w9, s2
; CHECK-NEXT: // fake_use: $z1
-; CHECK-NEXT: // fake_use: $z5
; CHECK-NEXT: // fake_use: $z4
; CHECK-NEXT: // fake_use: $z3
-; CHECK-NEXT: tbnz w9, #0, .LBB7_1
+; CHECK-NEXT: // fake_use: $z2
+; CHECK-NEXT: b.mi .LBB7_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret
entry:
@@ -527,11 +474,10 @@ define void @rewrite_extractelement_within_first_section_i32(ptr %x, i64 %n) #0
; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0]
; CHECK-NEXT: whilelo pn8.s, x8, x1, vlx2
; CHECK-NEXT: inch x8
-; CHECK-NEXT: pext { p2.s, p3.s }, pn8[0]
+; CHECK-NEXT: pext p2.s, pn8[0]
; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT: uzp1 p2.h, p2.h, p3.h
-; CHECK-NEXT: mov z0.h, p2/z, #1 // =0x1
-; CHECK-NEXT: umov w9, v0.h[3]
+; CHECK-NEXT: mov z0.s, p2/z, #1 // =0x1
+; CHECK-NEXT: mov w9, v0.s[3]
; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl]
; CHECK-NEXT: incb x0, all, mul #2
; CHECK-NEXT: // fake_use: $z1
@@ -571,31 +517,24 @@ define void @masked_load_passthru_non_poison(ptr %x, i64 %n) #0 {
; CHECK-NEXT: whilelo pn8.d, xzr, x1, vlx4
; CHECK-NEXT: .LBB9_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1]
-; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0]
+; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1]
+; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0]
; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4
-; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0, #3, mul vl]
-; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0]
-; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #2, mul vl]
-; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s
-; CHECK-NEXT: pext { p5.d, p6.d }, pn8[1]
-; CHECK-NEXT: ld1d { z2.d }, p1/z, [x0, #1, mul vl]
-; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s
-; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0]
+; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl]
+; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0]
; CHECK-NEXT: inch x8
; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
-; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h
; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
-; CHECK-NEXT: st1d { z0.d }, p3, [x0, #3, mul vl]
-; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1
-; CHECK-NEXT: st1d { z1.d }, p2, [x0, #2, mul vl]
-; CHECK-NEXT: st1d { z2.d }, p1, [x0, #1, mul vl]
-; CHECK-NEXT: st1d { z3.d }, p0, [x0]
+; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl]
+; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl]
+; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl]
+; CHECK-NEXT: st1d { z3.d }, p2, [x0]
; CHECK-NEXT: incb x0, all, mul #4
-; CHECK-NEXT: fmov w9, s0
-; CHECK-NEXT: tbnz w9, #0, .LBB9_1
+; CHECK-NEXT: b.mi .LBB9_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret
entry:
More information about the llvm-branch-commits
mailing list