[llvm-branch-commits] [llvm] [AArch64] Use multi-vector intrinsics for masked load/store users of predicate-as-counter (PR #220962)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Sep 3 07:57:16 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Benjamin Maxwell (MacDue)
<details>
<summary>Changes</summary>
If the user of the original wide mask is a masked load or store
intrinsic (matching the element size of the predicate-as-counter),
rewrite it directly to a masked multi-vector load/store.
This avoids materializing the vector mask and is easier to handle here
than later (e.g. in SelectionDAG), since we do not need to match the
concatenation of all `pext` segments of the predicate-as-counter.
Assisted-by: Codex
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
---
Patch is 21.72 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/220962.diff
2 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp (+110-1)
- (modified) llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll (+96-142)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp b/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp
index fa855d87793d3..7881751137425 100644
--- a/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp
@@ -229,6 +229,113 @@ static Value *createWhileLO(IRBuilder<> &Builder, unsigned ElementSizeInBits,
WhileLO, {Start, End, Builder.getInt32(VectorScale)}, "pac.mask");
}
+/// Returns the multi-vector load/store intrinsic ID for \p VectorScale.
+static Intrinsic::ID getPNLoadStoreIntrinsic(unsigned VectorScale,
+ bool IsLoad) {
+ if (VectorScale == 2)
+ return IsLoad ? Intrinsic::aarch64_sve_ld1_pn_x2
+ : Intrinsic::aarch64_sve_st1_pn_x2;
+ if (VectorScale == 4)
+ return IsLoad ? Intrinsic::aarch64_sve_ld1_pn_x4
+ : Intrinsic::aarch64_sve_st1_pn_x4;
+ llvm_unreachable("unsupported predicate-as-counter scale");
+}
+
+/// If \p UserI is a masked-load user of the original loop mask, rewrite it to
+/// a masked multi-vector load (using the predicate-as-counter) if the load
+/// access size matches the predicate-as-counter element size.
+static bool tryRewriteMaskedLoadUser(Instruction &UserI,
+ const MaskRewriteCandidate &C,
+ Value *Count) {
+ auto *II = dyn_cast<IntrinsicInst>(&UserI);
+ if (!II || II->getIntrinsicID() != Intrinsic::masked_load)
+ return false;
+
+ unsigned ElementSizeInBits =
+ getScalarSizeInBits(II->getModule()->getDataLayout(), II->getType());
+ if (ElementSizeInBits != C.ElementSizeInBits)
+ return false;
+
+ if (!isa<PoisonValue, UndefValue>(II->getArgOperand(2)))
+ return false;
+
+ IRBuilder<> Builder(II);
+ Builder.SetCurrentDebugLocation(II->getDebugLoc());
+
+ ElementCount LegalEC = getSVEElementCount(C.ElementSizeInBits);
+ Type *ScalarType = II->getType()->getScalarType();
+ auto *WideDataTy = VectorType::get(ScalarType, LegalEC * C.VectorScale);
+ auto *LegalDataTy = VectorType::get(ScalarType, LegalEC);
+
+ Module *M = II->getModule();
+ FunctionCallee LD1 = Intrinsic::getOrInsertDeclaration(
+ M, getPNLoadStoreIntrinsic(C.VectorScale, /*IsLoad=*/true),
+ {LegalDataTy, II->getArgOperand(0)->getType()});
+ auto *PNLoad =
+ Builder.CreateCall(LD1, {Count, II->getArgOperand(0)}, "pac.ld1");
+
+ // Copy pointer parameter attributes.
+ for (Attribute ParamAttr : II->getParamAttributes(0))
+ PNLoad->addParamAttr(1, ParamAttr);
+
+ // Concatenate all results into the original wide value.
+ Value *WideData = PoisonValue::get(WideDataTy);
+ for (unsigned Slice = 0; Slice != C.VectorScale; ++Slice) {
+ Value *Part = Builder.CreateExtractValue(PNLoad, Slice, "pac.data");
+ WideData = Builder.CreateInsertVector(WideDataTy, WideData, Part,
+ Slice * LegalEC.getKnownMinValue(),
+ "pac.vec");
+ }
+
+ II->replaceAllUsesWith(WideData);
+ II->eraseFromParent();
+ return true;
+}
+
+/// If \p UserI is a masked-store user of the original loop mask, rewrite it to
+/// a masked multi-vector store (using the predicate-as-counter) if the store
+/// access size matches the predicate-as-counter element size.
+static bool tryRewriteMaskedStoreUser(Instruction &UserI,
+ const MaskRewriteCandidate &C,
+ Value *Count) {
+ auto *II = dyn_cast<IntrinsicInst>(&UserI);
+ if (!II || II->getIntrinsicID() != Intrinsic::masked_store)
+ return false;
+
+ unsigned ElementSizeInBits = getScalarSizeInBits(
+ II->getModule()->getDataLayout(), II->getArgOperand(0)->getType());
+ if (ElementSizeInBits != C.ElementSizeInBits)
+ return false;
+
+ IRBuilder<> Builder(II);
+ Builder.SetCurrentDebugLocation(II->getDebugLoc());
+
+ Type *ScalarType = II->getArgOperand(0)->getType()->getScalarType();
+ ElementCount LegalEC = getSVEElementCount(C.ElementSizeInBits);
+ auto *LegalDataTy = VectorType::get(ScalarType, LegalEC);
+
+ SmallVector<Value *, 6> StoreArgs;
+ for (unsigned Slice = 0; Slice != C.VectorScale; ++Slice)
+ StoreArgs.push_back(Builder.CreateExtractVector(
+ LegalDataTy, II->getArgOperand(0), Slice * LegalEC.getKnownMinValue(),
+ "pac.data"));
+ StoreArgs.push_back(Count);
+ StoreArgs.push_back(II->getArgOperand(1));
+
+ Module *M = II->getModule();
+ FunctionCallee ST1 = Intrinsic::getOrInsertDeclaration(
+ M, getPNLoadStoreIntrinsic(C.VectorScale, /*IsLoad=*/false),
+ {LegalDataTy, II->getArgOperand(1)->getType()});
+ auto *PNStore = Builder.CreateCall(ST1, StoreArgs);
+
+ // Copy pointer parameter attributes.
+ for (Attribute ParamAttr : II->getParamAttributes(1))
+ PNStore->addParamAttr(C.VectorScale + 1, ParamAttr);
+
+ II->eraseFromParent();
+ return true;
+}
+
/// If \p UserI is an extractelement use of the original loop mask, attempt to
/// rewrite it to `extractelement(pext(count, 0), idx)` if the extract index is
/// known to be within the first mask section (which means pext index = 0). If
@@ -464,7 +571,9 @@ bool AArch64PredicateAsCounterLoopRewrites::rewriteCandidate(
Value *WideMask = nullptr;
for (Use *U : UsesToRewrite) {
auto *UserI = cast<Instruction>(U->getUser());
- if (tryRewriteExtractElement(*UserI, C, Count))
+ if (tryRewriteMaskedLoadUser(*UserI, C, Count) ||
+ tryRewriteMaskedStoreUser(*UserI, C, Count) ||
+ tryRewriteExtractElement(*UserI, C, Count))
continue;
if (!WideMask) {
diff --git a/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll b/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll
index dca70b3555b7d..f887048eb8ad5 100644
--- a/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll
+++ b/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll
@@ -11,22 +11,14 @@ define void @rewrite_masked_load_store_i64_vlx4(ptr %x, i64 %n) #0 {
; CHECK-NEXT: whilelo pn8.d, xzr, x1, vlx4
; CHECK-NEXT: .LBB0_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1]
-; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0]
+; CHECK-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x0]
+; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
+; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
+; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
+; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
+; CHECK-NEXT: st1d { z0.d - z3.d }, pn8, [x0]
; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4
-; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl]
-; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0]
; CHECK-NEXT: inch x8
-; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
-; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
-; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
-; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
-; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl]
-; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl]
-; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl]
-; CHECK-NEXT: st1d { z3.d }, p2, [x0]
; CHECK-NEXT: incb x0, all, mul #4
; CHECK-NEXT: b.mi .LBB0_1
; CHECK-NEXT: // %bb.2: // %exit
@@ -64,22 +56,14 @@ define void @rewrite_masked_load_store_i64_multi_pred_header(ptr %x, i64 %n, i1
; CHECK-NEXT: whilelo pn8.d, xzr, x1, vlx4
; CHECK-NEXT: .LBB1_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1]
-; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0]
+; CHECK-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x0]
+; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
+; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
+; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
+; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
+; CHECK-NEXT: st1d { z0.d - z3.d }, pn8, [x0]
; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4
-; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl]
-; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0]
; CHECK-NEXT: inch x8
-; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
-; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
-; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
-; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
-; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl]
-; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl]
-; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl]
-; CHECK-NEXT: st1d { z3.d }, p2, [x0]
; CHECK-NEXT: incb x0, all, mul #4
; CHECK-NEXT: b.mi .LBB1_1
; CHECK-NEXT: // %bb.2: // %exit
@@ -122,38 +106,22 @@ define void @shared_start_mask_between_loops(ptr %x, ptr %y, i64 %n) #0 {
; CHECK-NEXT: mov x9, x8
; CHECK-NEXT: .LBB2_1: // %loop1
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: pext { p0.d, p1.d }, pn9[1]
-; CHECK-NEXT: pext { p2.d, p3.d }, pn9[0]
+; CHECK-NEXT: ld1d { z0.d - z3.d }, pn9/z, [x0]
+; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
+; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
+; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
+; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
+; CHECK-NEXT: st1d { z0.d - z3.d }, pn9, [x0]
; CHECK-NEXT: whilelo pn9.d, x9, x2, vlx4
-; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl]
-; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0]
; CHECK-NEXT: inch x9
-; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
-; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
-; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
-; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
-; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl]
-; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl]
-; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl]
-; CHECK-NEXT: st1d { z3.d }, p2, [x0]
; CHECK-NEXT: incb x0, all, mul #4
; CHECK-NEXT: b.mi .LBB2_1
; CHECK-NEXT: .LBB2_2: // %loop2
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1]
-; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0]
+; CHECK-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x1]
+; CHECK-NEXT: st1d { z0.d - z3.d }, pn8, [x1]
; CHECK-NEXT: whilelo pn8.d, x8, x2, vlx4
-; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1, #3, mul vl]
-; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1, #2, mul vl]
-; CHECK-NEXT: ld1d { z2.d }, p3/z, [x1, #1, mul vl]
-; CHECK-NEXT: ld1d { z3.d }, p2/z, [x1]
; CHECK-NEXT: inch x8
-; CHECK-NEXT: st1d { z0.d }, p1, [x1, #3, mul vl]
-; CHECK-NEXT: st1d { z1.d }, p0, [x1, #2, mul vl]
-; CHECK-NEXT: st1d { z2.d }, p3, [x1, #1, mul vl]
-; CHECK-NEXT: st1d { z3.d }, p2, [x1]
; CHECK-NEXT: incb x1, all, mul #4
; CHECK-NEXT: b.mi .LBB2_2
; CHECK-NEXT: // %bb.3: // %exit
@@ -204,19 +172,16 @@ define void @rewrite_masked_load_store_i8_i32_induction(ptr %x, i32 %n) #0 {
; CHECK-NEXT: .LBB3_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: mov x10, x9
-; CHECK-NEXT: pext { p0.b, p1.b }, pn8[0]
-; CHECK-NEXT: mov w12, w9
-; CHECK-NEXT: decb x10, all, mul #2
+; CHECK-NEXT: mov w11, w9
; CHECK-NEXT: incb x9, all, mul #2
-; CHECK-NEXT: whilelo pn8.b, x12, x8, vlx2
+; CHECK-NEXT: decb x10, all, mul #2
; CHECK-NEXT: sxtw x10, w10
-; CHECK-NEXT: add x11, x0, x10
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x10]
-; CHECK-NEXT: ld1b { z1.b }, p1/z, [x11, #1, mul vl]
-; CHECK-NEXT: add z0.b, z0.b, #1 // =0x1
+; CHECK-NEXT: ld1b { z0.b, z1.b }, pn8/z, [x0, x10]
+; CHECK-NEXT: add x10, x0, x10
; CHECK-NEXT: add z1.b, z1.b, #1 // =0x1
-; CHECK-NEXT: st1b { z0.b }, p0, [x0, x10]
-; CHECK-NEXT: st1b { z1.b }, p1, [x11, #1, mul vl]
+; CHECK-NEXT: add z0.b, z0.b, #1 // =0x1
+; CHECK-NEXT: st1b { z0.b, z1.b }, pn8, [x10]
+; CHECK-NEXT: whilelo pn8.b, x11, x8, vlx2
; CHECK-NEXT: b.mi .LBB3_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret
@@ -253,15 +218,12 @@ define void @rewrite_masked_load_store_i32_vlx2(ptr %x, i64 %n) #0 {
; CHECK-NEXT: whilelo pn8.s, xzr, x1, vlx2
; CHECK-NEXT: .LBB4_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0]
+; CHECK-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0]
+; CHECK-NEXT: add z1.s, z1.s, #1 // =0x1
+; CHECK-NEXT: add z0.s, z0.s, #1 // =0x1
+; CHECK-NEXT: st1w { z0.s, z1.s }, pn8, [x0]
; CHECK-NEXT: whilelo pn8.s, x8, x1, vlx2
; CHECK-NEXT: inch x8
-; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT: add z0.s, z0.s, #1 // =0x1
-; CHECK-NEXT: add z1.s, z1.s, #1 // =0x1
-; CHECK-NEXT: st1w { z0.s }, p1, [x0, #1, mul vl]
-; CHECK-NEXT: st1w { z1.s }, p0, [x0]
; CHECK-NEXT: incb x0, all, mul #2
; CHECK-NEXT: b.mi .LBB4_1
; CHECK-NEXT: // %bb.2: // %exit
@@ -295,30 +257,27 @@ exit:
define void @mixed_data_vector_types(ptr %x, ptr %y, i64 %n) #0 {
; CHECK-LABEL: mixed_data_vector_types:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov x8, xzr
; CHECK-NEXT: whilelo pn8.d, xzr, x2, vlx4
+; CHECK-NEXT: cnth x8
; CHECK-NEXT: .LBB5_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0]
; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1]
-; CHECK-NEXT: add x9, x0, x8, lsl #3
-; CHECK-NEXT: uzp1 p4.s, p0.s, p1.s
-; CHECK-NEXT: add x10, x1, x8, lsl #2
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
-; CHECK-NEXT: uzp1 p5.s, p2.s, p3.s
-; CHECK-NEXT: ld1d { z1.d }, p3/z, [x9, #3, mul vl]
-; CHECK-NEXT: ld1d { z3.d }, p2/z, [x9, #2, mul vl]
-; CHECK-NEXT: ld1w { z2.s }, p4/z, [x1, x8, lsl #2]
-; CHECK-NEXT: inch x8
-; CHECK-NEXT: ld1d { z4.d }, p1/z, [x9, #1, mul vl]
-; CHECK-NEXT: ld1w { z5.s }, p5/z, [x10, #1, mul vl]
+; CHECK-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x0]
+; CHECK-NEXT: uzp1 p0.s, p0.s, p1.s
; CHECK-NEXT: whilelo pn8.d, x8, x2, vlx4
+; CHECK-NEXT: inch x8
+; CHECK-NEXT: uzp1 p1.s, p2.s, p3.s
+; CHECK-NEXT: incb x0, all, mul #4
+; CHECK-NEXT: ld1w { z5.s }, p0/z, [x1]
+; CHECK-NEXT: ld1w { z4.s }, p1/z, [x1, #1, mul vl]
+; CHECK-NEXT: incb x1, all, mul #2
; CHECK-NEXT: // fake_use: $z0
-; CHECK-NEXT: // fake_use: $z4
-; CHECK-NEXT: // fake_use: $z3
; CHECK-NEXT: // fake_use: $z1
; CHECK-NEXT: // fake_use: $z2
+; CHECK-NEXT: // fake_use: $z3
; CHECK-NEXT: // fake_use: $z5
+; CHECK-NEXT: // fake_use: $z4
; CHECK-NEXT: b.mi .LBB5_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret
@@ -354,26 +313,27 @@ define void @prefer_more_common_masked_access_size(ptr %x16, ptr %y32, i64 %n) #
; CHECK-LABEL: prefer_more_common_masked_access_size:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov x8, xzr
; CHECK-NEXT: whilelo pn8.h, xzr, x2, vlx2
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: .LBB6_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: pext { p0.h, p1.h }, pn8[0]
-; CHECK-NEXT: add x9, x0, x8, lsl #1
-; CHECK-NEXT: add x10, x1, x8, lsl #2
-; CHECK-NEXT: punpklo p2.h, p0.b
-; CHECK-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
-; CHECK-NEXT: st1h { z0.h }, p1, [x9, #1, mul vl]
-; CHECK-NEXT: punpkhi p3.h, p1.b
-; CHECK-NEXT: punpklo p4.h, p1.b
-; CHECK-NEXT: ld1w { z1.s }, p2/z, [x1, x8, lsl #2]
-; CHECK-NEXT: incb x8
-; CHECK-NEXT: punpkhi p0.h, p0.b
-; CHECK-NEXT: ld1w { z2.s }, p3/z, [x10, #3, mul vl]
-; CHECK-NEXT: ld1w { z3.s }, p4/z, [x10, #2, mul vl]
-; CHECK-NEXT: ld1w { z4.s }, p0/z, [x10, #1, mul vl]
+; CHECK-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
+; CHECK-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
+; CHECK-NEXT: punpkhi p2.h, p1.b
; CHECK-NEXT: whilelo pn8.h, x8, x2, vlx2
-; CHECK-NEXT: // fake_use: $z1
+; CHECK-NEXT: incb x8
+; CHECK-NEXT: punpklo p1.h, p1.b
+; CHECK-NEXT: incb x0, all, mul #2
+; CHECK-NEXT: punpkhi p3.h, p0.b
+; CHECK-NEXT: ld1w { z2.s }, p2/z, [x1, #3, mul vl]
+; CHECK-NEXT: punpklo p0.h, p0.b
+; CHECK-NEXT: ld1w { z3.s }, p1/z, [x1, #2, mul vl]
+; CHECK-NEXT: ld1w { z4.s }, p3/z, [x1, #1, mul vl]
+; CHECK-NEXT: ld1w { z5.s }, p0/z, [x1]
+; CHECK-NEXT: incb x1, all, mul #4
+; CHECK-NEXT: // fake_use: $z5
; CHECK-NEXT: // fake_use: $z4
; CHECK-NEXT: // fake_use: $z3
; CHECK-NEXT: // fake_use: $z2
@@ -412,28 +372,25 @@ define void @prefer_larger_access_size_on_tie(ptr %x16, ptr %y32, i64 %n) #0 {
; CHECK-LABEL: prefer_larger_access_size_on_tie:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov x8, xzr
; CHECK-NEXT: whilelo pn8.s, xzr, x2, vlx4
+; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: .LBB7_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0]
-; CHECK-NEXT: pext { p2.s, p3.s }, pn8[1]
-; CHECK-NEXT: add x9, x0, x8, lsl #1
-; CHECK-NEXT: uzp1 p4.h, p0.h, p1.h
-; CHECK-NEXT: add x10, x1, x8, lsl #2
-; CHECK-NEXT: uzp1 p5.h, p2.h, p3.h
-; CHECK-NEXT: st1h { z0.h }, p4, [x0, x8, lsl #1]
-; CHECK-NEXT: st1h { z0.h }, p5, [x9, #1, mul vl]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
-; CHECK-NEXT: incb x8
-; CHECK-NEXT: ld1w { z2.s }, p3/z, [x10, #3, mul vl]
-; CHECK-NEXT: ld1w { z3.s }, p2/z, [x10, #2, mul vl]
-; CHECK-NEXT: ld1w { z4.s }, p1/z, [x10, #1, mul vl]
+; CHECK-NEXT: pext { p0.s, p1.s }, pn8[1]
+; CHECK-NEXT: pext { p2.s, p3.s }, pn8[0]
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: uzp1 p1.h, p2.h, p3.h
+; CHECK-NEXT: st1h { z0.h }, p0, [x0, #1, mul vl]
+; CHECK-NEXT: st1h { z0.h }, p1, [x0]
+; CHECK-NEXT: incb x0, all, mul #2
+; CHECK-NEXT: ld1w { z4.s - z7.s }, pn8/z, [x1]
; CHECK-NEXT: whilelo pn8.s, x8, x2, vlx4
-; CHECK-NEXT: // fake_use: $z1
+; CHECK-NEXT: incb x8
+; CHECK-NEXT: incb x1, all, mul #4
; CHECK-NEXT: // fake_use: $z4
-; CHECK-NEXT: // fake_use: $z3
-; CHECK-NEXT: // fake_use: $z2
+; CHECK-NEXT: // fake_use: $z5
+; CHECK-NEXT: // fake_use: $z6
+; CHECK-NEXT: // fake_use: $z7
; CHECK-NEXT: b.mi .LBB7_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret
@@ -471,17 +428,15 @@ define void @rewrite_extractelement_within_first_section_i32(ptr %x, i64 %n) #0
; CHECK-NEXT: whilelo pn8.s, xzr, x1, vlx2
; CHECK-NEXT: .LBB8_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0]
+; CHECK-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0]
; CHECK-NEXT: whilelo pn8.s, x8, x1, vlx2
; CHECK-NEXT: inch x8
-; CHECK-NEXT: pext p2.s, pn8[0]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
-; CHECK-NEXT: mov z0.s, p2/z, #1 // =0x1
-; CHECK-NEXT: mov w9, v0.s[3]
-; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl]
+; CHECK-NEXT: pext p0.s, pn8[0]
; CHECK-NEXT: incb x0, all, mul #2
-; CHECK-NEXT: // fake_use: $z1
+; CHECK-NEXT: mov z2.s, p0/z, #1 // =0x1
+; CHECK-NEXT: mov w9, v2.s[3]
; CHECK-NEXT: // fake_use: $z0
+; CHECK-NEXT: // fake_use: $z1
; CHECK-NEXT: tbnz w9, #0, .LBB8_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: ret
@@ -518,21 +473,22 @@ define void @masked_load_passthru_non_poison(ptr %x, i64 %n) #0 {
; CHECK-NEXT: .LBB9_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1]
-; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0]
-; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4
; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl]
+; CHECK-NEXT: pext { p1.d, p2.d }, pn8[0]
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl]
-; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0]
+; CHECK-NEXT: ld1d { z2.d }, p2/z, [x0, #1, mul vl]
+; CHECK-NEXT: ld1d { z3.d }, p1/z, [x0]
+; CHECK-NEXT: movprfx z7, z0
+; CHECK-NEXT: add z7.d, z7.d, #1 // =0x1
+; CHECK-NEXT: movprfx z6, z1
+; CHECK-NEXT: add z6.d, z6.d, #1 // =0x1
+; CHECK-NEXT: movprfx z5, z2
+; CHECK-NEXT: add z5.d, z5.d, #1 // =0x1
+; C...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/220962
More information about the llvm-branch-commits
mailing list