[llvm] [RISCV] Fixed redundant add instructions generated during the fold of vscale add in rvv. (PR #225266)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Sep 26 00:13:20 PDT 2026
https://github.com/wangzhiyong99 updated https://github.com/llvm/llvm-project/pull/225266
>From e3f45a030f14a89a6eed91407ba4a1b49ea91f4a Mon Sep 17 00:00:00 2001
From: FoolgryGamer <1353470175 at qq.com>
Date: Tue, 22 Sep 2026 10:17:51 +0800
Subject: [PATCH 1/3] [RISCV] Fix incorrect vscale addition folding The
DAGCombiner was folding (A + vscale(C1)) + vscale(C2) into A + vscale(C1+C2)
even when the inner add had multiple uses, which could introduce an extra add
instruction on RISC-V.
This patch restricts the fold to the case where the inner add has
only one use.
Fixes #222933
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 2 +-
.../RISCV/rvv/vscale-add-fold-multi-use.ll | 65 +++++++++++++++++++
2 files changed, 66 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/RISCV/rvv/vscale-add-fold-multi-use.ll
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 17e2ab01bc11f..35fe060d53c8d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -3329,7 +3329,7 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
// fold a+vscale(c1)+vscale(c2) -> a+vscale(c1+c2)
if (N0.getOpcode() == ISD::ADD &&
N0.getOperand(1).getOpcode() == ISD::VSCALE &&
- N1.getOpcode() == ISD::VSCALE) {
+ N1.getOpcode() == ISD::VSCALE && N0.hasOneUse()) {
const APInt &VS0 = N0.getOperand(1)->getConstantOperandAPInt(0);
const APInt &VS1 = N1->getConstantOperandAPInt(0);
SDValue VS = DAG.getVScale(DL, VT, VS0 + VS1);
diff --git a/llvm/test/CodeGen/RISCV/rvv/vscale-add-fold-multi-use.ll b/llvm/test/CodeGen/RISCV/rvv/vscale-add-fold-multi-use.ll
new file mode 100644
index 0000000000000..a02b3797fcbaf
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vscale-add-fold-multi-use.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d < %s | FileCheck %s
+
+define <vscale x 64 x i32> @ret_split_nxv64i32(ptr %x) {
+; CHECK-LABEL: ret_split_nxv64i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: csrr a2, vlenb
+; CHECK-NEXT: slli a3, a2, 4
+; CHECK-NEXT: slli a2, a2, 3
+; CHECK-NEXT: add a4, a1, a3
+; CHECK-NEXT: add a5, a4, a2
+; CHECK-NEXT: vl8re32.v v8, (a5)
+; CHECK-NEXT: add a5, a1, a2
+; CHECK-NEXT: vl8re32.v v16, (a5)
+; CHECK-NEXT: vl8re32.v v24, (a4)
+; CHECK-NEXT: vl8re32.v v0, (a1)
+; CHECK-NEXT: vs8r.v v0, (a0)
+; CHECK-NEXT: add a3, a0, a3
+; CHECK-NEXT: vs8r.v v24, (a3)
+; CHECK-NEXT: add a0, a0, a2
+; CHECK-NEXT: vs8r.v v16, (a0)
+; CHECK-NEXT: add a2, a3, a2
+; CHECK-NEXT: vs8r.v v8, (a2)
+; CHECK-NEXT: ret
+ %v = load <vscale x 64 x i32>, ptr %x
+ ret <vscale x 64 x i32> %v
+}
+
+define <vscale x 32 x i64> @lrint_nxv32f32(<vscale x 32 x float> %x) {
+; CHECK-LABEL: lrint_nxv32f32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: csrr a1, vlenb
+; CHECK-NEXT: slli a2, a1, 4
+; CHECK-NEXT: add a2, a0, a2
+; CHECK-NEXT: vsetvli a3, zero, e32, m4, ta, ma
+; CHECK-NEXT: vfwcvt.x.f.v v24, v8
+; CHECK-NEXT: vfwcvt.x.f.v v0, v16
+; CHECK-NEXT: vs8r.v v24, (a0)
+; CHECK-NEXT: vs8r.v v0, (a2)
+; CHECK-NEXT: vfwcvt.x.f.v v24, v12
+; CHECK-NEXT: slli a1, a1, 3
+; CHECK-NEXT: add a0, a0, a1
+; CHECK-NEXT: vs8r.v v24, (a0)
+; CHECK-NEXT: vfwcvt.x.f.v v8, v20
+; CHECK-NEXT: add a1, a2, a1
+; CHECK-NEXT: vs8r.v v8, (a1)
+; CHECK-NEXT: ret
+ %a = call <vscale x 32 x i64> @llvm.lrint.nxv32i64.nxv32f32(<vscale x 32 x float> %x)
+ ret <vscale x 32 x i64> %a
+}
+
+define void @load_nxv48i32(ptr %x) {
+; CHECK-LABEL: load_nxv48i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vl8re32.v v8, (a0)
+; CHECK-NEXT: csrr a1, vlenb
+; CHECK-NEXT: slli a1, a1, 3
+; CHECK-NEXT: add a0, a0, a1
+; CHECK-NEXT: vl8re32.v v8, (a0)
+; CHECK-NEXT: add a0, a0, a1
+; CHECK-NEXT: vl8re32.v v8, (a0)
+; CHECK-NEXT: ret
+ %v = load volatile <vscale x 48 x i32>, ptr %x
+ ret void
+}
>From e37815ba31e9b9228140afbd0e96e4606108b05c Mon Sep 17 00:00:00 2001
From: FoolgryGamer <1353470175 at qq.com>
Date: Fri, 25 Sep 2026 16:06:08 +0800
Subject: [PATCH 2/3] [RISCV] Updated files that failed tests using
llvm/utils/update_llc_test_checks.py (NFC)
---
...plex-deinterleaving-reductions-scalable.ll | 25 +-
.../AArch64/named-vector-shuffles-sve.ll | 66 +-
llvm/test/CodeGen/AArch64/nontemporal-load.ll | 20 +-
.../scalable_masked_deinterleaved_loads.ll | 4 +-
.../scalable_masked_interleaved_stores.ll | 6 +-
llvm/test/CodeGen/AArch64/sve-aliasing.ll | 30 +-
.../AArch64/sve-calling-convention-mixed.ll | 84 +-
.../sve-extract-fixed-from-scalable-vector.ll | 53 +-
.../test/CodeGen/AArch64/sve-fptrunc-store.ll | 18 +-
.../AArch64/sve-gather-scatter-dag-combine.ll | 28 +-
.../test/CodeGen/AArch64/sve-insert-vector.ll | 35 +-
.../AArch64/sve-intrinsics-ldst-ext.ll | 50 +-
llvm/test/CodeGen/AArch64/sve-ldst-sext.ll | 41 +-
llvm/test/CodeGen/AArch64/sve-ldst-zext.ll | 41 +-
llvm/test/CodeGen/AArch64/sve-llrint.ll | 331 ++++----
.../AArch64/sve-load-store-legalisation.ll | 110 ++-
llvm/test/CodeGen/AArch64/sve-lrint.ll | 331 ++++----
.../CodeGen/AArch64/sve-masked-ldst-sext.ll | 19 +-
.../CodeGen/AArch64/sve-masked-ldst-zext.ll | 19 +-
.../AArch64/sve-masked-scatter-legalize.ll | 19 +-
.../AArch64/sve-multivector-load-stores.ll | 238 +++---
.../CodeGen/AArch64/sve-split-extract-elt.ll | 30 +-
.../CodeGen/AArch64/sve-split-insert-elt.ll | 29 +-
llvm/test/CodeGen/AArch64/sve-split-load.ll | 29 +-
llvm/test/CodeGen/AArch64/sve-split-store.ll | 29 +-
.../AArch64/sve-vector-interleave-widen.ll | 479 ++++++------
llvm/test/CodeGen/AArch64/zext-to-tbl.ll | 40 +-
.../CodeGen/RISCV/rvv/calling-conv-fastcc.ll | 144 ++--
llvm/test/CodeGen/RISCV/rvv/lrint-sdnode.ll | 102 +--
llvm/test/CodeGen/RISCV/rvv/lround-sdnode.ll | 108 +--
.../CodeGen/RISCV/rvv/vector-interleave.ll | 520 ++++++-------
llvm/test/CodeGen/RISCV/rvv/vsub-sdnode.ll | 716 +++++++++---------
32 files changed, 1990 insertions(+), 1804 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
index 480f5cca7ad7b..3b8ec4ab4b370 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
@@ -186,29 +186,32 @@ define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
; CHECK-NEXT: neg x9, x8
; CHECK-NEXT: mov w10, #1000 // =0x3e8
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: rdvl x11, #4
; CHECK-NEXT: and x9, x9, x10
-; CHECK-NEXT: rdvl x10, #4
+; CHECK-NEXT: rdvl x10, #2
; CHECK-NEXT: .LBB2_1: // %vector.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: add x12, x0, x10
+; CHECK-NEXT: add x13, x1, x10
; CHECK-NEXT: ldr z4, [x0, #1, mul vl]
; CHECK-NEXT: ldr z5, [x0]
-; CHECK-NEXT: subs x9, x9, x8
-; CHECK-NEXT: ldr z6, [x0, #3, mul vl]
-; CHECK-NEXT: ldr z7, [x1, #1, mul vl]
+; CHECK-NEXT: ldr z6, [x1, #1, mul vl]
+; CHECK-NEXT: ldr z7, [x12, #1, mul vl]
; CHECK-NEXT: ldr z16, [x1]
; CHECK-NEXT: ldr z17, [x0, #2, mul vl]
-; CHECK-NEXT: add x0, x0, x10
-; CHECK-NEXT: ldr z18, [x1, #3, mul vl]
+; CHECK-NEXT: subs x9, x9, x8
+; CHECK-NEXT: ldr z18, [x13, #1, mul vl]
; CHECK-NEXT: ldr z19, [x1, #2, mul vl]
-; CHECK-NEXT: add x1, x1, x10
+; CHECK-NEXT: add x1, x1, x11
+; CHECK-NEXT: fcmla z0.d, p0/m, z6.d, z4.d, #0
+; CHECK-NEXT: add x0, x0, x11
; CHECK-NEXT: fcmla z1.d, p0/m, z16.d, z5.d, #0
-; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #0
-; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z6.d, #0
; CHECK-NEXT: fcmla z2.d, p0/m, z19.d, z17.d, #0
+; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z7.d, #0
+; CHECK-NEXT: fcmla z0.d, p0/m, z6.d, z4.d, #90
; CHECK-NEXT: fcmla z1.d, p0/m, z16.d, z5.d, #90
-; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #90
-; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z6.d, #90
; CHECK-NEXT: fcmla z2.d, p0/m, z19.d, z17.d, #90
+; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z7.d, #90
; CHECK-NEXT: b.ne .LBB2_1
; CHECK-NEXT: // %bb.2: // %exit.block
; CHECK-NEXT: uzp1 z4.d, z2.d, z3.d
diff --git a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
index 747c134913931..75d125d8beda2 100644
--- a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
+++ b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
@@ -464,11 +464,13 @@ define <vscale x 8 x i32> @splice_nxv8i32_idx(<vscale x 8 x i32> %a, <vscale x 8
; CHECK: // %bb.0:
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-4
-; CHECK-NEXT: mov x8, sp
+; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: str z1, [sp, #1, mul vl]
+; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: orr x8, x8, #0x8
-; CHECK-NEXT: str z3, [sp, #3, mul vl]
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
+; CHECK-NEXT: orr x8, x9, #0x8
; CHECK-NEXT: str z2, [sp, #2, mul vl]
; CHECK-NEXT: ldr z0, [x8]
; CHECK-NEXT: ldr z1, [x8, #1, mul vl]
@@ -485,22 +487,28 @@ define <vscale x 16 x float> @splice_nxv16f32_16(<vscale x 16 x float> %a, <vsca
; CHECK: // %bb.0:
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-8
+; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: str z3, [sp, #3, mul vl]
-; CHECK-NEXT: mov x8, #16 // =0x10
+; CHECK-NEXT: add x10, x9, x8
+; CHECK-NEXT: str z3, [x10, #1, mul vl]
+; CHECK-NEXT: rdvl x10, #4
+; CHECK-NEXT: add x10, x9, x10
; CHECK-NEXT: str z2, [sp, #2, mul vl]
-; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: add x11, x10, x8
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: str z7, [sp, #7, mul vl]
+; CHECK-NEXT: str z7, [x11, #1, mul vl]
+; CHECK-NEXT: add x11, x9, #64
+; CHECK-NEXT: str z5, [x10, #1, mul vl]
+; CHECK-NEXT: add x8, x11, x8
+; CHECK-NEXT: str z6, [x10, #2, mul vl]
+; CHECK-NEXT: mov x10, #16 // =0x10
; CHECK-NEXT: str z4, [sp, #4, mul vl]
-; CHECK-NEXT: str z5, [sp, #5, mul vl]
-; CHECK-NEXT: str z6, [sp, #6, mul vl]
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x9, x8, lsl #2]
-; CHECK-NEXT: add x8, x9, #64
-; CHECK-NEXT: ldr z1, [x8, #1, mul vl]
-; CHECK-NEXT: ldr z2, [x8, #2, mul vl]
-; CHECK-NEXT: ldr z3, [x8, #3, mul vl]
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x9, x10, lsl #2]
+; CHECK-NEXT: ldr z3, [x8, #1, mul vl]
+; CHECK-NEXT: ldr z1, [x11, #1, mul vl]
+; CHECK-NEXT: ldr z2, [x11, #2, mul vl]
; CHECK-NEXT: addvl sp, sp, #8
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
@@ -1066,7 +1074,7 @@ define <vscale x 8 x i32> @splice_nxv8i32(<vscale x 8 x i32> %a, <vscale x 8 x i
; CHECK-NEXT: mov x9, #-8 // =0xfffffffffffffff8
; CHECK-NEXT: str z0, [sp]
; CHECK-NEXT: sub x10, x8, #32
-; CHECK-NEXT: str z3, [sp, #3, mul vl]
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
; CHECK-NEXT: str z2, [sp, #2, mul vl]
; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8, x9, lsl #2]
; CHECK-NEXT: ldr z1, [x10, #1, mul vl]
@@ -1083,24 +1091,28 @@ define <vscale x 16 x float> @splice_nxv16f32_neg17(<vscale x 16 x float> %a, <v
; CHECK: // %bb.0:
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-8
-; CHECK-NEXT: rdvl x8, #4
+; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: str z3, [sp, #3, mul vl]
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: mov x9, #-17 // =0xffffffffffffffef
+; CHECK-NEXT: add x10, x9, x8
+; CHECK-NEXT: str z3, [x10, #1, mul vl]
+; CHECK-NEXT: rdvl x10, #4
+; CHECK-NEXT: add x9, x9, x10
; CHECK-NEXT: str z2, [sp, #2, mul vl]
+; CHECK-NEXT: add x10, x9, x8
+; CHECK-NEXT: sub x11, x9, #68
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: str z7, [sp, #7, mul vl]
+; CHECK-NEXT: add x8, x11, x8
+; CHECK-NEXT: str z7, [x10, #1, mul vl]
+; CHECK-NEXT: mov x10, #-17 // =0xffffffffffffffef
+; CHECK-NEXT: str z5, [x9, #1, mul vl]
+; CHECK-NEXT: str z6, [x9, #2, mul vl]
; CHECK-NEXT: str z4, [sp, #4, mul vl]
-; CHECK-NEXT: str z5, [sp, #5, mul vl]
-; CHECK-NEXT: str z6, [sp, #6, mul vl]
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8, x9, lsl #2]
-; CHECK-NEXT: sub x8, x8, #68
-; CHECK-NEXT: ldr z1, [x8, #1, mul vl]
-; CHECK-NEXT: ldr z2, [x8, #2, mul vl]
-; CHECK-NEXT: ldr z3, [x8, #3, mul vl]
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x9, x10, lsl #2]
+; CHECK-NEXT: ldr z1, [x11, #1, mul vl]
+; CHECK-NEXT: ldr z2, [x11, #2, mul vl]
+; CHECK-NEXT: ldr z3, [x8, #1, mul vl]
; CHECK-NEXT: addvl sp, sp, #8
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/nontemporal-load.ll b/llvm/test/CodeGen/AArch64/nontemporal-load.ll
index 662a95e4477b7..ec33dbefb9309 100644
--- a/llvm/test/CodeGen/AArch64/nontemporal-load.ll
+++ b/llvm/test/CodeGen/AArch64/nontemporal-load.ll
@@ -609,21 +609,29 @@ define <vscale x 20 x float> @test_ldnp_v20f32_vscale(ptr %A) {
; CHECK-LE-LABEL: test_ldnp_v20f32_vscale:
; CHECK-LE: ; %bb.0:
; CHECK-LE-NEXT: ptrue p0.s
+; CHECK-LE-NEXT: rdvl x8, #1
+; CHECK-LE-NEXT: add x9, x0, x8
+; CHECK-LE-NEXT: ldnt1w { z2.s }, p0/z, [x9, #1, mul vl]
+; CHECK-LE-NEXT: add x9, x9, x8
+; CHECK-LE-NEXT: add x8, x9, x8
; CHECK-LE-NEXT: ldnt1w { z0.s }, p0/z, [x0]
; CHECK-LE-NEXT: ldnt1w { z1.s }, p0/z, [x0, #1, mul vl]
-; CHECK-LE-NEXT: ldnt1w { z2.s }, p0/z, [x0, #2, mul vl]
-; CHECK-LE-NEXT: ldnt1w { z3.s }, p0/z, [x0, #3, mul vl]
-; CHECK-LE-NEXT: ldnt1w { z4.s }, p0/z, [x0, #4, mul vl]
+; CHECK-LE-NEXT: ldnt1w { z3.s }, p0/z, [x9, #1, mul vl]
+; CHECK-LE-NEXT: ldnt1w { z4.s }, p0/z, [x8, #1, mul vl]
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: test_ldnp_v20f32_vscale:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: ptrue p0.s
+; CHECK-BE-NEXT: rdvl x8, #1
+; CHECK-BE-NEXT: add x9, x0, x8
+; CHECK-BE-NEXT: ldnt1w { z2.s }, p0/z, [x9, #1, mul vl]
+; CHECK-BE-NEXT: add x9, x9, x8
+; CHECK-BE-NEXT: add x8, x9, x8
; CHECK-BE-NEXT: ldnt1w { z0.s }, p0/z, [x0]
; CHECK-BE-NEXT: ldnt1w { z1.s }, p0/z, [x0, #1, mul vl]
-; CHECK-BE-NEXT: ldnt1w { z2.s }, p0/z, [x0, #2, mul vl]
-; CHECK-BE-NEXT: ldnt1w { z3.s }, p0/z, [x0, #3, mul vl]
-; CHECK-BE-NEXT: ldnt1w { z4.s }, p0/z, [x0, #4, mul vl]
+; CHECK-BE-NEXT: ldnt1w { z3.s }, p0/z, [x9, #1, mul vl]
+; CHECK-BE-NEXT: ldnt1w { z4.s }, p0/z, [x8, #1, mul vl]
; CHECK-BE-NEXT: ret
%lv = load<vscale x 20 x float>, ptr %A, align 8, !nontemporal !0
ret <vscale x 20 x float> %lv
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
index f5129c7c57826..868e03bde6e52 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
@@ -223,11 +223,13 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
; CHECK-LABEL: foo_ld4_nxv16i8_bad_mask2:
; CHECK: // %bb.0:
; CHECK-NEXT: zip1 p2.b, p1.b, p1.b
+; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: zip2 p1.b, p1.b, p1.b
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: zip2 p3.b, p0.b, p0.b
; CHECK-NEXT: ld1b { z3.b }, p2/z, [x0, #2, mul vl]
; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
-; CHECK-NEXT: ld1b { z2.b }, p1/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1b { z2.b }, p1/z, [x8, #1, mul vl]
; CHECK-NEXT: ld1b { z0.b }, p3/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
; CHECK-NEXT: uzp1 z4.b, z3.b, z2.b
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index 766777925e909..ccdbb21a6e92b 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -271,17 +271,19 @@ define void @foo_st4_nxv4i32_bad_mask2(<vscale x 8 x i1> %mask, <vscale x 4 x i3
; CHECK: // %bb.0:
; CHECK-NEXT: zip2 z4.s, z1.s, z3.s
; CHECK-NEXT: zip2 z5.s, z0.s, z2.s
+; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: zip2 p1.h, p0.h, p0.h
; CHECK-NEXT: zip1 z1.s, z1.s, z3.s
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: zip1 z0.s, z0.s, z2.s
; CHECK-NEXT: zip1 p0.h, p0.h, p0.h
-; CHECK-NEXT: punpkhi p2.h, p1.b
; CHECK-NEXT: zip2 z2.s, z5.s, z4.s
+; CHECK-NEXT: punpkhi p2.h, p1.b
; CHECK-NEXT: zip1 z3.s, z5.s, z4.s
; CHECK-NEXT: punpklo p1.h, p1.b
; CHECK-NEXT: zip2 z4.s, z0.s, z1.s
; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
-; CHECK-NEXT: st1w { z2.s }, p2, [x0, #3, mul vl]
+; CHECK-NEXT: st1w { z2.s }, p2, [x8, #1, mul vl]
; CHECK-NEXT: punpkhi p2.h, p0.b
; CHECK-NEXT: punpklo p0.h, p0.b
; CHECK-NEXT: st1w { z3.s }, p1, [x0, #2, mul vl]
diff --git a/llvm/test/CodeGen/AArch64/sve-aliasing.ll b/llvm/test/CodeGen/AArch64/sve-aliasing.ll
index a27429a256250..0c91472d2bc98 100644
--- a/llvm/test/CodeGen/AArch64/sve-aliasing.ll
+++ b/llvm/test/CodeGen/AArch64/sve-aliasing.ll
@@ -458,19 +458,21 @@ define void @triple_v16i8(ptr noalias nocapture noundef %l0) {
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: ldr z2, [x0, #2, mul vl]
-; CHECK-NEXT: movprfx z3, z0
-; CHECK-NEXT: mul z3.b, p0/m, z3.b, z0.b
-; CHECK-NEXT: movprfx z4, z1
-; CHECK-NEXT: mul z4.b, p0/m, z4.b, z1.b
-; CHECK-NEXT: movprfx z5, z2
-; CHECK-NEXT: mul z5.b, p0/m, z5.b, z2.b
-; CHECK-NEXT: eor z0.d, z3.d, z0.d
-; CHECK-NEXT: eor z1.d, z4.d, z1.d
-; CHECK-NEXT: eor z2.d, z5.d, z2.d
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: add x8, x0, x8
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: mul z2.b, p0/m, z2.b, z0.b
+; CHECK-NEXT: movprfx z3, z1
+; CHECK-NEXT: mul z3.b, p0/m, z3.b, z1.b
+; CHECK-NEXT: eor z0.d, z2.d, z0.d
+; CHECK-NEXT: eor z1.d, z3.d, z1.d
; CHECK-NEXT: str z0, [x0]
; CHECK-NEXT: str z1, [x0, #1, mul vl]
-; CHECK-NEXT: str z2, [x0, #2, mul vl]
+; CHECK-NEXT: ldr z0, [x8, #1, mul vl]
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: mul z1.b, p0/m, z1.b, z0.b
+; CHECK-NEXT: eor z0.d, z1.d, z0.d
+; CHECK-NEXT: str z0, [x8, #1, mul vl]
; CHECK-NEXT: ret
%l3 = load <vscale x 16 x i8>, ptr %l0, align 16
%l5 = mul <vscale x 16 x i8> %l3, %l3
@@ -497,6 +499,7 @@ define void @negative_tripletooshort_v16i8(ptr noalias nocapture noundef %l0) {
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ptrue p0.b
; CHECK-NEXT: cntw x8
+; CHECK-NEXT: add x9, x0, x8
; CHECK-NEXT: movprfx z1, z0
; CHECK-NEXT: mul z1.b, p0/m, z1.b, z0.b
; CHECK-NEXT: eor z0.d, z1.d, z0.d
@@ -506,12 +509,11 @@ define void @negative_tripletooshort_v16i8(ptr noalias nocapture noundef %l0) {
; CHECK-NEXT: mul z1.b, p0/m, z1.b, z0.b
; CHECK-NEXT: eor z0.d, z1.d, z0.d
; CHECK-NEXT: st1b { z0.b }, p0, [x0, x8]
-; CHECK-NEXT: cnth x8
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8]
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x9, x8]
; CHECK-NEXT: movprfx z1, z0
; CHECK-NEXT: mul z1.b, p0/m, z1.b, z0.b
; CHECK-NEXT: eor z0.d, z1.d, z0.d
-; CHECK-NEXT: st1b { z0.b }, p0, [x0, x8]
+; CHECK-NEXT: st1b { z0.b }, p0, [x9, x8]
; CHECK-NEXT: ret
%l3 = load <vscale x 16 x i8>, ptr %l0, align 16
%l5 = mul <vscale x 16 x i8> %l3, %l3
diff --git a/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll b/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll
index 3c5e788dabd56..c8cb64ffe1091 100644
--- a/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll
+++ b/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll
@@ -12,13 +12,17 @@ define float @foo1(ptr %x0, ptr %x1, ptr %x2) nounwind {
; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: fmov s0, #1.00000000
+; CHECK-NEXT: add x9, x9, x8
; CHECK-NEXT: ld4d { z1.d - z4.d }, p0/z, [x0]
+; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: mov x0, sp
; CHECK-NEXT: ld4d { z16.d - z19.d }, p0/z, [x1]
; CHECK-NEXT: ld1d { z5.d }, p0/z, [x2]
-; CHECK-NEXT: str z19, [sp, #3, mul vl]
-; CHECK-NEXT: str z18, [sp, #2, mul vl]
+; CHECK-NEXT: str z19, [x8, #1, mul vl]
+; CHECK-NEXT: str z18, [x9, #1, mul vl]
; CHECK-NEXT: str z17, [sp, #1, mul vl]
; CHECK-NEXT: str z16, [sp]
; CHECK-NEXT: bl callee1
@@ -57,21 +61,25 @@ define float @foo2(ptr %x0, ptr %x1) nounwind {
; CHECK-NEXT: sub sp, sp, #16
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: add x9, sp, #16
+; CHECK-NEXT: add x10, x9, x8
; CHECK-NEXT: fmov s0, #1.00000000
-; CHECK-NEXT: add x8, sp, #16
; CHECK-NEXT: mov w2, #2 // =0x2
-; CHECK-NEXT: mov w3, #3 // =0x3
-; CHECK-NEXT: mov w4, #4 // =0x4
; CHECK-NEXT: ld4d { z1.d - z4.d }, p0/z, [x0]
+; CHECK-NEXT: add x8, x10, x8
; CHECK-NEXT: mov w0, wzr
+; CHECK-NEXT: mov w3, #3 // =0x3
+; CHECK-NEXT: mov w4, #4 // =0x4
; CHECK-NEXT: mov w5, #5 // =0x5
; CHECK-NEXT: mov w6, #6 // =0x6
; CHECK-NEXT: mov w7, #7 // =0x7
; CHECK-NEXT: ld4d { z16.d - z19.d }, p0/z, [x1]
; CHECK-NEXT: mov w1, #1 // =0x1
-; CHECK-NEXT: str x8, [sp]
-; CHECK-NEXT: str z19, [x8, #3, mul vl]
-; CHECK-NEXT: str z18, [x8, #2, mul vl]
+; CHECK-NEXT: str z19, [x8, #1, mul vl]
+; CHECK-NEXT: add x8, sp, #16
+; CHECK-NEXT: str z18, [x10, #1, mul vl]
+; CHECK-NEXT: str x9, [sp]
; CHECK-NEXT: str z17, [x8, #1, mul vl]
; CHECK-NEXT: str z16, [x8]
; CHECK-NEXT: bl callee2
@@ -111,11 +119,14 @@ define float @foo3(ptr %x0, ptr %x1, ptr %x2) nounwind {
; CHECK-NEXT: ptrue p0.b
; CHECK-NEXT: fmov s0, #1.00000000
; CHECK-NEXT: fmov s1, #2.00000000
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: ld4d { z2.d - z5.d }, p0/z, [x0]
+; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: mov x0, sp
; CHECK-NEXT: ld3d { z16.d - z18.d }, p0/z, [x1]
; CHECK-NEXT: ld1d { z6.d }, p0/z, [x2]
-; CHECK-NEXT: str z18, [sp, #2, mul vl]
+; CHECK-NEXT: str z18, [x8, #1, mul vl]
; CHECK-NEXT: str z17, [sp, #1, mul vl]
; CHECK-NEXT: str z16, [sp]
; CHECK-NEXT: bl callee3
@@ -150,18 +161,24 @@ entry:
define double @foo4(double %x0, ptr %ptr1, ptr %ptr2, ptr %ptr3, <vscale x 8 x double> %x1, <vscale x 8 x double> %x2, <vscale x 2 x double> %x3) nounwind {
; CHECK-LABEL: foo4:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr z6, [x3, #1, mul vl]
-; CHECK-NEXT: ldr z7, [x3]
-; CHECK-NEXT: ldr z24, [x3, #3, mul vl]
-; CHECK-NEXT: ldr z25, [x3, #2, mul vl]
-; CHECK-NEXT: str z4, [x0, #3, mul vl]
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: ldr z24, [x3, #1, mul vl]
+; CHECK-NEXT: ldr z25, [x3]
+; CHECK-NEXT: add x9, x3, x8
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: ldr z7, [x9, #1, mul vl]
+; CHECK-NEXT: ldr z6, [x8, #1, mul vl]
+; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: str z3, [x0, #2, mul vl]
+; CHECK-NEXT: add x9, x0, x8
+; CHECK-NEXT: add x8, x1, x8
; CHECK-NEXT: str z2, [x0, #1, mul vl]
; CHECK-NEXT: str z1, [x0]
-; CHECK-NEXT: str z25, [x1, #2, mul vl]
-; CHECK-NEXT: str z24, [x1, #3, mul vl]
-; CHECK-NEXT: str z7, [x1]
-; CHECK-NEXT: str z6, [x1, #1, mul vl]
+; CHECK-NEXT: str z4, [x9, #1, mul vl]
+; CHECK-NEXT: str z7, [x1, #2, mul vl]
+; CHECK-NEXT: str z25, [x1]
+; CHECK-NEXT: str z24, [x1, #1, mul vl]
+; CHECK-NEXT: str z6, [x8, #1, mul vl]
; CHECK-NEXT: str z5, [x2]
; CHECK-NEXT: ret
entry:
@@ -175,18 +192,24 @@ define double @foo5(i32 %i0, i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, ptr %p
; CHECK-LABEL: foo5:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr x8, [sp]
+; CHECK-NEXT: rdvl x9, #1
+; CHECK-NEXT: add x10, x8, x9
; CHECK-NEXT: ldr z5, [x8, #1, mul vl]
; CHECK-NEXT: ldr z6, [x8]
-; CHECK-NEXT: ldr z7, [x8, #3, mul vl]
-; CHECK-NEXT: ldr z24, [x8, #2, mul vl]
-; CHECK-NEXT: str z4, [x6, #3, mul vl]
+; CHECK-NEXT: add x9, x10, x9
+; CHECK-NEXT: ldr z7, [x10, #1, mul vl]
+; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: ldr z24, [x9, #1, mul vl]
+; CHECK-NEXT: add x9, x6, x8
+; CHECK-NEXT: add x8, x7, x8
; CHECK-NEXT: str z3, [x6, #2, mul vl]
; CHECK-NEXT: str z2, [x6, #1, mul vl]
; CHECK-NEXT: str z1, [x6]
-; CHECK-NEXT: str z24, [x7, #2, mul vl]
-; CHECK-NEXT: str z7, [x7, #3, mul vl]
+; CHECK-NEXT: str z4, [x9, #1, mul vl]
+; CHECK-NEXT: str z7, [x7, #2, mul vl]
; CHECK-NEXT: str z6, [x7]
; CHECK-NEXT: str z5, [x7, #1, mul vl]
+; CHECK-NEXT: str z24, [x8, #1, mul vl]
; CHECK-NEXT: ret
entry:
store volatile <vscale x 8 x double> %x1, ptr %ptr1
@@ -197,16 +220,21 @@ entry:
define double @foo6(double %x0, double %x1, ptr %ptr1, ptr %ptr2, <vscale x 8 x double> %x2, <vscale x 6 x double> %x3) nounwind {
; CHECK-LABEL: foo6:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr z1, [x2]
-; CHECK-NEXT: ldr z6, [x2, #2, mul vl]
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: ldr z6, [x2]
; CHECK-NEXT: ldr z7, [x2, #1, mul vl]
-; CHECK-NEXT: str z5, [x0, #3, mul vl]
+; CHECK-NEXT: add x9, x2, x8
+; CHECK-NEXT: add x8, x1, x8
+; CHECK-NEXT: ldr z1, [x9, #1, mul vl]
+; CHECK-NEXT: rdvl x9, #2
; CHECK-NEXT: str z4, [x0, #2, mul vl]
+; CHECK-NEXT: add x9, x0, x9
; CHECK-NEXT: str z3, [x0, #1, mul vl]
; CHECK-NEXT: str z2, [x0]
+; CHECK-NEXT: str z5, [x9, #1, mul vl]
; CHECK-NEXT: str z7, [x1, #1, mul vl]
-; CHECK-NEXT: str z6, [x1, #2, mul vl]
-; CHECK-NEXT: str z1, [x1]
+; CHECK-NEXT: str z1, [x8, #1, mul vl]
+; CHECK-NEXT: str z6, [x1]
; CHECK-NEXT: ret
entry:
store volatile <vscale x 8 x double> %x2, ptr %ptr1
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
index 3473b063e35ee..0e8cc3abe70de 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
@@ -10,7 +10,10 @@ define <4 x i32> @extract_v4i32_nxv16i32_12(<vscale x 16 x i32> %arg) {
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: str z3, [sp, #3, mul vl]
+; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
; CHECK-NEXT: str z2, [sp, #2, mul vl]
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
@@ -46,7 +49,10 @@ define <4 x i16> @extract_v4i16_nxv32i16_8(<vscale x 32 x i16> %arg) {
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: str z3, [sp, #3, mul vl]
+; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
; CHECK-NEXT: str z2, [sp, #2, mul vl]
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
@@ -67,19 +73,22 @@ define <2 x i16> @extract_v2i16_nxv32i16_8(<vscale x 32 x i16> %arg) {
; CHECK-NEXT: addvl sp, sp, #-8
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xc0, 0x00, 0x1e, 0x22 // sp + 16 + 64 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: mov x8, sp
-; CHECK-NEXT: str z3, [sp, #3, mul vl]
+; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: addvl x11, sp, #4
+; CHECK-NEXT: add x10, x9, x8
+; CHECK-NEXT: add x8, x11, x8
+; CHECK-NEXT: str z3, [x10, #1, mul vl]
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
+; CHECK-NEXT: add x8, x9, #32
; CHECK-NEXT: str z2, [sp, #2, mul vl]
-; CHECK-NEXT: add x8, x8, #32
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: str z3, [sp, #7, mul vl]
; CHECK-NEXT: str z2, [sp, #6, mul vl]
; CHECK-NEXT: str z1, [sp, #5, mul vl]
; CHECK-NEXT: str z0, [sp, #4, mul vl]
; CHECK-NEXT: ld1 { v0.h }[0], [x8]
-; CHECK-NEXT: addvl x8, sp, #4
-; CHECK-NEXT: add x8, x8, #34
+; CHECK-NEXT: add x8, x11, #34
; CHECK-NEXT: ld1 { v0.h }[2], [x8]
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: addvl sp, sp, #8
@@ -96,17 +105,19 @@ define <2 x i64> @extract_v2i64_nxv8i64_8(<vscale x 8 x i64> %arg) {
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: cnth x8
-; CHECK-NEXT: mov w9, #8 // =0x8
-; CHECK-NEXT: str z3, [sp, #3, mul vl]
-; CHECK-NEXT: sub x8, x8, #2
+; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: cnth x10
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: sub x10, x10, #2
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
+; CHECK-NEXT: mov w8, #8 // =0x8
+; CHECK-NEXT: cmp x10, #8
+; CHECK-NEXT: csel x8, x10, x8, lo
; CHECK-NEXT: str z2, [sp, #2, mul vl]
-; CHECK-NEXT: cmp x8, #8
+; CHECK-NEXT: lsl x8, x8, #3
; CHECK-NEXT: str z1, [sp, #1, mul vl]
-; CHECK-NEXT: csel x8, x8, x9, lo
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: lsl x8, x8, #3
; CHECK-NEXT: ldr q0, [x9, x8]
; CHECK-NEXT: addvl sp, sp, #4
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -122,7 +133,10 @@ define <4 x float> @extract_v4f32_nxv16f32_12(<vscale x 16 x float> %arg) {
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: str z3, [sp, #3, mul vl]
+; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
; CHECK-NEXT: str z2, [sp, #2, mul vl]
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
@@ -165,13 +179,16 @@ define <4 x i1> @extract_v4i1_nxv32i1_16(<vscale x 32 x i1> %arg) {
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: punpkhi p2.h, p1.b
+; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: punpklo p1.h, p1.b
+; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: mov z0.h, p2/z, #1 // =0x1
; CHECK-NEXT: punpkhi p2.h, p0.b
; CHECK-NEXT: punpklo p0.h, p0.b
; CHECK-NEXT: mov z1.h, p1/z, #1 // =0x1
; CHECK-NEXT: mov z2.h, p2/z, #1 // =0x1
-; CHECK-NEXT: str z0, [sp, #3, mul vl]
+; CHECK-NEXT: str z0, [x8, #1, mul vl]
; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1
; CHECK-NEXT: str z1, [sp, #2, mul vl]
; CHECK-NEXT: str z2, [sp, #1, mul vl]
diff --git a/llvm/test/CodeGen/AArch64/sve-fptrunc-store.ll b/llvm/test/CodeGen/AArch64/sve-fptrunc-store.ll
index 573958771658c..8b4eae8789eac 100644
--- a/llvm/test/CodeGen/AArch64/sve-fptrunc-store.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fptrunc-store.ll
@@ -64,18 +64,20 @@ entry:
define void @fptrunc8_f64_f16(ptr %dst, ptr %src) {
; CHECK-LABEL: fptrunc8_f64_f16:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr z0, [x1, #3, mul vl]
-; CHECK-NEXT: ldr z1, [x1]
+; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: ldr z0, [x1, #1, mul vl]
+; CHECK-NEXT: ldr z2, [x1, #2, mul vl]
+; CHECK-NEXT: add x8, x1, x8
+; CHECK-NEXT: ldr z3, [x1]
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ldr z2, [x1, #1, mul vl]
-; CHECK-NEXT: ldr z3, [x1, #2, mul vl]
+; CHECK-NEXT: ldr z1, [x8, #1, mul vl]
; CHECK-NEXT: fcvt z0.h, p0/m, z0.d
-; CHECK-NEXT: fcvt z1.h, p0/m, z1.d
-; CHECK-NEXT: fcvt z3.h, p0/m, z3.d
; CHECK-NEXT: fcvt z2.h, p0/m, z2.d
+; CHECK-NEXT: fcvt z3.h, p0/m, z3.d
+; CHECK-NEXT: fcvt z1.h, p0/m, z1.d
; CHECK-NEXT: uzp1 z0.s, z3.s, z0.s
-; CHECK-NEXT: uzp1 z1.s, z1.s, z2.s
-; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
+; CHECK-NEXT: uzp1 z1.s, z2.s, z1.s
+; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: str z0, [x0]
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll b/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
index 66068e86ce8a7..06fc5c395a8d7 100644
--- a/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
+++ b/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
@@ -79,18 +79,20 @@ define <vscale x 16 x i8> @narrow_i64_gather_index_i8_zext(ptr %out, ptr %in, <v
; CHECK-LABEL: narrow_i64_gather_index_i8_zext:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: add x8, x1, x2
-; CHECK-NEXT: ld1b { z0.s }, p0/z, [x8, #3, mul vl]
-; CHECK-NEXT: ld1b { z1.s }, p0/z, [x8, #2, mul vl]
-; CHECK-NEXT: ld1b { z2.s }, p0/z, [x1, x2]
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: add x9, x1, x2
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: ld1b { z0.s }, p0/z, [x1, x2]
+; CHECK-NEXT: ld1b { z1.s }, p0/z, [x9, #1, mul vl]
+; CHECK-NEXT: ld1b { z2.s }, p0/z, [x9, #2, mul vl]
; CHECK-NEXT: ld1b { z3.s }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ld1b { z0.s }, p0/z, [x1, z0.s, uxtw]
; CHECK-NEXT: ld1b { z1.s }, p0/z, [x1, z1.s, uxtw]
; CHECK-NEXT: ld1b { z2.s }, p0/z, [x1, z2.s, uxtw]
; CHECK-NEXT: ld1b { z3.s }, p0/z, [x1, z3.s, uxtw]
-; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: uzp1 z1.h, z2.h, z3.h
-; CHECK-NEXT: uzp1 z0.b, z1.b, z0.b
+; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
; CHECK-NEXT: ret
%1 = getelementptr inbounds i8, ptr %in, i64 %ptr
%2 = bitcast ptr %1 to ptr
@@ -105,18 +107,20 @@ define <vscale x 16 x i8> @narrow_i64_gather_index_i8_sext(ptr %out, ptr %in, <v
; CHECK-LABEL: narrow_i64_gather_index_i8_sext:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: add x8, x1, x2
-; CHECK-NEXT: ld1sb { z0.s }, p0/z, [x8, #3, mul vl]
-; CHECK-NEXT: ld1sb { z1.s }, p0/z, [x8, #2, mul vl]
-; CHECK-NEXT: ld1sb { z2.s }, p0/z, [x1, x2]
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: add x9, x1, x2
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: ld1sb { z0.s }, p0/z, [x1, x2]
+; CHECK-NEXT: ld1sb { z1.s }, p0/z, [x9, #1, mul vl]
+; CHECK-NEXT: ld1sb { z2.s }, p0/z, [x9, #2, mul vl]
; CHECK-NEXT: ld1sb { z3.s }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ld1b { z0.s }, p0/z, [x1, z0.s, sxtw]
; CHECK-NEXT: ld1b { z1.s }, p0/z, [x1, z1.s, sxtw]
; CHECK-NEXT: ld1b { z2.s }, p0/z, [x1, z2.s, sxtw]
; CHECK-NEXT: ld1b { z3.s }, p0/z, [x1, z3.s, sxtw]
-; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: uzp1 z1.h, z2.h, z3.h
-; CHECK-NEXT: uzp1 z0.b, z1.b, z0.b
+; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
; CHECK-NEXT: ret
%1 = getelementptr inbounds i8, ptr %in, i64 %ptr
%2 = bitcast ptr %1 to ptr
diff --git a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll
index 041a065687b05..09ee82b4fce2d 100644
--- a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll
@@ -138,14 +138,19 @@ define <vscale x 16 x i8> @insert_v16i8_nxv16i8_idx16(<vscale x 16 x i8> %vec, <
define void @insert_nxv8i64_nxv16i64(<vscale x 8 x i64> %sv0, <vscale x 8 x i64> %sv1, ptr %out) {
; CHECK-LABEL: insert_nxv8i64_nxv16i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: str z7, [x0, #7, mul vl]
-; CHECK-NEXT: str z6, [x0, #6, mul vl]
-; CHECK-NEXT: str z5, [x0, #5, mul vl]
+; CHECK-NEXT: rdvl x8, #4
+; CHECK-NEXT: rdvl x9, #2
; CHECK-NEXT: str z4, [x0, #4, mul vl]
-; CHECK-NEXT: str z3, [x0, #3, mul vl]
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: str z2, [x0, #2, mul vl]
+; CHECK-NEXT: add x10, x0, x9
; CHECK-NEXT: str z1, [x0, #1, mul vl]
; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: str z6, [x8, #2, mul vl]
+; CHECK-NEXT: str z5, [x8, #1, mul vl]
+; CHECK-NEXT: add x8, x8, x9
+; CHECK-NEXT: str z3, [x10, #1, mul vl]
+; CHECK-NEXT: str z7, [x8, #1, mul vl]
; CHECK-NEXT: ret
%v0 = call <vscale x 16 x i64> @llvm.vector.insert.nxv8i64.nxv16i64(<vscale x 16 x i64> poison, <vscale x 8 x i64> %sv0, i64 0)
%v = call <vscale x 16 x i64> @llvm.vector.insert.nxv8i64.nxv16i64(<vscale x 16 x i64> %v0, <vscale x 8 x i64> %sv1, i64 8)
@@ -156,10 +161,12 @@ define void @insert_nxv8i64_nxv16i64(<vscale x 8 x i64> %sv0, <vscale x 8 x i64>
define void @insert_nxv8i64_nxv16i64_lo(<vscale x 8 x i64> %sv0, ptr %out) {
; CHECK-LABEL: insert_nxv8i64_nxv16i64_lo:
; CHECK: // %bb.0:
-; CHECK-NEXT: str z3, [x0, #3, mul vl]
+; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: str z2, [x0, #2, mul vl]
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: str z1, [x0, #1, mul vl]
; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
; CHECK-NEXT: ret
%v = call <vscale x 16 x i64> @llvm.vector.insert.nxv8i64.nxv16i64(<vscale x 16 x i64> poison, <vscale x 8 x i64> %sv0, i64 0)
store <vscale x 16 x i64> %v, ptr %out
@@ -169,10 +176,14 @@ define void @insert_nxv8i64_nxv16i64_lo(<vscale x 8 x i64> %sv0, ptr %out) {
define void @insert_nxv8i64_nxv16i64_hi(<vscale x 8 x i64> %sv0, ptr %out) {
; CHECK-LABEL: insert_nxv8i64_nxv16i64_hi:
; CHECK: // %bb.0:
-; CHECK-NEXT: str z3, [x0, #7, mul vl]
-; CHECK-NEXT: str z2, [x0, #6, mul vl]
-; CHECK-NEXT: str z1, [x0, #5, mul vl]
+; CHECK-NEXT: rdvl x8, #4
+; CHECK-NEXT: rdvl x9, #2
; CHECK-NEXT: str z0, [x0, #4, mul vl]
+; CHECK-NEXT: add x8, x0, x8
+; CHECK-NEXT: str z2, [x8, #2, mul vl]
+; CHECK-NEXT: str z1, [x8, #1, mul vl]
+; CHECK-NEXT: add x8, x8, x9
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
; CHECK-NEXT: ret
%v = call <vscale x 16 x i64> @llvm.vector.insert.nxv8i64.nxv16i64(<vscale x 16 x i64> poison, <vscale x 8 x i64> %sv0, i64 8)
store <vscale x 16 x i64> %v, ptr %out
@@ -187,14 +198,18 @@ define void @insert_v2i64_nxv16i64(<2 x i64> %sv0, <2 x i64> %sv1, ptr %out) uwt
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
+; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT: str z0, [sp]
; CHECK-NEXT: str q1, [sp, #32]
-; CHECK-NEXT: ldr z0, [sp, #3, mul vl]
+; CHECK-NEXT: add x9, x9, x8
+; CHECK-NEXT: add x8, x0, x8
+; CHECK-NEXT: ldr z0, [x9, #1, mul vl]
; CHECK-NEXT: ldr z1, [sp, #2, mul vl]
; CHECK-NEXT: ldr z2, [sp, #1, mul vl]
; CHECK-NEXT: ldr z3, [sp]
-; CHECK-NEXT: str z0, [x0, #3, mul vl]
+; CHECK-NEXT: str z0, [x8, #1, mul vl]
; CHECK-NEXT: str z1, [x0, #2, mul vl]
; CHECK-NEXT: str z2, [x0, #1, mul vl]
; CHECK-NEXT: str z3, [x0]
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
index 9698f1a6768fd..5103bbb7258a5 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
@@ -9,10 +9,12 @@ define <vscale x 16 x i32> @ld1b_i8_sext_i32(ptr %base) {
; CHECK-LABEL: ld1b_i8_sext_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1sb { z0.s }, p0/z, [x0]
; CHECK-NEXT: ld1sb { z1.s }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z2.s }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1sb { z3.s }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1sb { z3.s }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%wide.load = load <vscale x 16 x i8>, ptr %base
%res = sext <vscale x 16 x i8> %wide.load to <vscale x 16 x i32>
@@ -23,10 +25,12 @@ define <vscale x 16 x i32> @ld1b_i8_zext_i32(ptr %base) {
; CHECK-LABEL: ld1b_i8_zext_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0]
; CHECK-NEXT: ld1b { z1.s }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z2.s }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1b { z3.s }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1b { z3.s }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%wide.load = load <vscale x 16 x i8>, ptr %base
%res = zext <vscale x 16 x i8> %wide.load to <vscale x 16 x i32>
@@ -36,15 +40,20 @@ define <vscale x 16 x i32> @ld1b_i8_zext_i32(ptr %base) {
define <vscale x 16 x i64> @ld1b_i8_sext(ptr %base) {
; CHECK-LABEL: ld1b_i8_sext:
; CHECK: // %bb.0:
+; CHECK-NEXT: cnth x9
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cntw x8
+; CHECK-NEXT: add x9, x0, x9
+; CHECK-NEXT: add x10, x0, x8
+; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1sb { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ld1sb { z4.d }, p0/z, [x0, #4, mul vl]
-; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x0, #5, mul vl]
-; CHECK-NEXT: ld1sb { z6.d }, p0/z, [x0, #6, mul vl]
-; CHECK-NEXT: ld1sb { z7.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x10, #1, mul vl]
+; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x9, #1, mul vl]
+; CHECK-NEXT: ld1sb { z6.d }, p0/z, [x9, #2, mul vl]
+; CHECK-NEXT: ld1sb { z7.d }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%wide.load = load <vscale x 16 x i8>, ptr %base
%res = sext <vscale x 16 x i8> %wide.load to <vscale x 16 x i64>
@@ -54,15 +63,20 @@ define <vscale x 16 x i64> @ld1b_i8_sext(ptr %base) {
define <vscale x 16 x i64> @ld1b_i8_zext(ptr %base) {
; CHECK-LABEL: ld1b_i8_zext:
; CHECK: // %bb.0:
+; CHECK-NEXT: cnth x9
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cntw x8
+; CHECK-NEXT: add x9, x0, x9
+; CHECK-NEXT: add x10, x0, x8
+; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1b { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1b { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ld1b { z4.d }, p0/z, [x0, #4, mul vl]
-; CHECK-NEXT: ld1b { z5.d }, p0/z, [x0, #5, mul vl]
-; CHECK-NEXT: ld1b { z6.d }, p0/z, [x0, #6, mul vl]
-; CHECK-NEXT: ld1b { z7.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: ld1b { z3.d }, p0/z, [x10, #1, mul vl]
+; CHECK-NEXT: ld1b { z5.d }, p0/z, [x9, #1, mul vl]
+; CHECK-NEXT: ld1b { z6.d }, p0/z, [x9, #2, mul vl]
+; CHECK-NEXT: ld1b { z7.d }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%wide.load = load <vscale x 16 x i8>, ptr %base
%res = zext <vscale x 16 x i8> %wide.load to <vscale x 16 x i64>
@@ -77,10 +91,12 @@ define <vscale x 8 x i64> @ld1h_i16_sext(ptr %base) {
; CHECK-LABEL: ld1h_i16_sext:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1sh { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1sh { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sh { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1sh { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1sh { z3.d }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%wide.load = load <vscale x 8 x i16>, ptr %base
%res = sext <vscale x 8 x i16> %wide.load to <vscale x 8 x i64>
@@ -91,10 +107,12 @@ define <vscale x 8 x i64> @ld1h_i16_zext(ptr %base) {
; CHECK-LABEL: ld1h_i16_zext:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1h { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1h { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1h { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1h { z3.d }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%wide.load = load <vscale x 8 x i16>, ptr %base
%res = zext <vscale x 8 x i16> %wide.load to <vscale x 8 x i64>
@@ -172,10 +190,12 @@ define <vscale x 8 x i64> @zload_8i8_8i64(ptr %a) {
; CHECK-LABEL: zload_8i8_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cntw x8
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1b { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1b { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1b { z3.d }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%aval = load <vscale x 8 x i8>, ptr %a
%aext = zext <vscale x 8 x i8> %aval to <vscale x 8 x i64>
@@ -222,10 +242,12 @@ define <vscale x 8 x i64> @sload_8i8_8i64(ptr %a) {
; CHECK-LABEL: sload_8i8_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cntw x8
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1sb { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%aval = load <vscale x 8 x i8>, ptr %a
%aext = sext <vscale x 8 x i8> %aval to <vscale x 8 x i64>
diff --git a/llvm/test/CodeGen/AArch64/sve-ldst-sext.ll b/llvm/test/CodeGen/AArch64/sve-ldst-sext.ll
index 78abf3239343c..e05d37dcec07c 100644
--- a/llvm/test/CodeGen/AArch64/sve-ldst-sext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-ldst-sext.ll
@@ -72,10 +72,12 @@ define <vscale x 8 x i64> @sload_nxv8i16(ptr %a) {
; CHECK-LABEL: sload_nxv8i16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1sh { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1sh { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sh { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1sh { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1sh { z3.d }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%load = load <vscale x 8 x i16>, ptr %a, align 2
%ext = sext <vscale x 8 x i16> %load to <vscale x 8 x i64>
@@ -150,10 +152,12 @@ define <vscale x 8 x i64> @sload_8i8_8i64(ptr %a) {
; CHECK-LABEL: sload_8i8_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cntw x8
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1sb { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%aval = load <vscale x 8 x i8>, ptr %a, align 1
%aext = sext <vscale x 8 x i8> %aval to <vscale x 8 x i64>
@@ -220,13 +224,16 @@ define <vscale x 8 x i32> @sload_x2_8i8_8i32(ptr %a, ptr %b) {
define <vscale x 8 x i64> @sload_x2_8i8_8i64(ptr %a, ptr %b) {
; CHECK-LABEL: sload_x2_8i8_8i64:
; CHECK: // %bb.0:
+; CHECK-NEXT: cntw x8
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: add x9, x0, x8
+; CHECK-NEXT: add x8, x1, x8
+; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x9, #1, mul vl]
; CHECK-NEXT: ld1sb { z2.d }, p0/z, [x0, #2, mul vl]
; CHECK-NEXT: ld1sb { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1sb { z4.d }, p0/z, [x1]
-; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x1, #3, mul vl]
+; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ld1sb { z6.d }, p0/z, [x1, #2, mul vl]
; CHECK-NEXT: ld1sb { z7.d }, p0/z, [x1, #1, mul vl]
; CHECK-NEXT: add z0.d, z0.d, z4.d
@@ -377,21 +384,31 @@ define <vscale x 16 x i64> @load_frozen_before_zext_multiuse5_dst_illegal(ptr %s
; CHECK-LABEL: load_frozen_before_zext_multiuse5_dst_illegal:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cntw x8
; CHECK-NEXT: mov z24.d, #3 // =0x3
+; CHECK-NEXT: add x9, x0, x8
+; CHECK-NEXT: rdvl x10, #4
+; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x9, #1, mul vl]
+; CHECK-NEXT: cnth x9
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]
+; CHECK-NEXT: add x9, x0, x9
; CHECK-NEXT: ld1sb { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: ld1sb { z4.d }, p0/z, [x0, #4, mul vl]
-; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x0, #5, mul vl]
-; CHECK-NEXT: ld1sb { z6.d }, p0/z, [x0, #6, mul vl]
-; CHECK-NEXT: ld1sb { z7.d }, p0/z, [x0, #7, mul vl]
-; CHECK-NEXT: str z24, [x1, #6, mul vl]
-; CHECK-NEXT: str z24, [x1, #7, mul vl]
+; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x9, #1, mul vl]
+; CHECK-NEXT: ld1sb { z6.d }, p0/z, [x9, #2, mul vl]
+; CHECK-NEXT: add x9, x1, x10
+; CHECK-NEXT: rdvl x10, #2
+; CHECK-NEXT: ld1sb { z7.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: add x8, x9, x10
+; CHECK-NEXT: str z24, [x9, #2, mul vl]
+; CHECK-NEXT: str z24, [x8, #1, mul vl]
+; CHECK-NEXT: add x8, x1, x10
; CHECK-NEXT: str z24, [x1, #4, mul vl]
-; CHECK-NEXT: str z24, [x1, #5, mul vl]
+; CHECK-NEXT: str z24, [x9, #1, mul vl]
; CHECK-NEXT: str z24, [x1, #2, mul vl]
-; CHECK-NEXT: str z24, [x1, #3, mul vl]
+; CHECK-NEXT: str z24, [x8, #1, mul vl]
; CHECK-NEXT: str z24, [x1]
; CHECK-NEXT: str z24, [x1, #1, mul vl]
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-ldst-zext.ll b/llvm/test/CodeGen/AArch64/sve-ldst-zext.ll
index 0a1cf0cacbf2f..f4d3677effc88 100644
--- a/llvm/test/CodeGen/AArch64/sve-ldst-zext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-ldst-zext.ll
@@ -72,10 +72,12 @@ define <vscale x 8 x i64> @zload_nxv8i16(ptr %a) {
; CHECK-LABEL: zload_nxv8i16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1h { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1h { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1h { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1h { z3.d }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%load = load <vscale x 8 x i16>, ptr %a, align 2
%ext = zext <vscale x 8 x i16> %load to <vscale x 8 x i64>
@@ -150,10 +152,12 @@ define <vscale x 8 x i64> @zload_8i8_8i64(ptr %a) {
; CHECK-LABEL: zload_8i8_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cntw x8
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1b { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1b { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1b { z3.d }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%aval = load <vscale x 8 x i8>, ptr %a, align 1
%aext = zext <vscale x 8 x i8> %aval to <vscale x 8 x i64>
@@ -220,13 +224,16 @@ define <vscale x 8 x i32> @zload_x2_8i8_8i32(ptr %a, ptr %b) {
define <vscale x 8 x i64> @zload_x2_8i8_8i64(ptr %a, ptr %b) {
; CHECK-LABEL: zload_x2_8i8_8i64:
; CHECK: // %bb.0:
+; CHECK-NEXT: cntw x8
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ld1b { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: add x9, x0, x8
+; CHECK-NEXT: add x8, x1, x8
+; CHECK-NEXT: ld1b { z3.d }, p0/z, [x9, #1, mul vl]
; CHECK-NEXT: ld1b { z2.d }, p0/z, [x0, #2, mul vl]
; CHECK-NEXT: ld1b { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1b { z4.d }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z5.d }, p0/z, [x1, #3, mul vl]
+; CHECK-NEXT: ld1b { z5.d }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ld1b { z6.d }, p0/z, [x1, #2, mul vl]
; CHECK-NEXT: ld1b { z7.d }, p0/z, [x1, #1, mul vl]
; CHECK-NEXT: add z0.d, z0.d, z4.d
@@ -368,21 +375,31 @@ define <vscale x 16 x i64> @load_frozen_before_zext_multiuse5_dst_illegal(ptr %s
; CHECK-LABEL: load_frozen_before_zext_multiuse5_dst_illegal:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cntw x8
; CHECK-NEXT: mov z24.d, #3 // =0x3
+; CHECK-NEXT: add x9, x0, x8
+; CHECK-NEXT: rdvl x10, #4
+; CHECK-NEXT: ld1b { z3.d }, p0/z, [x9, #1, mul vl]
+; CHECK-NEXT: cnth x9
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
+; CHECK-NEXT: add x9, x0, x9
; CHECK-NEXT: ld1b { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1b { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: ld1b { z4.d }, p0/z, [x0, #4, mul vl]
-; CHECK-NEXT: ld1b { z5.d }, p0/z, [x0, #5, mul vl]
-; CHECK-NEXT: ld1b { z6.d }, p0/z, [x0, #6, mul vl]
-; CHECK-NEXT: ld1b { z7.d }, p0/z, [x0, #7, mul vl]
-; CHECK-NEXT: str z24, [x1, #6, mul vl]
-; CHECK-NEXT: str z24, [x1, #7, mul vl]
+; CHECK-NEXT: ld1b { z5.d }, p0/z, [x9, #1, mul vl]
+; CHECK-NEXT: ld1b { z6.d }, p0/z, [x9, #2, mul vl]
+; CHECK-NEXT: add x9, x1, x10
+; CHECK-NEXT: rdvl x10, #2
+; CHECK-NEXT: ld1b { z7.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: add x8, x9, x10
+; CHECK-NEXT: str z24, [x9, #2, mul vl]
+; CHECK-NEXT: str z24, [x8, #1, mul vl]
+; CHECK-NEXT: add x8, x1, x10
; CHECK-NEXT: str z24, [x1, #4, mul vl]
-; CHECK-NEXT: str z24, [x1, #5, mul vl]
+; CHECK-NEXT: str z24, [x9, #1, mul vl]
; CHECK-NEXT: str z24, [x1, #2, mul vl]
-; CHECK-NEXT: str z24, [x1, #3, mul vl]
+; CHECK-NEXT: str z24, [x8, #1, mul vl]
; CHECK-NEXT: str z24, [x1]
; CHECK-NEXT: str z24, [x1, #1, mul vl]
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-llrint.ll b/llvm/test/CodeGen/AArch64/sve-llrint.ll
index 8ecf378d5623d..9e9ef518f5029 100644
--- a/llvm/test/CodeGen/AArch64/sve-llrint.ll
+++ b/llvm/test/CodeGen/AArch64/sve-llrint.ll
@@ -126,77 +126,87 @@ define <vscale x 32 x i64> @llrint_v32i64_v32f16(<vscale x 32 x half> %x) {
; CHECK: // %bb.0:
; CHECK-NEXT: uunpkhi z4.s, z3.h
; CHECK-NEXT: uunpklo z3.s, z3.h
+; CHECK-NEXT: rdvl x10, #8
+; CHECK-NEXT: uunpkhi z5.s, z2.h
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: uunpkhi z7.s, z2.h
+; CHECK-NEXT: rdvl x11, #4
; CHECK-NEXT: uunpklo z2.s, z2.h
-; CHECK-NEXT: uunpkhi z24.s, z0.h
-; CHECK-NEXT: uunpkhi z25.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpkhi z5.d, z4.s
-; CHECK-NEXT: uunpklo z4.d, z4.s
-; CHECK-NEXT: uunpkhi z6.d, z3.s
+; CHECK-NEXT: uunpklo z27.s, z1.h
+; CHECK-NEXT: rdvl x9, #2
+; CHECK-NEXT: uunpkhi z1.s, z1.h
+; CHECK-NEXT: uunpklo z25.s, z0.h
+; CHECK-NEXT: add x10, x8, x10
+; CHECK-NEXT: uunpklo z6.d, z4.s
+; CHECK-NEXT: uunpkhi z7.d, z3.s
+; CHECK-NEXT: add x12, x10, x11
; CHECK-NEXT: uunpklo z3.d, z3.s
-; CHECK-NEXT: uunpkhi z26.d, z7.s
-; CHECK-NEXT: uunpklo z7.d, z7.s
-; CHECK-NEXT: uunpkhi z27.d, z2.s
-; CHECK-NEXT: uunpkhi z28.d, z24.s
+; CHECK-NEXT: uunpkhi z24.d, z5.s
+; CHECK-NEXT: add x13, x10, x9
+; CHECK-NEXT: uunpklo z5.d, z5.s
+; CHECK-NEXT: uunpkhi z0.s, z0.h
+; CHECK-NEXT: uunpkhi z26.d, z2.s
; CHECK-NEXT: uunpklo z2.d, z2.s
-; CHECK-NEXT: uunpkhi z29.d, z25.s
-; CHECK-NEXT: uunpklo z25.d, z25.s
-; CHECK-NEXT: frintx z5.h, p0/m, z5.h
-; CHECK-NEXT: frintx z4.h, p0/m, z4.h
+; CHECK-NEXT: uunpkhi z29.d, z1.s
+; CHECK-NEXT: uunpklo z1.d, z1.s
; CHECK-NEXT: frintx z6.h, p0/m, z6.h
+; CHECK-NEXT: frintx z7.h, p0/m, z7.h
+; CHECK-NEXT: uunpkhi z30.d, z27.s
; CHECK-NEXT: frintx z3.h, p0/m, z3.h
+; CHECK-NEXT: frintx z24.h, p0/m, z24.h
+; CHECK-NEXT: uunpkhi z4.d, z4.s
+; CHECK-NEXT: frintx z5.h, p0/m, z5.h
+; CHECK-NEXT: uunpklo z28.d, z0.s
+; CHECK-NEXT: uunpkhi z0.d, z0.s
; CHECK-NEXT: frintx z26.h, p0/m, z26.h
-; CHECK-NEXT: frintx z7.h, p0/m, z7.h
-; CHECK-NEXT: frintx z27.h, p0/m, z27.h
; CHECK-NEXT: frintx z2.h, p0/m, z2.h
-; CHECK-NEXT: frintx z28.h, p0/m, z28.h
; CHECK-NEXT: frintx z29.h, p0/m, z29.h
-; CHECK-NEXT: frintx z25.h, p0/m, z25.h
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
-; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.h
; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.h
+; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.h
+; CHECK-NEXT: frintx z1.h, p0/m, z1.h
; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.h
+; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.h
+; CHECK-NEXT: frintx z30.h, p0/m, z30.h
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
+; CHECK-NEXT: frintx z0.h, p0/m, z0.h
+; CHECK-NEXT: frintx z28.h, p0/m, z28.h
; CHECK-NEXT: fcvtzs z26.d, p0/m, z26.h
-; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.h
-; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.h
+; CHECK-NEXT: frintx z4.h, p0/m, z4.h
; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.h
-; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.h
+; CHECK-NEXT: str z6, [x12, #2, mul vl]
+; CHECK-NEXT: uunpklo z6.d, z27.s
; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.h
-; CHECK-NEXT: fcvtzs z25.d, p0/m, z25.h
-; CHECK-NEXT: str z5, [x8, #15, mul vl]
-; CHECK-NEXT: uunpkhi z5.d, z1.s
-; CHECK-NEXT: uunpklo z1.d, z1.s
-; CHECK-NEXT: str z4, [x8, #14, mul vl]
-; CHECK-NEXT: uunpkhi z4.d, z0.s
-; CHECK-NEXT: uunpklo z0.d, z0.s
-; CHECK-NEXT: str z3, [x8, #12, mul vl]
-; CHECK-NEXT: uunpklo z3.d, z24.s
-; CHECK-NEXT: str z6, [x8, #13, mul vl]
-; CHECK-NEXT: str z26, [x8, #11, mul vl]
-; CHECK-NEXT: frintx z5.h, p0/m, z5.h
-; CHECK-NEXT: frintx z1.h, p0/m, z1.h
-; CHECK-NEXT: str z7, [x8, #10, mul vl]
-; CHECK-NEXT: frintx z4.h, p0/m, z4.h
-; CHECK-NEXT: frintx z0.h, p0/m, z0.h
-; CHECK-NEXT: str z27, [x8, #9, mul vl]
-; CHECK-NEXT: frintx z3.h, p0/m, z3.h
-; CHECK-NEXT: str z2, [x8, #8, mul vl]
-; CHECK-NEXT: str z29, [x8, #7, mul vl]
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
+; CHECK-NEXT: str z7, [x12, #1, mul vl]
; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.h
-; CHECK-NEXT: str z25, [x8, #6, mul vl]
-; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.h
+; CHECK-NEXT: fcvtzs z30.d, p0/m, z30.h
+; CHECK-NEXT: str z3, [x10, #4, mul vl]
+; CHECK-NEXT: uunpkhi z3.d, z25.s
; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.h
-; CHECK-NEXT: str z28, [x8, #3, mul vl]
+; CHECK-NEXT: str z24, [x13, #1, mul vl]
+; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.h
+; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.h
+; CHECK-NEXT: str z5, [x10, #2, mul vl]
+; CHECK-NEXT: uunpklo z5.d, z25.s
+; CHECK-NEXT: frintx z6.h, p0/m, z6.h
+; CHECK-NEXT: str z26, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x8, x11
+; CHECK-NEXT: add x11, x10, x9
+; CHECK-NEXT: str z2, [x8, #8, mul vl]
+; CHECK-NEXT: frintx z3.h, p0/m, z3.h
+; CHECK-NEXT: str z29, [x11, #1, mul vl]
+; CHECK-NEXT: str z1, [x10, #2, mul vl]
+; CHECK-NEXT: frintx z5.h, p0/m, z5.h
+; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.h
+; CHECK-NEXT: str z30, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x8, x9
; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.h
-; CHECK-NEXT: str z5, [x8, #5, mul vl]
-; CHECK-NEXT: str z1, [x8, #4, mul vl]
-; CHECK-NEXT: str z3, [x8, #2, mul vl]
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
+; CHECK-NEXT: str z6, [x8, #4, mul vl]
+; CHECK-NEXT: str z0, [x10, #1, mul vl]
+; CHECK-NEXT: str z28, [x8, #2, mul vl]
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
+; CHECK-NEXT: str z5, [x8]
+; CHECK-NEXT: add x8, x12, x9
; CHECK-NEXT: str z4, [x8, #1, mul vl]
-; CHECK-NEXT: str z0, [x8]
; CHECK-NEXT: ret
%a = call <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv32f16(<vscale x 32 x half> %x)
ret <vscale x 32 x i64> %a
@@ -323,71 +333,81 @@ declare <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f32(<vscale x 16 x float>
define <vscale x 32 x i64> @llrint_v32i64_v32f32(<vscale x 32 x float> %x) {
; CHECK-LABEL: llrint_v32i64_v32f32:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z24.d, z7.s
-; CHECK-NEXT: uunpkhi z25.d, z6.s
-; CHECK-NEXT: uunpklo z6.d, z6.s
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: uunpklo z7.d, z7.s
-; CHECK-NEXT: uunpkhi z26.d, z5.s
-; CHECK-NEXT: uunpklo z5.d, z5.s
-; CHECK-NEXT: uunpkhi z27.d, z4.s
-; CHECK-NEXT: uunpkhi z28.d, z1.s
+; CHECK-NEXT: uunpklo z24.d, z6.s
+; CHECK-NEXT: uunpklo z25.d, z5.s
+; CHECK-NEXT: rdvl x9, #8
+; CHECK-NEXT: uunpkhi z26.d, z4.s
; CHECK-NEXT: uunpklo z4.d, z4.s
-; CHECK-NEXT: uunpkhi z29.d, z3.s
-; CHECK-NEXT: uunpklo z3.d, z3.s
+; CHECK-NEXT: add x9, x8, x9
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: uunpklo z28.d, z3.s
+; CHECK-NEXT: rdvl x10, #4
+; CHECK-NEXT: uunpkhi z29.d, z2.s
+; CHECK-NEXT: uunpklo z30.d, z1.s
+; CHECK-NEXT: rdvl x12, #2
; CHECK-NEXT: frintx z24.s, p0/m, z24.s
-; CHECK-NEXT: frintx z6.s, p0/m, z6.s
-; CHECK-NEXT: uunpklo z1.d, z1.s
-; CHECK-NEXT: frintx z7.s, p0/m, z7.s
; CHECK-NEXT: frintx z25.s, p0/m, z25.s
+; CHECK-NEXT: add x11, x8, x10
; CHECK-NEXT: frintx z26.s, p0/m, z26.s
-; CHECK-NEXT: frintx z5.s, p0/m, z5.s
-; CHECK-NEXT: frintx z27.s, p0/m, z27.s
-; CHECK-NEXT: frintx z28.s, p0/m, z28.s
; CHECK-NEXT: frintx z4.s, p0/m, z4.s
-; CHECK-NEXT: frintx z29.s, p0/m, z29.s
-; CHECK-NEXT: frintx z3.s, p0/m, z3.s
+; CHECK-NEXT: add x13, x8, x12
+; CHECK-NEXT: uunpklo z2.d, z2.s
+; CHECK-NEXT: uunpkhi z1.d, z1.s
+; CHECK-NEXT: uunpklo z31.d, z0.s
+; CHECK-NEXT: uunpkhi z0.d, z0.s
+; CHECK-NEXT: uunpkhi z27.d, z7.s
+; CHECK-NEXT: uunpkhi z6.d, z6.s
; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.s
-; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.s
-; CHECK-NEXT: frintx z1.s, p0/m, z1.s
-; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.s
; CHECK-NEXT: fcvtzs z25.d, p0/m, z25.s
+; CHECK-NEXT: uunpklo z7.d, z7.s
; CHECK-NEXT: fcvtzs z26.d, p0/m, z26.s
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.s
-; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.s
-; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.s
; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.s
-; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.s
-; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.s
-; CHECK-NEXT: str z24, [x8, #15, mul vl]
-; CHECK-NEXT: uunpkhi z24.d, z2.s
-; CHECK-NEXT: uunpklo z2.d, z2.s
-; CHECK-NEXT: str z6, [x8, #12, mul vl]
-; CHECK-NEXT: uunpkhi z6.d, z0.s
-; CHECK-NEXT: uunpklo z0.d, z0.s
-; CHECK-NEXT: str z7, [x8, #14, mul vl]
-; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.s
-; CHECK-NEXT: str z25, [x8, #13, mul vl]
-; CHECK-NEXT: str z26, [x8, #11, mul vl]
-; CHECK-NEXT: frintx z24.s, p0/m, z24.s
+; CHECK-NEXT: uunpkhi z3.d, z3.s
+; CHECK-NEXT: frintx z28.s, p0/m, z28.s
+; CHECK-NEXT: frintx z29.s, p0/m, z29.s
; CHECK-NEXT: frintx z2.s, p0/m, z2.s
-; CHECK-NEXT: str z5, [x8, #10, mul vl]
-; CHECK-NEXT: frintx z6.s, p0/m, z6.s
+; CHECK-NEXT: frintx z1.s, p0/m, z1.s
+; CHECK-NEXT: frintx z30.s, p0/m, z30.s
; CHECK-NEXT: frintx z0.s, p0/m, z0.s
-; CHECK-NEXT: str z27, [x8, #9, mul vl]
+; CHECK-NEXT: str z24, [x9, #4, mul vl]
+; CHECK-NEXT: frintx z31.s, p0/m, z31.s
+; CHECK-NEXT: frintx z7.s, p0/m, z7.s
+; CHECK-NEXT: str z25, [x9, #2, mul vl]
+; CHECK-NEXT: frintx z6.s, p0/m, z6.s
+; CHECK-NEXT: frintx z3.s, p0/m, z3.s
+; CHECK-NEXT: str z26, [x9, #1, mul vl]
+; CHECK-NEXT: frintx z27.s, p0/m, z27.s
+; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.s
; CHECK-NEXT: str z4, [x8, #8, mul vl]
-; CHECK-NEXT: str z29, [x8, #7, mul vl]
-; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.s
+; CHECK-NEXT: uunpkhi z4.d, z5.s
+; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.s
; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.s
-; CHECK-NEXT: str z3, [x8, #6, mul vl]
-; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.s
+; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.s
+; CHECK-NEXT: fcvtzs z30.d, p0/m, z30.s
; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.s
-; CHECK-NEXT: str z28, [x8, #3, mul vl]
-; CHECK-NEXT: str z1, [x8, #2, mul vl]
-; CHECK-NEXT: str z24, [x8, #5, mul vl]
+; CHECK-NEXT: fcvtzs z31.d, p0/m, z31.s
+; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.s
+; CHECK-NEXT: str z28, [x11, #2, mul vl]
+; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.s
+; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.s
+; CHECK-NEXT: str z29, [x11, #1, mul vl]
+; CHECK-NEXT: frintx z4.s, p0/m, z4.s
+; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.s
; CHECK-NEXT: str z2, [x8, #4, mul vl]
+; CHECK-NEXT: str z1, [x13, #1, mul vl]
+; CHECK-NEXT: str z30, [x8, #2, mul vl]
+; CHECK-NEXT: str z0, [x8, #1, mul vl]
+; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.s
+; CHECK-NEXT: str z31, [x8]
+; CHECK-NEXT: add x8, x9, x10
+; CHECK-NEXT: add x9, x9, x12
+; CHECK-NEXT: str z7, [x8, #2, mul vl]
; CHECK-NEXT: str z6, [x8, #1, mul vl]
-; CHECK-NEXT: str z0, [x8]
+; CHECK-NEXT: add x8, x8, x12
+; CHECK-NEXT: str z4, [x9, #1, mul vl]
+; CHECK-NEXT: add x9, x11, x12
+; CHECK-NEXT: str z3, [x9, #1, mul vl]
+; CHECK-NEXT: str z27, [x8, #1, mul vl]
; CHECK-NEXT: ret
%a = call <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv32f32(<vscale x 32 x float> %x)
ret <vscale x 32 x i64> %a
@@ -479,71 +499,96 @@ declare <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f64(<vscale x 16 x double
define <vscale x 32 x i64> @llrint_v32f64(<vscale x 32 x double> %x) {
; CHECK-LABEL: llrint_v32f64:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldr z0, [x0, #15, mul vl]
-; CHECK-NEXT: ldr z1, [x0, #14, mul vl]
+; CHECK-NEXT: rdvl x9, #1
+; CHECK-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-NEXT: ldr z1, [x0]
+; CHECK-NEXT: add x10, x0, x9
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ldr z2, [x0, #13, mul vl]
-; CHECK-NEXT: ldr z3, [x0, #12, mul vl]
-; CHECK-NEXT: ldr z4, [x0, #11, mul vl]
-; CHECK-NEXT: ldr z5, [x0, #10, mul vl]
-; CHECK-NEXT: ldr z6, [x0, #9, mul vl]
-; CHECK-NEXT: ldr z7, [x0, #8, mul vl]
-; CHECK-NEXT: frintx z0.d, p0/m, z0.d
+; CHECK-NEXT: add x11, x10, x9
+; CHECK-NEXT: ldr z2, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x11, x9
+; CHECK-NEXT: ldr z3, [x11, #1, mul vl]
; CHECK-NEXT: frintx z1.d, p0/m, z1.d
-; CHECK-NEXT: ldr z24, [x0, #7, mul vl]
-; CHECK-NEXT: ldr z25, [x0, #6, mul vl]
+; CHECK-NEXT: ldr z4, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: frintx z0.d, p0/m, z0.d
+; CHECK-NEXT: add x11, x10, x9
+; CHECK-NEXT: ldr z5, [x10, #1, mul vl]
; CHECK-NEXT: frintx z2.d, p0/m, z2.d
+; CHECK-NEXT: add x10, x11, x9
+; CHECK-NEXT: ldr z6, [x11, #1, mul vl]
; CHECK-NEXT: frintx z3.d, p0/m, z3.d
-; CHECK-NEXT: ldr z26, [x0, #5, mul vl]
-; CHECK-NEXT: ldr z27, [x0, #4, mul vl]
+; CHECK-NEXT: ldr z7, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x10, x9
; CHECK-NEXT: frintx z4.d, p0/m, z4.d
-; CHECK-NEXT: ldr z28, [x0, #3, mul vl]
-; CHECK-NEXT: ldr z29, [x0, #2, mul vl]
+; CHECK-NEXT: ldr z24, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x10, x9
; CHECK-NEXT: frintx z5.d, p0/m, z5.d
-; CHECK-NEXT: ldr z30, [x0, #1, mul vl]
-; CHECK-NEXT: ldr z31, [x0]
+; CHECK-NEXT: add x11, x10, x9
+; CHECK-NEXT: ldr z25, [x10, #1, mul vl]
; CHECK-NEXT: frintx z6.d, p0/m, z6.d
+; CHECK-NEXT: add x10, x11, x9
+; CHECK-NEXT: ldr z26, [x11, #1, mul vl]
; CHECK-NEXT: frintx z7.d, p0/m, z7.d
+; CHECK-NEXT: ldr z27, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d
+; CHECK-NEXT: ldr z28, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d
+; CHECK-NEXT: add x11, x10, x9
; CHECK-NEXT: frintx z24.d, p0/m, z24.d
+; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.d
+; CHECK-NEXT: add x9, x11, x9
+; CHECK-NEXT: ldr z29, [x11, #1, mul vl]
+; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d
+; CHECK-NEXT: ldr z31, [x10, #1, mul vl]
+; CHECK-NEXT: ldr z30, [x9, #1, mul vl]
; CHECK-NEXT: frintx z25.d, p0/m, z25.d
+; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.d
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.d
; CHECK-NEXT: frintx z26.d, p0/m, z26.d
; CHECK-NEXT: frintx z27.d, p0/m, z27.d
; CHECK-NEXT: frintx z28.d, p0/m, z28.d
-; CHECK-NEXT: frintx z29.d, p0/m, z29.d
-; CHECK-NEXT: frintx z30.d, p0/m, z30.d
-; CHECK-NEXT: frintx z31.d, p0/m, z31.d
-; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d
-; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d
-; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d
-; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.d
-; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.d
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.d
+; CHECK-NEXT: rdvl x9, #2
; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d
+; CHECK-NEXT: frintx z29.d, p0/m, z29.d
+; CHECK-NEXT: add x10, x8, x9
; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.d
+; CHECK-NEXT: frintx z31.d, p0/m, z31.d
+; CHECK-NEXT: str z0, [x8, #1, mul vl]
+; CHECK-NEXT: str z1, [x8]
+; CHECK-NEXT: frintx z30.d, p0/m, z30.d
; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.d
+; CHECK-NEXT: str z2, [x8, #2, mul vl]
; CHECK-NEXT: fcvtzs z25.d, p0/m, z25.d
; CHECK-NEXT: fcvtzs z26.d, p0/m, z26.d
+; CHECK-NEXT: str z3, [x10, #1, mul vl]
+; CHECK-NEXT: rdvl x10, #4
; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.d
+; CHECK-NEXT: add x11, x8, x10
+; CHECK-NEXT: str z4, [x8, #4, mul vl]
; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.d
+; CHECK-NEXT: str z5, [x11, #1, mul vl]
+; CHECK-NEXT: fcvtzs z31.d, p0/m, z31.d
; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.d
+; CHECK-NEXT: str z6, [x11, #2, mul vl]
+; CHECK-NEXT: add x11, x11, x9
; CHECK-NEXT: fcvtzs z30.d, p0/m, z30.d
-; CHECK-NEXT: fcvtzs z31.d, p0/m, z31.d
-; CHECK-NEXT: str z0, [x8, #15, mul vl]
-; CHECK-NEXT: str z1, [x8, #14, mul vl]
-; CHECK-NEXT: str z2, [x8, #13, mul vl]
-; CHECK-NEXT: str z3, [x8, #12, mul vl]
-; CHECK-NEXT: str z4, [x8, #11, mul vl]
-; CHECK-NEXT: str z5, [x8, #10, mul vl]
-; CHECK-NEXT: str z6, [x8, #9, mul vl]
-; CHECK-NEXT: str z7, [x8, #8, mul vl]
-; CHECK-NEXT: str z24, [x8, #7, mul vl]
-; CHECK-NEXT: str z25, [x8, #6, mul vl]
-; CHECK-NEXT: str z26, [x8, #5, mul vl]
-; CHECK-NEXT: str z27, [x8, #4, mul vl]
-; CHECK-NEXT: str z28, [x8, #3, mul vl]
+; CHECK-NEXT: str z7, [x11, #1, mul vl]
+; CHECK-NEXT: rdvl x11, #8
+; CHECK-NEXT: str z24, [x8, #8, mul vl]
+; CHECK-NEXT: add x8, x8, x11
+; CHECK-NEXT: add x11, x8, x9
+; CHECK-NEXT: str z25, [x8, #1, mul vl]
+; CHECK-NEXT: str z26, [x8, #2, mul vl]
+; CHECK-NEXT: str z27, [x11, #1, mul vl]
+; CHECK-NEXT: str z28, [x8, #4, mul vl]
+; CHECK-NEXT: add x8, x8, x10
+; CHECK-NEXT: str z31, [x8, #1, mul vl]
; CHECK-NEXT: str z29, [x8, #2, mul vl]
+; CHECK-NEXT: add x8, x8, x9
; CHECK-NEXT: str z30, [x8, #1, mul vl]
-; CHECK-NEXT: str z31, [x8]
; CHECK-NEXT: ret
%a = call <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv16f64(<vscale x 32 x double> %x)
ret <vscale x 32 x i64> %a
diff --git a/llvm/test/CodeGen/AArch64/sve-load-store-legalisation.ll b/llvm/test/CodeGen/AArch64/sve-load-store-legalisation.ll
index 4f9dc9beb9e08..4c3a15b453840 100644
--- a/llvm/test/CodeGen/AArch64/sve-load-store-legalisation.ll
+++ b/llvm/test/CodeGen/AArch64/sve-load-store-legalisation.ll
@@ -213,9 +213,12 @@ define void @sve_load_store_nxv13i8(ptr %a, ptr %b) {
define void @sve_load_store_nxv14i8(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv14i8:
; CHECK: // %bb.0:
+; CHECK-NEXT: cnth x8
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: add x9, x0, x8
; CHECK-NEXT: ptrue p1.s
-; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT: add x8, x1, x8
+; CHECK-NEXT: ld1b { z0.d }, p0/z, [x9, #2, mul vl]
; CHECK-NEXT: ptrue p2.h
; CHECK-NEXT: ld1b { z1.s }, p1/z, [x0, #2, mul vl]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
@@ -229,7 +232,7 @@ define void @sve_load_store_nxv14i8(ptr %a, ptr %b) {
; CHECK-NEXT: st1b { z0.h }, p2, [x1]
; CHECK-NEXT: uunpklo z2.d, z2.s
; CHECK-NEXT: st1b { z1.s }, p1, [x1, #2, mul vl]
-; CHECK-NEXT: st1b { z2.d }, p0, [x1, #6, mul vl]
+; CHECK-NEXT: st1b { z2.d }, p0, [x8, #2, mul vl]
; CHECK-NEXT: ret
%c = load <vscale x 14 x i8>, ptr %a
store <vscale x 14 x i8> %c, ptr %b
@@ -363,11 +366,13 @@ define void @sve_load_store_nxv22i8(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv22i8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: cntw x8, all, mul #5
+; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: ldr z2, [x0]
; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: add x9, x0, x8
+; CHECK-NEXT: add x8, x1, x8
; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, #4, mul vl]
-; CHECK-NEXT: ld1b { z1.d }, p1/z, [x0, x8]
+; CHECK-NEXT: ld1b { z1.d }, p1/z, [x9, #2, mul vl]
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b
; CHECK-NEXT: uunpklo z0.h, z0.b
@@ -379,8 +384,8 @@ define void @sve_load_store_nxv22i8(ptr %a, ptr %b) {
; CHECK-NEXT: uunpkhi z1.s, z0.h
; CHECK-NEXT: uunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z1.d, z1.s
-; CHECK-NEXT: st1b { z1.d }, p1, [x1, x8]
; CHECK-NEXT: st1b { z0.s }, p0, [x1, #4, mul vl]
+; CHECK-NEXT: st1b { z1.d }, p1, [x8, #2, mul vl]
; CHECK-NEXT: str z2, [x1]
; CHECK-NEXT: ret
%c = load <vscale x 22 x i8>, ptr %a
@@ -445,11 +450,13 @@ define void @sve_load_store_nxv25i8(ptr %a, ptr %b) {
define void @sve_load_store_nxv26i8(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv26i8:
; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cnth x8, all, mul #3
; CHECK-NEXT: ldr z2, [x0]
+; CHECK-NEXT: add x9, x0, x8
; CHECK-NEXT: ptrue p1.h
-; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, x8]
+; CHECK-NEXT: add x8, x1, x8
+; CHECK-NEXT: ld1b { z0.d }, p0/z, [x9, #4, mul vl]
; CHECK-NEXT: ld1b { z1.h }, p1/z, [x0, #2, mul vl]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
@@ -457,9 +464,9 @@ define void @sve_load_store_nxv26i8(ptr %a, ptr %b) {
; CHECK-NEXT: uunpkhi z1.h, z0.b
; CHECK-NEXT: uunpklo z0.h, z0.b
; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z1.d, z1.s
-; CHECK-NEXT: st1b { z1.d }, p0, [x1, x8]
; CHECK-NEXT: st1b { z0.h }, p1, [x1, #2, mul vl]
+; CHECK-NEXT: uunpklo z1.d, z1.s
+; CHECK-NEXT: st1b { z1.d }, p0, [x8, #4, mul vl]
; CHECK-NEXT: str z2, [x1]
; CHECK-NEXT: ret
%c = load <vscale x 26 x i8>, ptr %a
@@ -489,19 +496,22 @@ define void @sve_load_store_nxv27i8(ptr %a, ptr %b) {
define void @sve_load_store_nxv28i8(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv28i8:
; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: ptrue p0.s
; CHECK-NEXT: ldr z2, [x0]
+; CHECK-NEXT: add x9, x0, x8
; CHECK-NEXT: ptrue p1.h
-; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT: add x8, x1, x8
+; CHECK-NEXT: ld1b { z0.s }, p0/z, [x9, #2, mul vl]
; CHECK-NEXT: ld1b { z1.h }, p1/z, [x0, #2, mul vl]
-; CHECK-NEXT: str z2, [x1]
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
; CHECK-NEXT: uzp1 z0.b, z1.b, z0.b
; CHECK-NEXT: uunpkhi z1.h, z0.b
; CHECK-NEXT: uunpklo z0.h, z0.b
; CHECK-NEXT: uunpklo z1.s, z1.h
; CHECK-NEXT: st1b { z0.h }, p1, [x1, #2, mul vl]
-; CHECK-NEXT: st1b { z1.s }, p0, [x1, #6, mul vl]
+; CHECK-NEXT: st1b { z1.s }, p0, [x8, #2, mul vl]
+; CHECK-NEXT: str z2, [x1]
; CHECK-NEXT: ret
%c = load <vscale x 28 x i8>, ptr %a
store <vscale x 28 x i8> %c, ptr %b
@@ -530,13 +540,18 @@ define void @sve_load_store_nxv29i8(ptr %a, ptr %b) {
define void @sve_load_store_nxv30i8(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv30i8:
; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: cnth x9
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cntw x8, all, mul #7
-; CHECK-NEXT: ldr z3, [x0]
+; CHECK-NEXT: add x10, x0, x8
; CHECK-NEXT: ptrue p1.s
-; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, x8]
+; CHECK-NEXT: ldr z3, [x0]
+; CHECK-NEXT: add x11, x10, x9
; CHECK-NEXT: ptrue p2.h
-; CHECK-NEXT: ld1b { z1.s }, p1/z, [x0, #6, mul vl]
+; CHECK-NEXT: add x8, x1, x8
+; CHECK-NEXT: ld1b { z0.d }, p0/z, [x11, #2, mul vl]
+; CHECK-NEXT: ld1b { z1.s }, p1/z, [x10, #2, mul vl]
+; CHECK-NEXT: add x9, x8, x9
; CHECK-NEXT: ld1b { z2.h }, p2/z, [x0, #2, mul vl]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
@@ -545,10 +560,10 @@ define void @sve_load_store_nxv30i8(ptr %a, ptr %b) {
; CHECK-NEXT: uunpklo z0.h, z0.b
; CHECK-NEXT: uunpkhi z2.s, z1.h
; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z2.d, z2.s
-; CHECK-NEXT: st1b { z2.d }, p0, [x1, x8]
; CHECK-NEXT: st1b { z0.h }, p2, [x1, #2, mul vl]
-; CHECK-NEXT: st1b { z1.s }, p1, [x1, #6, mul vl]
+; CHECK-NEXT: uunpklo z2.d, z2.s
+; CHECK-NEXT: st1b { z1.s }, p1, [x8, #2, mul vl]
+; CHECK-NEXT: st1b { z2.d }, p0, [x9, #2, mul vl]
; CHECK-NEXT: str z3, [x1]
; CHECK-NEXT: ret
%c = load <vscale x 30 x i8>, ptr %a
@@ -802,19 +817,22 @@ define void @sve_load_store_nxv13i16(ptr %a, ptr %b) {
define void @sve_load_store_nxv14i16(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv14i16:
; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: ldr z2, [x0]
+; CHECK-NEXT: add x9, x0, x8
; CHECK-NEXT: ptrue p1.s
-; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT: add x8, x1, x8
+; CHECK-NEXT: ld1h { z0.d }, p0/z, [x9, #2, mul vl]
; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0, #2, mul vl]
-; CHECK-NEXT: str z2, [x1]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
; CHECK-NEXT: uunpkhi z1.s, z0.h
; CHECK-NEXT: uunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z1.d, z1.s
; CHECK-NEXT: st1h { z0.s }, p1, [x1, #2, mul vl]
-; CHECK-NEXT: st1h { z1.d }, p0, [x1, #6, mul vl]
+; CHECK-NEXT: st1h { z1.d }, p0, [x8, #2, mul vl]
+; CHECK-NEXT: str z2, [x1]
; CHECK-NEXT: ret
%c = load <vscale x 14 x i16>, ptr %a
store <vscale x 14 x i16> %c, ptr %b
@@ -1237,14 +1255,17 @@ define void @sve_load_store_nxv13f16(ptr %a, ptr %b) {
define void @sve_load_store_nxv14f16(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv14f16:
; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: ldr z2, [x0]
; CHECK-NEXT: ptrue p1.s
-; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT: add x9, x0, x8
+; CHECK-NEXT: add x8, x1, x8
+; CHECK-NEXT: ld1h { z0.d }, p0/z, [x9, #2, mul vl]
; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0, #2, mul vl]
-; CHECK-NEXT: str z2, [x1]
-; CHECK-NEXT: st1h { z0.d }, p0, [x1, #6, mul vl]
+; CHECK-NEXT: st1h { z0.d }, p0, [x8, #2, mul vl]
; CHECK-NEXT: st1h { z1.s }, p1, [x1, #2, mul vl]
+; CHECK-NEXT: str z2, [x1]
; CHECK-NEXT: ret
%c = load <vscale x 14 x half>, ptr %a
store <vscale x 14 x half> %c, ptr %b
@@ -1665,14 +1686,17 @@ define void @sve_load_store_nxv13bf16(ptr %a, ptr %b) {
define void @sve_load_store_nxv14bf16(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv14bf16:
; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: ldr z2, [x0]
; CHECK-NEXT: ptrue p1.s
-; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT: add x9, x0, x8
+; CHECK-NEXT: add x8, x1, x8
+; CHECK-NEXT: ld1h { z0.d }, p0/z, [x9, #2, mul vl]
; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0, #2, mul vl]
-; CHECK-NEXT: str z2, [x1]
-; CHECK-NEXT: st1h { z0.d }, p0, [x1, #6, mul vl]
+; CHECK-NEXT: st1h { z0.d }, p0, [x8, #2, mul vl]
; CHECK-NEXT: st1h { z1.s }, p1, [x1, #2, mul vl]
+; CHECK-NEXT: str z2, [x1]
; CHECK-NEXT: ret
%c = load <vscale x 14 x bfloat>, ptr %a
store <vscale x 14 x bfloat> %c, ptr %b
@@ -1963,20 +1987,23 @@ define <vscale x 14 x i16> @sve_sextload_nxv14i8(ptr %a, ptr %b) {
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: cntd x8, all, mul #7
; CHECK-NEXT: cnth x9
-; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: ptrue p1.d
; CHECK-NEXT: whilelo p0.h, x9, x8
+; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: whilelo p0.h, xzr, x8
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: ld1sb { z2.h }, p0/z, [x0]
-; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: ptrue p0.s
; CHECK-NEXT: uunpkhi z1.s, z0.h
; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: str z2, [sp]
; CHECK-NEXT: uunpklo z1.d, z1.s
-; CHECK-NEXT: st1h { z0.s }, p1, [sp, #2, mul vl]
-; CHECK-NEXT: ldr z0, [sp]
-; CHECK-NEXT: st1h { z1.d }, p0, [sp, #6, mul vl]
+; CHECK-NEXT: st1h { z1.d }, p1, [x8, #2, mul vl]
+; CHECK-NEXT: st1h { z0.s }, p0, [sp, #2, mul vl]
+; CHECK-NEXT: str z2, [sp]
; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z0, [sp]
; CHECK-NEXT: addvl sp, sp, #2
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
@@ -2485,20 +2512,23 @@ define <vscale x 14 x i16> @sve_zextload_nxv14i8(ptr %a, ptr %b) {
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: cntd x8, all, mul #7
; CHECK-NEXT: cnth x9
-; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: ptrue p1.d
; CHECK-NEXT: whilelo p0.h, x9, x8
+; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: whilelo p0.h, xzr, x8
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: ld1sb { z2.h }, p0/z, [x0]
-; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: ptrue p0.s
; CHECK-NEXT: uunpkhi z1.s, z0.h
; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: str z2, [sp]
; CHECK-NEXT: uunpklo z1.d, z1.s
-; CHECK-NEXT: st1h { z0.s }, p1, [sp, #2, mul vl]
-; CHECK-NEXT: ldr z0, [sp]
-; CHECK-NEXT: st1h { z1.d }, p0, [sp, #6, mul vl]
+; CHECK-NEXT: st1h { z1.d }, p1, [x8, #2, mul vl]
+; CHECK-NEXT: st1h { z0.s }, p0, [sp, #2, mul vl]
+; CHECK-NEXT: str z2, [sp]
; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z0, [sp]
; CHECK-NEXT: addvl sp, sp, #2
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-lrint.ll b/llvm/test/CodeGen/AArch64/sve-lrint.ll
index 889d4a1c8f73a..7efff72a15662 100644
--- a/llvm/test/CodeGen/AArch64/sve-lrint.ll
+++ b/llvm/test/CodeGen/AArch64/sve-lrint.ll
@@ -121,77 +121,87 @@ define <vscale x 32 x i64> @lrint_v32f16(<vscale x 32 x half> %x) {
; CHECK: // %bb.0:
; CHECK-NEXT: uunpkhi z4.s, z3.h
; CHECK-NEXT: uunpklo z3.s, z3.h
+; CHECK-NEXT: rdvl x10, #8
+; CHECK-NEXT: uunpkhi z5.s, z2.h
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: uunpkhi z7.s, z2.h
+; CHECK-NEXT: rdvl x11, #4
; CHECK-NEXT: uunpklo z2.s, z2.h
-; CHECK-NEXT: uunpkhi z24.s, z0.h
-; CHECK-NEXT: uunpkhi z25.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpkhi z5.d, z4.s
-; CHECK-NEXT: uunpklo z4.d, z4.s
-; CHECK-NEXT: uunpkhi z6.d, z3.s
+; CHECK-NEXT: uunpklo z27.s, z1.h
+; CHECK-NEXT: rdvl x9, #2
+; CHECK-NEXT: uunpkhi z1.s, z1.h
+; CHECK-NEXT: uunpklo z25.s, z0.h
+; CHECK-NEXT: add x10, x8, x10
+; CHECK-NEXT: uunpklo z6.d, z4.s
+; CHECK-NEXT: uunpkhi z7.d, z3.s
+; CHECK-NEXT: add x12, x10, x11
; CHECK-NEXT: uunpklo z3.d, z3.s
-; CHECK-NEXT: uunpkhi z26.d, z7.s
-; CHECK-NEXT: uunpklo z7.d, z7.s
-; CHECK-NEXT: uunpkhi z27.d, z2.s
-; CHECK-NEXT: uunpkhi z28.d, z24.s
+; CHECK-NEXT: uunpkhi z24.d, z5.s
+; CHECK-NEXT: add x13, x10, x9
+; CHECK-NEXT: uunpklo z5.d, z5.s
+; CHECK-NEXT: uunpkhi z0.s, z0.h
+; CHECK-NEXT: uunpkhi z26.d, z2.s
; CHECK-NEXT: uunpklo z2.d, z2.s
-; CHECK-NEXT: uunpkhi z29.d, z25.s
-; CHECK-NEXT: uunpklo z25.d, z25.s
-; CHECK-NEXT: frintx z5.h, p0/m, z5.h
-; CHECK-NEXT: frintx z4.h, p0/m, z4.h
+; CHECK-NEXT: uunpkhi z29.d, z1.s
+; CHECK-NEXT: uunpklo z1.d, z1.s
; CHECK-NEXT: frintx z6.h, p0/m, z6.h
+; CHECK-NEXT: frintx z7.h, p0/m, z7.h
+; CHECK-NEXT: uunpkhi z30.d, z27.s
; CHECK-NEXT: frintx z3.h, p0/m, z3.h
+; CHECK-NEXT: frintx z24.h, p0/m, z24.h
+; CHECK-NEXT: uunpkhi z4.d, z4.s
+; CHECK-NEXT: frintx z5.h, p0/m, z5.h
+; CHECK-NEXT: uunpklo z28.d, z0.s
+; CHECK-NEXT: uunpkhi z0.d, z0.s
; CHECK-NEXT: frintx z26.h, p0/m, z26.h
-; CHECK-NEXT: frintx z7.h, p0/m, z7.h
-; CHECK-NEXT: frintx z27.h, p0/m, z27.h
; CHECK-NEXT: frintx z2.h, p0/m, z2.h
-; CHECK-NEXT: frintx z28.h, p0/m, z28.h
; CHECK-NEXT: frintx z29.h, p0/m, z29.h
-; CHECK-NEXT: frintx z25.h, p0/m, z25.h
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
-; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.h
; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.h
+; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.h
+; CHECK-NEXT: frintx z1.h, p0/m, z1.h
; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.h
+; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.h
+; CHECK-NEXT: frintx z30.h, p0/m, z30.h
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
+; CHECK-NEXT: frintx z0.h, p0/m, z0.h
+; CHECK-NEXT: frintx z28.h, p0/m, z28.h
; CHECK-NEXT: fcvtzs z26.d, p0/m, z26.h
-; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.h
-; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.h
+; CHECK-NEXT: frintx z4.h, p0/m, z4.h
; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.h
-; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.h
+; CHECK-NEXT: str z6, [x12, #2, mul vl]
+; CHECK-NEXT: uunpklo z6.d, z27.s
; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.h
-; CHECK-NEXT: fcvtzs z25.d, p0/m, z25.h
-; CHECK-NEXT: str z5, [x8, #15, mul vl]
-; CHECK-NEXT: uunpkhi z5.d, z1.s
-; CHECK-NEXT: uunpklo z1.d, z1.s
-; CHECK-NEXT: str z4, [x8, #14, mul vl]
-; CHECK-NEXT: uunpkhi z4.d, z0.s
-; CHECK-NEXT: uunpklo z0.d, z0.s
-; CHECK-NEXT: str z3, [x8, #12, mul vl]
-; CHECK-NEXT: uunpklo z3.d, z24.s
-; CHECK-NEXT: str z6, [x8, #13, mul vl]
-; CHECK-NEXT: str z26, [x8, #11, mul vl]
-; CHECK-NEXT: frintx z5.h, p0/m, z5.h
-; CHECK-NEXT: frintx z1.h, p0/m, z1.h
-; CHECK-NEXT: str z7, [x8, #10, mul vl]
-; CHECK-NEXT: frintx z4.h, p0/m, z4.h
-; CHECK-NEXT: frintx z0.h, p0/m, z0.h
-; CHECK-NEXT: str z27, [x8, #9, mul vl]
-; CHECK-NEXT: frintx z3.h, p0/m, z3.h
-; CHECK-NEXT: str z2, [x8, #8, mul vl]
-; CHECK-NEXT: str z29, [x8, #7, mul vl]
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
+; CHECK-NEXT: str z7, [x12, #1, mul vl]
; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.h
-; CHECK-NEXT: str z25, [x8, #6, mul vl]
-; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.h
+; CHECK-NEXT: fcvtzs z30.d, p0/m, z30.h
+; CHECK-NEXT: str z3, [x10, #4, mul vl]
+; CHECK-NEXT: uunpkhi z3.d, z25.s
; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.h
-; CHECK-NEXT: str z28, [x8, #3, mul vl]
+; CHECK-NEXT: str z24, [x13, #1, mul vl]
+; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.h
+; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.h
+; CHECK-NEXT: str z5, [x10, #2, mul vl]
+; CHECK-NEXT: uunpklo z5.d, z25.s
+; CHECK-NEXT: frintx z6.h, p0/m, z6.h
+; CHECK-NEXT: str z26, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x8, x11
+; CHECK-NEXT: add x11, x10, x9
+; CHECK-NEXT: str z2, [x8, #8, mul vl]
+; CHECK-NEXT: frintx z3.h, p0/m, z3.h
+; CHECK-NEXT: str z29, [x11, #1, mul vl]
+; CHECK-NEXT: str z1, [x10, #2, mul vl]
+; CHECK-NEXT: frintx z5.h, p0/m, z5.h
+; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.h
+; CHECK-NEXT: str z30, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x8, x9
; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.h
-; CHECK-NEXT: str z5, [x8, #5, mul vl]
-; CHECK-NEXT: str z1, [x8, #4, mul vl]
-; CHECK-NEXT: str z3, [x8, #2, mul vl]
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
+; CHECK-NEXT: str z6, [x8, #4, mul vl]
+; CHECK-NEXT: str z0, [x10, #1, mul vl]
+; CHECK-NEXT: str z28, [x8, #2, mul vl]
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
+; CHECK-NEXT: str z5, [x8]
+; CHECK-NEXT: add x8, x12, x9
; CHECK-NEXT: str z4, [x8, #1, mul vl]
-; CHECK-NEXT: str z0, [x8]
; CHECK-NEXT: ret
%a = call <vscale x 32 x i64> @llvm.lrint.nxv32i64.nxv32f16(<vscale x 32 x half> %x)
ret <vscale x 32 x i64> %a
@@ -312,71 +322,81 @@ define <vscale x 16 x i64> @lrint_v16f32(<vscale x 16 x float> %x) {
define <vscale x 32 x i64> @lrint_v32f32(<vscale x 32 x float> %x) {
; CHECK-LABEL: lrint_v32f32:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z24.d, z7.s
-; CHECK-NEXT: uunpkhi z25.d, z6.s
-; CHECK-NEXT: uunpklo z6.d, z6.s
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: uunpklo z7.d, z7.s
-; CHECK-NEXT: uunpkhi z26.d, z5.s
-; CHECK-NEXT: uunpklo z5.d, z5.s
-; CHECK-NEXT: uunpkhi z27.d, z4.s
-; CHECK-NEXT: uunpkhi z28.d, z1.s
+; CHECK-NEXT: uunpklo z24.d, z6.s
+; CHECK-NEXT: uunpklo z25.d, z5.s
+; CHECK-NEXT: rdvl x9, #8
+; CHECK-NEXT: uunpkhi z26.d, z4.s
; CHECK-NEXT: uunpklo z4.d, z4.s
-; CHECK-NEXT: uunpkhi z29.d, z3.s
-; CHECK-NEXT: uunpklo z3.d, z3.s
+; CHECK-NEXT: add x9, x8, x9
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: uunpklo z28.d, z3.s
+; CHECK-NEXT: rdvl x10, #4
+; CHECK-NEXT: uunpkhi z29.d, z2.s
+; CHECK-NEXT: uunpklo z30.d, z1.s
+; CHECK-NEXT: rdvl x12, #2
; CHECK-NEXT: frintx z24.s, p0/m, z24.s
-; CHECK-NEXT: frintx z6.s, p0/m, z6.s
-; CHECK-NEXT: uunpklo z1.d, z1.s
-; CHECK-NEXT: frintx z7.s, p0/m, z7.s
; CHECK-NEXT: frintx z25.s, p0/m, z25.s
+; CHECK-NEXT: add x11, x8, x10
; CHECK-NEXT: frintx z26.s, p0/m, z26.s
-; CHECK-NEXT: frintx z5.s, p0/m, z5.s
-; CHECK-NEXT: frintx z27.s, p0/m, z27.s
-; CHECK-NEXT: frintx z28.s, p0/m, z28.s
; CHECK-NEXT: frintx z4.s, p0/m, z4.s
-; CHECK-NEXT: frintx z29.s, p0/m, z29.s
-; CHECK-NEXT: frintx z3.s, p0/m, z3.s
+; CHECK-NEXT: add x13, x8, x12
+; CHECK-NEXT: uunpklo z2.d, z2.s
+; CHECK-NEXT: uunpkhi z1.d, z1.s
+; CHECK-NEXT: uunpklo z31.d, z0.s
+; CHECK-NEXT: uunpkhi z0.d, z0.s
+; CHECK-NEXT: uunpkhi z27.d, z7.s
+; CHECK-NEXT: uunpkhi z6.d, z6.s
; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.s
-; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.s
-; CHECK-NEXT: frintx z1.s, p0/m, z1.s
-; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.s
; CHECK-NEXT: fcvtzs z25.d, p0/m, z25.s
+; CHECK-NEXT: uunpklo z7.d, z7.s
; CHECK-NEXT: fcvtzs z26.d, p0/m, z26.s
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.s
-; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.s
-; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.s
; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.s
-; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.s
-; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.s
-; CHECK-NEXT: str z24, [x8, #15, mul vl]
-; CHECK-NEXT: uunpkhi z24.d, z2.s
-; CHECK-NEXT: uunpklo z2.d, z2.s
-; CHECK-NEXT: str z6, [x8, #12, mul vl]
-; CHECK-NEXT: uunpkhi z6.d, z0.s
-; CHECK-NEXT: uunpklo z0.d, z0.s
-; CHECK-NEXT: str z7, [x8, #14, mul vl]
-; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.s
-; CHECK-NEXT: str z25, [x8, #13, mul vl]
-; CHECK-NEXT: str z26, [x8, #11, mul vl]
-; CHECK-NEXT: frintx z24.s, p0/m, z24.s
+; CHECK-NEXT: uunpkhi z3.d, z3.s
+; CHECK-NEXT: frintx z28.s, p0/m, z28.s
+; CHECK-NEXT: frintx z29.s, p0/m, z29.s
; CHECK-NEXT: frintx z2.s, p0/m, z2.s
-; CHECK-NEXT: str z5, [x8, #10, mul vl]
-; CHECK-NEXT: frintx z6.s, p0/m, z6.s
+; CHECK-NEXT: frintx z1.s, p0/m, z1.s
+; CHECK-NEXT: frintx z30.s, p0/m, z30.s
; CHECK-NEXT: frintx z0.s, p0/m, z0.s
-; CHECK-NEXT: str z27, [x8, #9, mul vl]
+; CHECK-NEXT: str z24, [x9, #4, mul vl]
+; CHECK-NEXT: frintx z31.s, p0/m, z31.s
+; CHECK-NEXT: frintx z7.s, p0/m, z7.s
+; CHECK-NEXT: str z25, [x9, #2, mul vl]
+; CHECK-NEXT: frintx z6.s, p0/m, z6.s
+; CHECK-NEXT: frintx z3.s, p0/m, z3.s
+; CHECK-NEXT: str z26, [x9, #1, mul vl]
+; CHECK-NEXT: frintx z27.s, p0/m, z27.s
+; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.s
; CHECK-NEXT: str z4, [x8, #8, mul vl]
-; CHECK-NEXT: str z29, [x8, #7, mul vl]
-; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.s
+; CHECK-NEXT: uunpkhi z4.d, z5.s
+; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.s
; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.s
-; CHECK-NEXT: str z3, [x8, #6, mul vl]
-; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.s
+; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.s
+; CHECK-NEXT: fcvtzs z30.d, p0/m, z30.s
; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.s
-; CHECK-NEXT: str z28, [x8, #3, mul vl]
-; CHECK-NEXT: str z1, [x8, #2, mul vl]
-; CHECK-NEXT: str z24, [x8, #5, mul vl]
+; CHECK-NEXT: fcvtzs z31.d, p0/m, z31.s
+; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.s
+; CHECK-NEXT: str z28, [x11, #2, mul vl]
+; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.s
+; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.s
+; CHECK-NEXT: str z29, [x11, #1, mul vl]
+; CHECK-NEXT: frintx z4.s, p0/m, z4.s
+; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.s
; CHECK-NEXT: str z2, [x8, #4, mul vl]
+; CHECK-NEXT: str z1, [x13, #1, mul vl]
+; CHECK-NEXT: str z30, [x8, #2, mul vl]
+; CHECK-NEXT: str z0, [x8, #1, mul vl]
+; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.s
+; CHECK-NEXT: str z31, [x8]
+; CHECK-NEXT: add x8, x9, x10
+; CHECK-NEXT: add x9, x9, x12
+; CHECK-NEXT: str z7, [x8, #2, mul vl]
; CHECK-NEXT: str z6, [x8, #1, mul vl]
-; CHECK-NEXT: str z0, [x8]
+; CHECK-NEXT: add x8, x8, x12
+; CHECK-NEXT: str z4, [x9, #1, mul vl]
+; CHECK-NEXT: add x9, x11, x12
+; CHECK-NEXT: str z3, [x9, #1, mul vl]
+; CHECK-NEXT: str z27, [x8, #1, mul vl]
; CHECK-NEXT: ret
%a = call <vscale x 32 x i64> @llvm.lrint.nxv32i64.nxv32f32(<vscale x 32 x float> %x)
ret <vscale x 32 x i64> %a
@@ -462,71 +482,96 @@ define <vscale x 16 x i64> @lrint_v16f64(<vscale x 16 x double> %x) {
define <vscale x 32 x i64> @lrint_v32f64(<vscale x 32 x double> %x) {
; CHECK-LABEL: lrint_v32f64:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldr z0, [x0, #15, mul vl]
-; CHECK-NEXT: ldr z1, [x0, #14, mul vl]
+; CHECK-NEXT: rdvl x9, #1
+; CHECK-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-NEXT: ldr z1, [x0]
+; CHECK-NEXT: add x10, x0, x9
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ldr z2, [x0, #13, mul vl]
-; CHECK-NEXT: ldr z3, [x0, #12, mul vl]
-; CHECK-NEXT: ldr z4, [x0, #11, mul vl]
-; CHECK-NEXT: ldr z5, [x0, #10, mul vl]
-; CHECK-NEXT: ldr z6, [x0, #9, mul vl]
-; CHECK-NEXT: ldr z7, [x0, #8, mul vl]
-; CHECK-NEXT: frintx z0.d, p0/m, z0.d
+; CHECK-NEXT: add x11, x10, x9
+; CHECK-NEXT: ldr z2, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x11, x9
+; CHECK-NEXT: ldr z3, [x11, #1, mul vl]
; CHECK-NEXT: frintx z1.d, p0/m, z1.d
-; CHECK-NEXT: ldr z24, [x0, #7, mul vl]
-; CHECK-NEXT: ldr z25, [x0, #6, mul vl]
+; CHECK-NEXT: ldr z4, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: frintx z0.d, p0/m, z0.d
+; CHECK-NEXT: add x11, x10, x9
+; CHECK-NEXT: ldr z5, [x10, #1, mul vl]
; CHECK-NEXT: frintx z2.d, p0/m, z2.d
+; CHECK-NEXT: add x10, x11, x9
+; CHECK-NEXT: ldr z6, [x11, #1, mul vl]
; CHECK-NEXT: frintx z3.d, p0/m, z3.d
-; CHECK-NEXT: ldr z26, [x0, #5, mul vl]
-; CHECK-NEXT: ldr z27, [x0, #4, mul vl]
+; CHECK-NEXT: ldr z7, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x10, x9
; CHECK-NEXT: frintx z4.d, p0/m, z4.d
-; CHECK-NEXT: ldr z28, [x0, #3, mul vl]
-; CHECK-NEXT: ldr z29, [x0, #2, mul vl]
+; CHECK-NEXT: ldr z24, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x10, x9
; CHECK-NEXT: frintx z5.d, p0/m, z5.d
-; CHECK-NEXT: ldr z30, [x0, #1, mul vl]
-; CHECK-NEXT: ldr z31, [x0]
+; CHECK-NEXT: add x11, x10, x9
+; CHECK-NEXT: ldr z25, [x10, #1, mul vl]
; CHECK-NEXT: frintx z6.d, p0/m, z6.d
+; CHECK-NEXT: add x10, x11, x9
+; CHECK-NEXT: ldr z26, [x11, #1, mul vl]
; CHECK-NEXT: frintx z7.d, p0/m, z7.d
+; CHECK-NEXT: ldr z27, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d
+; CHECK-NEXT: ldr z28, [x10, #1, mul vl]
+; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d
+; CHECK-NEXT: add x11, x10, x9
; CHECK-NEXT: frintx z24.d, p0/m, z24.d
+; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.d
+; CHECK-NEXT: add x9, x11, x9
+; CHECK-NEXT: ldr z29, [x11, #1, mul vl]
+; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d
+; CHECK-NEXT: ldr z31, [x10, #1, mul vl]
+; CHECK-NEXT: ldr z30, [x9, #1, mul vl]
; CHECK-NEXT: frintx z25.d, p0/m, z25.d
+; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.d
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.d
; CHECK-NEXT: frintx z26.d, p0/m, z26.d
; CHECK-NEXT: frintx z27.d, p0/m, z27.d
; CHECK-NEXT: frintx z28.d, p0/m, z28.d
-; CHECK-NEXT: frintx z29.d, p0/m, z29.d
-; CHECK-NEXT: frintx z30.d, p0/m, z30.d
-; CHECK-NEXT: frintx z31.d, p0/m, z31.d
-; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d
-; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d
-; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d
-; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.d
-; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.d
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.d
+; CHECK-NEXT: rdvl x9, #2
; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d
+; CHECK-NEXT: frintx z29.d, p0/m, z29.d
+; CHECK-NEXT: add x10, x8, x9
; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.d
+; CHECK-NEXT: frintx z31.d, p0/m, z31.d
+; CHECK-NEXT: str z0, [x8, #1, mul vl]
+; CHECK-NEXT: str z1, [x8]
+; CHECK-NEXT: frintx z30.d, p0/m, z30.d
; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.d
+; CHECK-NEXT: str z2, [x8, #2, mul vl]
; CHECK-NEXT: fcvtzs z25.d, p0/m, z25.d
; CHECK-NEXT: fcvtzs z26.d, p0/m, z26.d
+; CHECK-NEXT: str z3, [x10, #1, mul vl]
+; CHECK-NEXT: rdvl x10, #4
; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.d
+; CHECK-NEXT: add x11, x8, x10
+; CHECK-NEXT: str z4, [x8, #4, mul vl]
; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.d
+; CHECK-NEXT: str z5, [x11, #1, mul vl]
+; CHECK-NEXT: fcvtzs z31.d, p0/m, z31.d
; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.d
+; CHECK-NEXT: str z6, [x11, #2, mul vl]
+; CHECK-NEXT: add x11, x11, x9
; CHECK-NEXT: fcvtzs z30.d, p0/m, z30.d
-; CHECK-NEXT: fcvtzs z31.d, p0/m, z31.d
-; CHECK-NEXT: str z0, [x8, #15, mul vl]
-; CHECK-NEXT: str z1, [x8, #14, mul vl]
-; CHECK-NEXT: str z2, [x8, #13, mul vl]
-; CHECK-NEXT: str z3, [x8, #12, mul vl]
-; CHECK-NEXT: str z4, [x8, #11, mul vl]
-; CHECK-NEXT: str z5, [x8, #10, mul vl]
-; CHECK-NEXT: str z6, [x8, #9, mul vl]
-; CHECK-NEXT: str z7, [x8, #8, mul vl]
-; CHECK-NEXT: str z24, [x8, #7, mul vl]
-; CHECK-NEXT: str z25, [x8, #6, mul vl]
-; CHECK-NEXT: str z26, [x8, #5, mul vl]
-; CHECK-NEXT: str z27, [x8, #4, mul vl]
-; CHECK-NEXT: str z28, [x8, #3, mul vl]
+; CHECK-NEXT: str z7, [x11, #1, mul vl]
+; CHECK-NEXT: rdvl x11, #8
+; CHECK-NEXT: str z24, [x8, #8, mul vl]
+; CHECK-NEXT: add x8, x8, x11
+; CHECK-NEXT: add x11, x8, x9
+; CHECK-NEXT: str z25, [x8, #1, mul vl]
+; CHECK-NEXT: str z26, [x8, #2, mul vl]
+; CHECK-NEXT: str z27, [x11, #1, mul vl]
+; CHECK-NEXT: str z28, [x8, #4, mul vl]
+; CHECK-NEXT: add x8, x8, x10
+; CHECK-NEXT: str z31, [x8, #1, mul vl]
; CHECK-NEXT: str z29, [x8, #2, mul vl]
+; CHECK-NEXT: add x8, x8, x9
; CHECK-NEXT: str z30, [x8, #1, mul vl]
-; CHECK-NEXT: str z31, [x8]
; CHECK-NEXT: ret
%a = call <vscale x 32 x i64> @llvm.lrint.nxv32i64.nxv16f64(<vscale x 32 x double> %x)
ret <vscale x 32 x i64> %a
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll
index 5c506ab20e6f4..82407d56a9671 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll
@@ -231,19 +231,22 @@ define <vscale x 8 x i64> @masked_sload_x2_8i8_8i64(ptr %a, ptr %b, <vscale x 8
; CHECK-LABEL: masked_sload_x2_8i8_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: punpkhi p1.h, p0.b
-; CHECK-NEXT: punpklo p0.h, p0.b
-; CHECK-NEXT: punpkhi p2.h, p1.b
+; CHECK-NEXT: cntw x8
+; CHECK-NEXT: punpklo p2.h, p0.b
+; CHECK-NEXT: add x9, x0, x8
+; CHECK-NEXT: add x8, x1, x8
+; CHECK-NEXT: punpkhi p0.h, p1.b
; CHECK-NEXT: punpklo p1.h, p1.b
-; CHECK-NEXT: punpkhi p3.h, p0.b
-; CHECK-NEXT: ld1sb { z3.d }, p2/z, [x0, #3, mul vl]
-; CHECK-NEXT: ld1sb { z5.d }, p2/z, [x1, #3, mul vl]
-; CHECK-NEXT: punpklo p0.h, p0.b
+; CHECK-NEXT: punpkhi p3.h, p2.b
+; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x9, #1, mul vl]
+; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: punpklo p2.h, p2.b
; CHECK-NEXT: ld1sb { z2.d }, p1/z, [x0, #2, mul vl]
; CHECK-NEXT: ld1sb { z6.d }, p1/z, [x1, #2, mul vl]
; CHECK-NEXT: ld1sb { z1.d }, p3/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z7.d }, p3/z, [x1, #1, mul vl]
-; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]
-; CHECK-NEXT: ld1sb { z4.d }, p0/z, [x1]
+; CHECK-NEXT: ld1sb { z0.d }, p2/z, [x0]
+; CHECK-NEXT: ld1sb { z4.d }, p2/z, [x1]
; CHECK-NEXT: add z3.d, z3.d, z5.d
; CHECK-NEXT: add z2.d, z2.d, z6.d
; CHECK-NEXT: add z1.d, z1.d, z7.d
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-zext.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-zext.ll
index 09e276f1cc433..535fb5f4f9660 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-ldst-zext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-zext.ll
@@ -226,19 +226,22 @@ define <vscale x 8 x i64> @masked_zload_x2_8i8_8i64(ptr %a, ptr %b, <vscale x 8
; CHECK-LABEL: masked_zload_x2_8i8_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: punpkhi p1.h, p0.b
-; CHECK-NEXT: punpklo p0.h, p0.b
-; CHECK-NEXT: punpkhi p2.h, p1.b
+; CHECK-NEXT: cntw x8
+; CHECK-NEXT: punpklo p2.h, p0.b
+; CHECK-NEXT: add x9, x0, x8
+; CHECK-NEXT: add x8, x1, x8
+; CHECK-NEXT: punpkhi p0.h, p1.b
; CHECK-NEXT: punpklo p1.h, p1.b
-; CHECK-NEXT: punpkhi p3.h, p0.b
-; CHECK-NEXT: ld1b { z3.d }, p2/z, [x0, #3, mul vl]
-; CHECK-NEXT: ld1b { z5.d }, p2/z, [x1, #3, mul vl]
-; CHECK-NEXT: punpklo p0.h, p0.b
+; CHECK-NEXT: punpkhi p3.h, p2.b
+; CHECK-NEXT: ld1b { z3.d }, p0/z, [x9, #1, mul vl]
+; CHECK-NEXT: ld1b { z5.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: punpklo p2.h, p2.b
; CHECK-NEXT: ld1b { z2.d }, p1/z, [x0, #2, mul vl]
; CHECK-NEXT: ld1b { z6.d }, p1/z, [x1, #2, mul vl]
; CHECK-NEXT: ld1b { z1.d }, p3/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z7.d }, p3/z, [x1, #1, mul vl]
-; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
-; CHECK-NEXT: ld1b { z4.d }, p0/z, [x1]
+; CHECK-NEXT: ld1b { z0.d }, p2/z, [x0]
+; CHECK-NEXT: ld1b { z4.d }, p2/z, [x1]
; CHECK-NEXT: add z3.d, z3.d, z5.d
; CHECK-NEXT: add z2.d, z2.d, z6.d
; CHECK-NEXT: add z1.d, z1.d, z7.d
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter-legalize.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter-legalize.ll
index 622040eeb2f33..302e22db91734 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter-legalize.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter-legalize.ll
@@ -89,18 +89,25 @@ define void @masked_scatter_nxv8f32(<vscale x 8 x float> %data, ptr %base, <vsca
define void @masked_scatter_nxv32i32(<vscale x 32 x i32> %data, ptr %base, <vscale x 32 x i32> %offsets, <vscale x 32 x i1> %mask) #0 {
; CHECK-LABEL: masked_scatter_nxv32i32:
; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: punpklo p2.h, p0.b
; CHECK-NEXT: ldr z30, [x1, #1, mul vl]
+; CHECK-NEXT: add x9, x1, x8
; CHECK-NEXT: ldr z31, [x1]
; CHECK-NEXT: punpkhi p0.h, p0.b
-; CHECK-NEXT: ldr z28, [x1, #3, mul vl]
-; CHECK-NEXT: ldr z29, [x1, #2, mul vl]
+; CHECK-NEXT: add x10, x9, x8
; CHECK-NEXT: punpklo p3.h, p2.b
-; CHECK-NEXT: ldr z24, [x1, #7, mul vl]
-; CHECK-NEXT: ldr z25, [x1, #6, mul vl]
+; CHECK-NEXT: ldr z29, [x9, #1, mul vl]
+; CHECK-NEXT: add x11, x10, x8
; CHECK-NEXT: punpkhi p2.h, p2.b
-; CHECK-NEXT: ldr z26, [x1, #5, mul vl]
-; CHECK-NEXT: ldr z27, [x1, #4, mul vl]
+; CHECK-NEXT: ldr z28, [x10, #1, mul vl]
+; CHECK-NEXT: add x12, x11, x8
+; CHECK-NEXT: ldr z27, [x11, #1, mul vl]
+; CHECK-NEXT: add x13, x12, x8
+; CHECK-NEXT: ldr z26, [x12, #1, mul vl]
+; CHECK-NEXT: add x8, x13, x8
+; CHECK-NEXT: ldr z25, [x13, #1, mul vl]
+; CHECK-NEXT: ldr z24, [x8, #1, mul vl]
; CHECK-NEXT: st1w { z0.s }, p3, [x0, z31.s, sxtw #2]
; CHECK-NEXT: st1w { z1.s }, p2, [x0, z30.s, sxtw #2]
; CHECK-NEXT: punpklo p2.h, p0.b
diff --git a/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll b/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
index 714b6a0874cd7..cda0c7aef56d1 100644
--- a/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
@@ -615,18 +615,20 @@ define void @load_store_2x_vectors_bf16_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_i8_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_i8_r:
; SVE2p1: // %bb.0:
+; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
-; SVE2p1-NEXT: ldr z2, [x0]
-; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0]
+; SVE2p1-NEXT: incb x8, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
; SVE2p1-NEXT: add z0.b, z0.b, #5 // =0x5
; SVE2p1-NEXT: add z1.b, z1.b, #5 // =0x5
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: add z2.b, z2.b, #5 // =0x5
-; SVE2p1-NEXT: add z3.b, z3.b, #5 // =0x5
; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
-; SVE2p1-NEXT: str z2, [x0]
-; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
+; SVE2p1-NEXT: add z3.b, z3.b, #5 // =0x5
+; SVE2p1-NEXT: str z1, [x0]
+; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i8_r:
@@ -659,19 +661,21 @@ define void @load_store_4x_vectors_i8_r(ptr %addr) {
define void @load_store_4x_vectors_i8_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_i8_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: ptrue p0.b
; SVE2p1-NEXT: add x8, x0, x1
-; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: ptrue p0.b
+; SVE2p1-NEXT: add x9, x0, x1
+; SVE2p1-NEXT: incb x8, all, mul #2
+; SVE2p1-NEXT: ldr z1, [x9, #2, mul vl]
+; SVE2p1-NEXT: ldr z2, [x9, #1, mul vl]
; SVE2p1-NEXT: ld1b { z0.b }, p0/z, [x0, x1]
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: add z1.b, z1.b, #5 // =0x5
; SVE2p1-NEXT: add z2.b, z2.b, #5 // =0x5
-; SVE2p1-NEXT: add z3.b, z3.b, #5 // =0x5
; SVE2p1-NEXT: add z0.b, z0.b, #5 // =0x5
+; SVE2p1-NEXT: add z3.b, z3.b, #5 // =0x5
; SVE2p1-NEXT: st1b { z0.b }, p0, [x0, x1]
-; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
+; SVE2p1-NEXT: str z1, [x9, #2, mul vl]
+; SVE2p1-NEXT: str z2, [x9, #1, mul vl]
; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
@@ -708,18 +712,20 @@ define void @load_store_4x_vectors_i8_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_i16_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_i16_r:
; SVE2p1: // %bb.0:
+; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
-; SVE2p1-NEXT: ldr z2, [x0]
-; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0]
+; SVE2p1-NEXT: incb x8, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
; SVE2p1-NEXT: add z0.h, z0.h, #5 // =0x5
; SVE2p1-NEXT: add z1.h, z1.h, #5 // =0x5
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: add z2.h, z2.h, #5 // =0x5
-; SVE2p1-NEXT: add z3.h, z3.h, #5 // =0x5
; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
-; SVE2p1-NEXT: str z2, [x0]
-; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
+; SVE2p1-NEXT: add z3.h, z3.h, #5 // =0x5
+; SVE2p1-NEXT: str z1, [x0]
+; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i16_r:
@@ -752,20 +758,22 @@ define void @load_store_4x_vectors_i16_r(ptr %addr) {
define void @load_store_4x_vectors_i16_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_i16_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: ptrue p0.h
; SVE2p1-NEXT: add x8, x0, x1, lsl #1
+; SVE2p1-NEXT: ptrue p0.h
+; SVE2p1-NEXT: mov x9, x8
; SVE2p1-NEXT: ld1h { z0.h }, p0/z, [x0, x1, lsl #1]
; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: incb x9, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x8, #1, mul vl]
; SVE2p1-NEXT: add z0.h, z0.h, #5 // =0x5
; SVE2p1-NEXT: add z1.h, z1.h, #5 // =0x5
+; SVE2p1-NEXT: ldr z3, [x9, #1, mul vl]
; SVE2p1-NEXT: add z2.h, z2.h, #5 // =0x5
-; SVE2p1-NEXT: add z3.h, z3.h, #5 // =0x5
; SVE2p1-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1]
+; SVE2p1-NEXT: add z3.h, z3.h, #5 // =0x5
; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
-; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z2, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z3, [x9, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i16_rr:
@@ -801,18 +809,20 @@ define void @load_store_4x_vectors_i16_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_i32_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_i32_r:
; SVE2p1: // %bb.0:
+; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
-; SVE2p1-NEXT: ldr z2, [x0]
-; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0]
+; SVE2p1-NEXT: incb x8, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
; SVE2p1-NEXT: add z0.s, z0.s, #5 // =0x5
; SVE2p1-NEXT: add z1.s, z1.s, #5 // =0x5
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: add z2.s, z2.s, #5 // =0x5
-; SVE2p1-NEXT: add z3.s, z3.s, #5 // =0x5
; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
-; SVE2p1-NEXT: str z2, [x0]
-; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
+; SVE2p1-NEXT: add z3.s, z3.s, #5 // =0x5
+; SVE2p1-NEXT: str z1, [x0]
+; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i32_r:
@@ -845,20 +855,22 @@ define void @load_store_4x_vectors_i32_r(ptr %addr) {
define void @load_store_4x_vectors_i32_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_i32_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: ptrue p0.s
; SVE2p1-NEXT: add x8, x0, x1, lsl #2
+; SVE2p1-NEXT: ptrue p0.s
+; SVE2p1-NEXT: mov x9, x8
; SVE2p1-NEXT: ld1w { z0.s }, p0/z, [x0, x1, lsl #2]
; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: incb x9, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x8, #1, mul vl]
; SVE2p1-NEXT: add z0.s, z0.s, #5 // =0x5
; SVE2p1-NEXT: add z1.s, z1.s, #5 // =0x5
+; SVE2p1-NEXT: ldr z3, [x9, #1, mul vl]
; SVE2p1-NEXT: add z2.s, z2.s, #5 // =0x5
-; SVE2p1-NEXT: add z3.s, z3.s, #5 // =0x5
; SVE2p1-NEXT: st1w { z0.s }, p0, [x0, x1, lsl #2]
+; SVE2p1-NEXT: add z3.s, z3.s, #5 // =0x5
; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
-; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z2, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z3, [x9, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i32_rr:
@@ -894,18 +906,20 @@ define void @load_store_4x_vectors_i32_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_i64_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_i64_r:
; SVE2p1: // %bb.0:
+; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
-; SVE2p1-NEXT: ldr z2, [x0]
-; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0]
+; SVE2p1-NEXT: incb x8, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
; SVE2p1-NEXT: add z0.d, z0.d, #5 // =0x5
; SVE2p1-NEXT: add z1.d, z1.d, #5 // =0x5
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: add z2.d, z2.d, #5 // =0x5
-; SVE2p1-NEXT: add z3.d, z3.d, #5 // =0x5
; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
-; SVE2p1-NEXT: str z2, [x0]
-; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
+; SVE2p1-NEXT: add z3.d, z3.d, #5 // =0x5
+; SVE2p1-NEXT: str z1, [x0]
+; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i64_r:
@@ -938,20 +952,22 @@ define void @load_store_4x_vectors_i64_r(ptr %addr) {
define void @load_store_4x_vectors_i64_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_i64_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: ptrue p0.d
; SVE2p1-NEXT: add x8, x0, x1, lsl #3
+; SVE2p1-NEXT: ptrue p0.d
+; SVE2p1-NEXT: mov x9, x8
; SVE2p1-NEXT: ld1d { z0.d }, p0/z, [x0, x1, lsl #3]
; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: incb x9, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x8, #1, mul vl]
; SVE2p1-NEXT: add z0.d, z0.d, #5 // =0x5
; SVE2p1-NEXT: add z1.d, z1.d, #5 // =0x5
+; SVE2p1-NEXT: ldr z3, [x9, #1, mul vl]
; SVE2p1-NEXT: add z2.d, z2.d, #5 // =0x5
-; SVE2p1-NEXT: add z3.d, z3.d, #5 // =0x5
; SVE2p1-NEXT: st1d { z0.d }, p0, [x0, x1, lsl #3]
+; SVE2p1-NEXT: add z3.d, z3.d, #5 // =0x5
; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
-; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z2, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z3, [x9, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i64_rr:
@@ -987,19 +1003,21 @@ define void @load_store_4x_vectors_i64_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_f16_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_f16_r:
; SVE2p1: // %bb.0:
+; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0]
+; SVE2p1-NEXT: incb x8, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
; SVE2p1-NEXT: ptrue p0.h
-; SVE2p1-NEXT: ldr z2, [x0]
-; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
; SVE2p1-NEXT: fadd z0.h, p0/m, z0.h, #1.0
; SVE2p1-NEXT: fadd z1.h, p0/m, z1.h, #1.0
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: fadd z2.h, p0/m, z2.h, #1.0
; SVE2p1-NEXT: fadd z3.h, p0/m, z3.h, #1.0
; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
-; SVE2p1-NEXT: str z2, [x0]
-; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
+; SVE2p1-NEXT: str z1, [x0]
+; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_f16_r:
@@ -1034,20 +1052,22 @@ define void @load_store_4x_vectors_f16_r(ptr %addr) {
define void @load_store_4x_vectors_f16_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_f16_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: ptrue p0.h
; SVE2p1-NEXT: add x8, x0, x1, lsl #1
+; SVE2p1-NEXT: ptrue p0.h
+; SVE2p1-NEXT: mov x9, x8
; SVE2p1-NEXT: ld1h { z0.h }, p0/z, [x0, x1, lsl #1]
; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: incb x9, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x8, #1, mul vl]
; SVE2p1-NEXT: fadd z0.h, p0/m, z0.h, #1.0
; SVE2p1-NEXT: fadd z1.h, p0/m, z1.h, #1.0
+; SVE2p1-NEXT: ldr z3, [x9, #1, mul vl]
; SVE2p1-NEXT: fadd z2.h, p0/m, z2.h, #1.0
; SVE2p1-NEXT: fadd z3.h, p0/m, z3.h, #1.0
; SVE2p1-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1]
; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
-; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z2, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z3, [x9, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_f16_rr:
@@ -1085,19 +1105,21 @@ define void @load_store_4x_vectors_f16_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_f32_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_f32_r:
; SVE2p1: // %bb.0:
+; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0]
+; SVE2p1-NEXT: incb x8, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
; SVE2p1-NEXT: ptrue p0.s
-; SVE2p1-NEXT: ldr z2, [x0]
-; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
; SVE2p1-NEXT: fadd z0.s, p0/m, z0.s, #1.0
; SVE2p1-NEXT: fadd z1.s, p0/m, z1.s, #1.0
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: fadd z2.s, p0/m, z2.s, #1.0
; SVE2p1-NEXT: fadd z3.s, p0/m, z3.s, #1.0
; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
-; SVE2p1-NEXT: str z2, [x0]
-; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
+; SVE2p1-NEXT: str z1, [x0]
+; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_f32_r:
@@ -1132,20 +1154,22 @@ define void @load_store_4x_vectors_f32_r(ptr %addr) {
define void @load_store_4x_vectors_f32_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_f32_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: ptrue p0.s
; SVE2p1-NEXT: add x8, x0, x1, lsl #2
+; SVE2p1-NEXT: ptrue p0.s
+; SVE2p1-NEXT: mov x9, x8
; SVE2p1-NEXT: ld1w { z0.s }, p0/z, [x0, x1, lsl #2]
; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: incb x9, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x8, #1, mul vl]
; SVE2p1-NEXT: fadd z0.s, p0/m, z0.s, #1.0
; SVE2p1-NEXT: fadd z1.s, p0/m, z1.s, #1.0
+; SVE2p1-NEXT: ldr z3, [x9, #1, mul vl]
; SVE2p1-NEXT: fadd z2.s, p0/m, z2.s, #1.0
; SVE2p1-NEXT: fadd z3.s, p0/m, z3.s, #1.0
; SVE2p1-NEXT: st1w { z0.s }, p0, [x0, x1, lsl #2]
; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
-; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z2, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z3, [x9, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_f32_rr:
@@ -1183,19 +1207,21 @@ define void @load_store_4x_vectors_f32_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_f64_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_f64_r:
; SVE2p1: // %bb.0:
+; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0]
+; SVE2p1-NEXT: incb x8, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
; SVE2p1-NEXT: ptrue p0.d
-; SVE2p1-NEXT: ldr z2, [x0]
-; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
; SVE2p1-NEXT: fadd z0.d, p0/m, z0.d, #1.0
; SVE2p1-NEXT: fadd z1.d, p0/m, z1.d, #1.0
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: fadd z2.d, p0/m, z2.d, #1.0
; SVE2p1-NEXT: fadd z3.d, p0/m, z3.d, #1.0
; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
-; SVE2p1-NEXT: str z2, [x0]
-; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
+; SVE2p1-NEXT: str z1, [x0]
+; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_f64_r:
@@ -1230,20 +1256,22 @@ define void @load_store_4x_vectors_f64_r(ptr %addr) {
define void @load_store_4x_vectors_f64_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_f64_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: ptrue p0.d
; SVE2p1-NEXT: add x8, x0, x1, lsl #3
+; SVE2p1-NEXT: ptrue p0.d
+; SVE2p1-NEXT: mov x9, x8
; SVE2p1-NEXT: ld1d { z0.d }, p0/z, [x0, x1, lsl #3]
; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: incb x9, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x8, #1, mul vl]
; SVE2p1-NEXT: fadd z0.d, p0/m, z0.d, #1.0
; SVE2p1-NEXT: fadd z1.d, p0/m, z1.d, #1.0
+; SVE2p1-NEXT: ldr z3, [x9, #1, mul vl]
; SVE2p1-NEXT: fadd z2.d, p0/m, z2.d, #1.0
; SVE2p1-NEXT: fadd z3.d, p0/m, z3.d, #1.0
; SVE2p1-NEXT: st1d { z0.d }, p0, [x0, x1, lsl #3]
; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
-; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z2, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z3, [x9, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_f64_rr:
@@ -1281,19 +1309,21 @@ define void @load_store_4x_vectors_f64_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_bf16_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_bf16_r:
; SVE2p1: // %bb.0:
+; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: fmov z0.h, #1.87500000
; SVE2p1-NEXT: ldr z1, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z2, [x0, #3, mul vl]
-; SVE2p1-NEXT: ldr z3, [x0]
-; SVE2p1-NEXT: ldr z4, [x0, #1, mul vl]
+; SVE2p1-NEXT: incb x8, all, mul #2
+; SVE2p1-NEXT: ldr z2, [x0]
+; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
; SVE2p1-NEXT: bfadd z1.h, z1.h, z0.h
+; SVE2p1-NEXT: ldr z4, [x8, #1, mul vl]
; SVE2p1-NEXT: bfadd z2.h, z2.h, z0.h
; SVE2p1-NEXT: str z1, [x0, #2, mul vl]
; SVE2p1-NEXT: bfadd z1.h, z3.h, z0.h
+; SVE2p1-NEXT: str z2, [x0]
+; SVE2p1-NEXT: str z1, [x0, #1, mul vl]
; SVE2p1-NEXT: bfadd z0.h, z4.h, z0.h
-; SVE2p1-NEXT: str z2, [x0, #3, mul vl]
-; SVE2p1-NEXT: str z1, [x0]
-; SVE2p1-NEXT: str z0, [x0, #1, mul vl]
+; SVE2p1-NEXT: str z0, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_bf16_r:
@@ -1328,21 +1358,23 @@ define void @load_store_4x_vectors_bf16_r(ptr %addr) {
define void @load_store_4x_vectors_bf16_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_bf16_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: ptrue p0.h
; SVE2p1-NEXT: add x8, x0, x1, lsl #1
-; SVE2p1-NEXT: fmov z0.h, #1.87500000
-; SVE2p1-NEXT: ld1h { z1.h }, p0/z, [x0, x1, lsl #1]
+; SVE2p1-NEXT: ptrue p0.h
+; SVE2p1-NEXT: fmov z1.h, #1.87500000
+; SVE2p1-NEXT: mov x9, x8
+; SVE2p1-NEXT: ld1h { z0.h }, p0/z, [x0, x1, lsl #1]
; SVE2p1-NEXT: ldr z2, [x8, #2, mul vl]
-; SVE2p1-NEXT: ldr z3, [x8, #3, mul vl]
-; SVE2p1-NEXT: ldr z4, [x8, #1, mul vl]
-; SVE2p1-NEXT: bfadd z1.h, z1.h, z0.h
-; SVE2p1-NEXT: bfadd z2.h, z2.h, z0.h
-; SVE2p1-NEXT: st1h { z1.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-NEXT: bfadd z1.h, z3.h, z0.h
-; SVE2p1-NEXT: bfadd z0.h, z4.h, z0.h
+; SVE2p1-NEXT: incb x9, all, mul #2
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: bfadd z0.h, z0.h, z1.h
+; SVE2p1-NEXT: bfadd z2.h, z2.h, z1.h
+; SVE2p1-NEXT: ldr z4, [x9, #1, mul vl]
+; SVE2p1-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1]
+; SVE2p1-NEXT: bfadd z0.h, z3.h, z1.h
; SVE2p1-NEXT: str z2, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z1, [x8, #3, mul vl]
; SVE2p1-NEXT: str z0, [x8, #1, mul vl]
+; SVE2p1-NEXT: bfadd z1.h, z4.h, z1.h
+; SVE2p1-NEXT: str z1, [x9, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_bf16_rr:
diff --git a/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll b/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll
index 0bc8cb8bc5003..d7ef9b520373d 100644
--- a/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll
+++ b/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll
@@ -91,16 +91,18 @@ define i64 @split_extract_8i64_idx(<vscale x 8 x i64> %a, i32 %idx) {
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: mov w10, w0
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
; CHECK-NEXT: cnth x8
-; CHECK-NEXT: mov w9, w0
-; CHECK-NEXT: str z3, [sp, #3, mul vl]
; CHECK-NEXT: sub x8, x8, #1
; CHECK-NEXT: str z2, [sp, #2, mul vl]
-; CHECK-NEXT: cmp x9, x8
+; CHECK-NEXT: cmp x10, x8
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: csel x8, x9, x8, lo
-; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: csel x8, x10, x8, lo
; CHECK-NEXT: ldr x0, [x9, x8, lsl #3]
; CHECK-NEXT: addvl sp, sp, #4
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -159,17 +161,19 @@ define i32 @split_extract_16i32(<vscale x 16 x i32> %a) {
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: mov w9, #34464 // =0x86a0
-; CHECK-NEXT: str z3, [sp, #3, mul vl]
-; CHECK-NEXT: movk w9, #1, lsl #16
-; CHECK-NEXT: sub x8, x8, #1
+; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: rdvl x10, #1
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: sub x10, x10, #1
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
+; CHECK-NEXT: mov w8, #34464 // =0x86a0
+; CHECK-NEXT: movk w8, #1, lsl #16
; CHECK-NEXT: str z2, [sp, #2, mul vl]
-; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cmp x10, x8
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: csel x8, x8, x9, lo
-; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: csel x8, x10, x8, lo
; CHECK-NEXT: ldr w0, [x9, x8, lsl #2]
; CHECK-NEXT: addvl sp, sp, #4
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll b/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll
index 6116473d9588e..3fccd94bdb508 100644
--- a/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll
+++ b/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll
@@ -71,20 +71,22 @@ define <vscale x 8 x i64> @split_insert_8i64_idx(<vscale x 8 x i64> %a, i64 %elt
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: cnth x8
+; CHECK-NEXT: cnth x10
+; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: str z3, [sp, #3, mul vl]
-; CHECK-NEXT: sub x8, x8, #1
+; CHECK-NEXT: sub x10, x10, #1
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: cmp x1, x10
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
+; CHECK-NEXT: csel x10, x1, x10, lo
; CHECK-NEXT: str z2, [sp, #2, mul vl]
-; CHECK-NEXT: cmp x1, x8
; CHECK-NEXT: str z1, [sp, #1, mul vl]
-; CHECK-NEXT: csel x8, x1, x8, lo
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: str x0, [x9, x8, lsl #3]
+; CHECK-NEXT: str x0, [x9, x10, lsl #3]
; CHECK-NEXT: ldr z0, [sp]
+; CHECK-NEXT: ldr z3, [x8, #1, mul vl]
; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
-; CHECK-NEXT: ldr z3, [sp, #3, mul vl]
; CHECK-NEXT: addvl sp, sp, #4
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
@@ -133,20 +135,21 @@ define <vscale x 32 x i16> @split_insert_32i16(<vscale x 32 x i16> %a, i16 %elt)
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: mov w9, #128 // =0x80
-; CHECK-NEXT: str z3, [sp, #3, mul vl]
+; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: mov w11, #128 // =0x80
+; CHECK-NEXT: add x10, x9, x8
; CHECK-NEXT: sub x8, x8, #1
-; CHECK-NEXT: str z2, [sp, #2, mul vl]
; CHECK-NEXT: cmp x8, #128
+; CHECK-NEXT: str z3, [x10, #1, mul vl]
+; CHECK-NEXT: csel x8, x8, x11, lo
+; CHECK-NEXT: str z2, [sp, #2, mul vl]
; CHECK-NEXT: str z1, [sp, #1, mul vl]
-; CHECK-NEXT: csel x8, x8, x9, lo
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: str z0, [sp]
; CHECK-NEXT: strh w0, [x9, x8, lsl #1]
; CHECK-NEXT: ldr z0, [sp]
+; CHECK-NEXT: ldr z3, [x10, #1, mul vl]
; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
-; CHECK-NEXT: ldr z3, [sp, #3, mul vl]
; CHECK-NEXT: addvl sp, sp, #4
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-split-load.ll b/llvm/test/CodeGen/AArch64/sve-split-load.ll
index e1dd66c9d249a..c14d6d10ad49c 100644
--- a/llvm/test/CodeGen/AArch64/sve-split-load.ll
+++ b/llvm/test/CodeGen/AArch64/sve-split-load.ll
@@ -26,9 +26,11 @@ define <vscale x 16 x i16> @load_split_i16(ptr %a) {
define <vscale x 24 x i16> @load_split_24i16(ptr %a) {
; CHECK-LABEL: load_split_24i16:
; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
-; CHECK-NEXT: ldr z2, [x0, #2, mul vl]
+; CHECK-NEXT: add x8, x0, x8
+; CHECK-NEXT: ldr z2, [x8, #1, mul vl]
; CHECK-NEXT: ret
%load = load <vscale x 24 x i16>, ptr %a
ret <vscale x 24 x i16> %load
@@ -37,10 +39,12 @@ define <vscale x 24 x i16> @load_split_24i16(ptr %a) {
define <vscale x 32 x i16> @load_split_32i16(ptr %a) {
; CHECK-LABEL: load_split_32i16:
; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ldr z2, [x0, #2, mul vl]
-; CHECK-NEXT: ldr z3, [x0, #3, mul vl]
+; CHECK-NEXT: ldr z3, [x8, #1, mul vl]
; CHECK-NEXT: ret
%load = load <vscale x 32 x i16>, ptr %a
ret <vscale x 32 x i16> %load
@@ -49,14 +53,19 @@ define <vscale x 32 x i16> @load_split_32i16(ptr %a) {
define <vscale x 16 x i64> @load_split_16i64(ptr %a) {
; CHECK-LABEL: load_split_16i64:
; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
+; CHECK-NEXT: add x9, x0, x8
; CHECK-NEXT: ldr z2, [x0, #2, mul vl]
-; CHECK-NEXT: ldr z3, [x0, #3, mul vl]
; CHECK-NEXT: ldr z4, [x0, #4, mul vl]
-; CHECK-NEXT: ldr z5, [x0, #5, mul vl]
-; CHECK-NEXT: ldr z6, [x0, #6, mul vl]
-; CHECK-NEXT: ldr z7, [x0, #7, mul vl]
+; CHECK-NEXT: ldr z3, [x9, #1, mul vl]
+; CHECK-NEXT: rdvl x9, #4
+; CHECK-NEXT: add x9, x0, x9
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: ldr z5, [x9, #1, mul vl]
+; CHECK-NEXT: ldr z6, [x9, #2, mul vl]
+; CHECK-NEXT: ldr z7, [x8, #1, mul vl]
; CHECK-NEXT: ret
%load = load <vscale x 16 x i64>, ptr %a
ret <vscale x 16 x i64> %load
@@ -87,13 +96,15 @@ define <vscale x 32 x i16> @masked_load_split_32i16(ptr %a, <vscale x 32 x i1> %
; CHECK-LABEL: masked_load_split_32i16:
; CHECK: // %bb.0:
; CHECK-NEXT: punpklo p2.h, p0.b
+; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: punpkhi p0.h, p0.b
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: punpklo p3.h, p1.b
; CHECK-NEXT: ld1h { z0.h }, p2/z, [x0]
; CHECK-NEXT: punpkhi p1.h, p1.b
; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1h { z2.h }, p3/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1h { z3.h }, p1/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1h { z3.h }, p1/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%load = call <vscale x 32 x i16> @llvm.masked.load.nxv32i16(ptr %a, i32 1, <vscale x 32 x i1> %pg, <vscale x 32 x i16> poison)
ret <vscale x 32 x i16> %load
@@ -115,7 +126,9 @@ define <vscale x 8 x i64> @masked_load_split_8i64(ptr %a, <vscale x 8 x i1> %pg)
; CHECK-LABEL: masked_load_split_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: punpklo p1.h, p0.b
+; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: punpkhi p0.h, p0.b
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: punpklo p2.h, p1.b
; CHECK-NEXT: punpkhi p1.h, p1.b
; CHECK-NEXT: punpklo p3.h, p0.b
@@ -123,7 +136,7 @@ define <vscale x 8 x i64> @masked_load_split_8i64(ptr %a, <vscale x 8 x i1> %pg)
; CHECK-NEXT: punpkhi p0.h, p0.b
; CHECK-NEXT: ld1d { z1.d }, p1/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1d { z3.d }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ret
%load = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64(ptr %a, i32 1, <vscale x 8 x i1> %pg, <vscale x 8 x i64> poison)
ret <vscale x 8 x i64> %load
diff --git a/llvm/test/CodeGen/AArch64/sve-split-store.ll b/llvm/test/CodeGen/AArch64/sve-split-store.ll
index b1419b3f679cf..774c2058253c8 100644
--- a/llvm/test/CodeGen/AArch64/sve-split-store.ll
+++ b/llvm/test/CodeGen/AArch64/sve-split-store.ll
@@ -26,10 +26,12 @@ define void @store_split_i16(<vscale x 16 x i16> %data, ptr %a) {
define void @store_split_16i32(<vscale x 16 x i32> %data, ptr %a) {
; CHECK-LABEL: store_split_16i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: str z3, [x0, #3, mul vl]
+; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: str z2, [x0, #2, mul vl]
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: str z1, [x0, #1, mul vl]
; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: str z3, [x8, #1, mul vl]
; CHECK-NEXT: ret
store <vscale x 16 x i32> %data, ptr %a
ret void
@@ -38,14 +40,19 @@ define void @store_split_16i32(<vscale x 16 x i32> %data, ptr %a) {
define void @store_split_16i64(<vscale x 16 x i64> %data, ptr %a) {
; CHECK-LABEL: store_split_16i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: str z7, [x0, #7, mul vl]
-; CHECK-NEXT: str z6, [x0, #6, mul vl]
-; CHECK-NEXT: str z5, [x0, #5, mul vl]
+; CHECK-NEXT: rdvl x8, #4
+; CHECK-NEXT: rdvl x9, #2
; CHECK-NEXT: str z4, [x0, #4, mul vl]
-; CHECK-NEXT: str z3, [x0, #3, mul vl]
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: str z2, [x0, #2, mul vl]
+; CHECK-NEXT: add x10, x0, x9
; CHECK-NEXT: str z1, [x0, #1, mul vl]
; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: str z6, [x8, #2, mul vl]
+; CHECK-NEXT: str z5, [x8, #1, mul vl]
+; CHECK-NEXT: add x8, x8, x9
+; CHECK-NEXT: str z3, [x10, #1, mul vl]
+; CHECK-NEXT: str z7, [x8, #1, mul vl]
; CHECK-NEXT: ret
store <vscale x 16 x i64> %data, ptr %a
ret void
@@ -75,13 +82,15 @@ define void @masked_store_split_32i8(<vscale x 32 x i8> %data, ptr %a, <vscale x
define void @masked_store_split_32i16(<vscale x 32 x i16> %data, ptr %a, <vscale x 32 x i1> %pg) {
; CHECK-LABEL: masked_store_split_32i16:
; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: punpkhi p2.h, p1.b
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: punpklo p1.h, p1.b
-; CHECK-NEXT: punpkhi p3.h, p0.b
-; CHECK-NEXT: st1h { z3.h }, p2, [x0, #3, mul vl]
+; CHECK-NEXT: st1h { z3.h }, p2, [x8, #1, mul vl]
+; CHECK-NEXT: punpkhi p2.h, p0.b
; CHECK-NEXT: punpklo p0.h, p0.b
; CHECK-NEXT: st1h { z2.h }, p1, [x0, #2, mul vl]
-; CHECK-NEXT: st1h { z1.h }, p3, [x0, #1, mul vl]
+; CHECK-NEXT: st1h { z1.h }, p2, [x0, #1, mul vl]
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
call void @llvm.masked.store.nxv32i16(<vscale x 32 x i16> %data, ptr %a, i32 1, <vscale x 32 x i1> %pg)
@@ -104,10 +113,12 @@ define void @masked_store_split_8i64(<vscale x 8 x i64> %data, ptr %a, <vscale x
; CHECK-LABEL: masked_store_split_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: punpkhi p1.h, p0.b
+; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: punpklo p0.h, p0.b
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: punpkhi p2.h, p1.b
; CHECK-NEXT: punpklo p1.h, p1.b
-; CHECK-NEXT: st1d { z3.d }, p2, [x0, #3, mul vl]
+; CHECK-NEXT: st1d { z3.d }, p2, [x8, #1, mul vl]
; CHECK-NEXT: punpkhi p2.h, p0.b
; CHECK-NEXT: punpklo p0.h, p0.b
; CHECK-NEXT: st1d { z2.d }, p1, [x0, #2, mul vl]
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave-widen.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave-widen.ll
index d67488981bbcd..a198abc0093c5 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-interleave-widen.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave-widen.ll
@@ -116,65 +116,70 @@ define void @interleave4_v3f32(ptr %dst, <vscale x 3 x float> %a, <vscale x 3
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x98, 0x01, 0x1e, 0x22 // sp + 16 + 152 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: mov w9, #3 // =0x3
; CHECK-NEXT: zip2 z4.s, z1.s, z3.s
-; CHECK-NEXT: lsr x8, x8, #4
; CHECK-NEXT: zip2 z5.s, z0.s, z2.s
+; CHECK-NEXT: lsr x9, x8, #4
+; CHECK-NEXT: mov w11, #3 // =0x3
; CHECK-NEXT: zip1 z1.s, z1.s, z3.s
; CHECK-NEXT: zip1 z0.s, z0.s, z2.s
-; CHECK-NEXT: mul x8, x8, x9
-; CHECK-NEXT: cntw x9, all, mul #3
+; CHECK-NEXT: rdvl x10, #2
+; CHECK-NEXT: addvl x12, sp, #9
+; CHECK-NEXT: mul x9, x9, x11
; CHECK-NEXT: zip2 z2.s, z5.s, z4.s
; CHECK-NEXT: zip1 z3.s, z5.s, z4.s
+; CHECK-NEXT: add x11, x12, x10
+; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: zip2 z4.s, z0.s, z1.s
; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
-; CHECK-NEXT: str z2, [sp, #12, mul vl]
-; CHECK-NEXT: whilelo p0.s, xzr, x8
-; CHECK-NEXT: addvl x8, sp, #9
+; CHECK-NEXT: str z2, [x11, #1, mul vl]
+; CHECK-NEXT: cntw x11, all, mul #3
; CHECK-NEXT: str z3, [sp, #11, mul vl]
+; CHECK-NEXT: whilelo p0.s, xzr, x9
; CHECK-NEXT: str z4, [sp, #10, mul vl]
-; CHECK-NEXT: add x8, x8, x9
+; CHECK-NEXT: add x9, x12, x11
; CHECK-NEXT: str z0, [sp, #9, mul vl]
+; CHECK-NEXT: addvl x12, sp, #5
; CHECK-NEXT: str z0, [sp, #13, mul vl]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x8]
-; CHECK-NEXT: addvl x8, sp, #13
-; CHECK-NEXT: add x8, x8, x9
-; CHECK-NEXT: addvl x9, sp, #5
-; CHECK-NEXT: st1w { z1.s }, p0, [x8]
-; CHECK-NEXT: cnth x8, all, mul #3
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x9]
+; CHECK-NEXT: addvl x9, sp, #13
+; CHECK-NEXT: add x9, x9, x11
+; CHECK-NEXT: add x11, x12, x10
+; CHECK-NEXT: st1w { z1.s }, p0, [x9]
+; CHECK-NEXT: cnth x9, all, mul #3
+; CHECK-NEXT: str z2, [x11, #1, mul vl]
+; CHECK-NEXT: add x11, x12, x9
+; CHECK-NEXT: addvl x12, sp, #1
; CHECK-NEXT: ldr z1, [sp, #14, mul vl]
-; CHECK-NEXT: str z2, [sp, #8, mul vl]
-; CHECK-NEXT: add x9, x9, x8
; CHECK-NEXT: str z3, [sp, #7, mul vl]
+; CHECK-NEXT: add x10, x12, x10
; CHECK-NEXT: str z4, [sp, #6, mul vl]
+; CHECK-NEXT: str z0, [sp, #5, mul vl]
; CHECK-NEXT: str z1, [sp, #16, mul vl]
; CHECK-NEXT: ldr z1, [sp, #13, mul vl]
-; CHECK-NEXT: str z0, [sp, #5, mul vl]
; CHECK-NEXT: str z1, [sp, #15, mul vl]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x9]
-; CHECK-NEXT: addvl x9, sp, #15
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: addvl x9, sp, #1
-; CHECK-NEXT: st1w { z1.s }, p0, [x8]
-; CHECK-NEXT: cntw x8, all, mul #9
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x11]
+; CHECK-NEXT: addvl x11, sp, #15
+; CHECK-NEXT: add x9, x11, x9
+; CHECK-NEXT: st1w { z1.s }, p0, [x9]
+; CHECK-NEXT: cntw x9, all, mul #9
+; CHECK-NEXT: str z2, [x10, #1, mul vl]
+; CHECK-NEXT: add x9, x12, x9
+; CHECK-NEXT: mov x10, sp
; CHECK-NEXT: str z0, [sp, #1, mul vl]
; CHECK-NEXT: ldr z0, [sp, #17, mul vl]
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: str z2, [sp, #4, mul vl]
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: str z3, [sp, #3, mul vl]
; CHECK-NEXT: str z4, [sp, #2, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8]
-; CHECK-NEXT: cntw x8
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: st1w { z0.s }, p0, [x8]
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x9]
+; CHECK-NEXT: cntw x9
+; CHECK-NEXT: add x9, x10, x9
+; CHECK-NEXT: st1w { z0.s }, p0, [x9]
; CHECK-NEXT: ldr z0, [sp, #16, mul vl]
; CHECK-NEXT: ldr z1, [sp, #15, mul vl]
; CHECK-NEXT: ldr z2, [sp]
; CHECK-NEXT: str z0, [x0, #1, mul vl]
; CHECK-NEXT: str z1, [x0]
-; CHECK-NEXT: str z2, [x0, #2, mul vl]
+; CHECK-NEXT: str z2, [x8, #1, mul vl]
; CHECK-NEXT: addvl sp, sp, #19
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
@@ -193,109 +198,122 @@ define void @interleave6_v3fp32(ptr %dst, <vscale x 3 x float> %a, <vscale x 3
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xa8, 0x03, 0x1e, 0x22 // sp + 16 + 424 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: zip2 z26.s, z2.s, z5.s
-; CHECK-NEXT: zip1 z7.s, z2.s, z5.s
-; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: rdvl x8, #4
; CHECK-NEXT: zip2 z25.s, z1.s, z4.s
-; CHECK-NEXT: zip1 z6.s, z1.s, z4.s
-; CHECK-NEXT: lsr x8, x8, #4
+; CHECK-NEXT: zip1 z7.s, z2.s, z5.s
+; CHECK-NEXT: addvl x11, sp, #31
; CHECK-NEXT: zip2 z24.s, z0.s, z3.s
+; CHECK-NEXT: zip1 z6.s, z1.s, z4.s
+; CHECK-NEXT: add x9, x11, x8
; CHECK-NEXT: zip1 z5.s, z0.s, z3.s
-; CHECK-NEXT: mov w9, #3 // =0x3
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: mul x8, x8, x9
-; CHECK-NEXT: addvl x9, sp, #31
-; CHECK-NEXT: rdvl x10, #4
+; CHECK-NEXT: rdvl x10, #2
+; CHECK-NEXT: mov w13, #3 // =0x3
; CHECK-NEXT: ptrue p1.d
; CHECK-NEXT: st3w { z24.s - z26.s }, p0, [sp, #3, mul vl]
-; CHECK-NEXT: ldr z1, [sp, #5, mul vl]
-; CHECK-NEXT: ldr z2, [sp, #4, mul vl]
-; CHECK-NEXT: ldr z0, [sp, #3, mul vl]
+; CHECK-NEXT: ldr z0, [sp, #5, mul vl]
; CHECK-NEXT: st3w { z5.s - z7.s }, p0, [sp]
-; CHECK-NEXT: whilelo p0.s, xzr, x8
-; CHECK-NEXT: cntw x8, all, mul #3
+; CHECK-NEXT: str z0, [x9, #1, mul vl]
+; CHECK-NEXT: add x9, x11, x10
+; CHECK-NEXT: ldr z1, [sp, #3, mul vl]
+; CHECK-NEXT: str z1, [x9, #1, mul vl]
+; CHECK-NEXT: rdvl x9, #1
+; CHECK-NEXT: lsr x12, x9, #4
+; CHECK-NEXT: ldr z2, [sp, #4, mul vl]
; CHECK-NEXT: ldr z3, [sp, #2, mul vl]
; CHECK-NEXT: ldr z4, [sp, #1, mul vl]
; CHECK-NEXT: ldr z5, [sp]
-; CHECK-NEXT: add x9, x9, x8
-; CHECK-NEXT: str z1, [sp, #36, mul vl]
+; CHECK-NEXT: mul x12, x12, x13
+; CHECK-NEXT: addvl x13, sp, #15
; CHECK-NEXT: str z2, [sp, #35, mul vl]
-; CHECK-NEXT: str z0, [sp, #34, mul vl]
; CHECK-NEXT: str z3, [sp, #33, mul vl]
; CHECK-NEXT: str z4, [sp, #32, mul vl]
; CHECK-NEXT: str z5, [sp, #31, mul vl]
+; CHECK-NEXT: whilelo p0.s, xzr, x12
+; CHECK-NEXT: cntw x12, all, mul #3
; CHECK-NEXT: str z5, [sp, #39, mul vl]
-; CHECK-NEXT: ld1w { z6.s }, p0/z, [x9]
-; CHECK-NEXT: addvl x9, sp, #31
-; CHECK-NEXT: addvl x9, x9, #8
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: addvl x9, sp, #23
-; CHECK-NEXT: st1w { z6.s }, p0, [x8]
-; CHECK-NEXT: cnth x8, all, mul #3
+; CHECK-NEXT: add x11, x11, x12
+; CHECK-NEXT: ld1w { z6.s }, p0/z, [x11]
+; CHECK-NEXT: addvl x11, sp, #31
+; CHECK-NEXT: addvl x11, x11, #8
+; CHECK-NEXT: add x11, x11, x12
+; CHECK-NEXT: addvl x12, sp, #23
+; CHECK-NEXT: st1w { z6.s }, p0, [x11]
+; CHECK-NEXT: add x11, x12, x8
+; CHECK-NEXT: str z0, [x11, #1, mul vl]
+; CHECK-NEXT: add x11, x12, x10
+; CHECK-NEXT: str z1, [x11, #1, mul vl]
+; CHECK-NEXT: cnth x11, all, mul #3
; CHECK-NEXT: ldr z6, [sp, #40, mul vl]
-; CHECK-NEXT: str z1, [sp, #28, mul vl]
-; CHECK-NEXT: add x9, x9, x8
; CHECK-NEXT: str z2, [sp, #27, mul vl]
-; CHECK-NEXT: str z0, [sp, #26, mul vl]
-; CHECK-NEXT: str z6, [sp, #42, mul vl]
-; CHECK-NEXT: ldr z6, [sp, #39, mul vl]
+; CHECK-NEXT: add x12, x12, x11
; CHECK-NEXT: str z3, [sp, #25, mul vl]
; CHECK-NEXT: str z4, [sp, #24, mul vl]
+; CHECK-NEXT: str z6, [sp, #42, mul vl]
+; CHECK-NEXT: ldr z6, [sp, #39, mul vl]
; CHECK-NEXT: str z5, [sp, #23, mul vl]
; CHECK-NEXT: str z6, [sp, #41, mul vl]
-; CHECK-NEXT: ld1w { z6.s }, p0/z, [x9]
-; CHECK-NEXT: addvl x9, sp, #31
-; CHECK-NEXT: addvl x9, x9, #10
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: addvl x9, sp, #15
-; CHECK-NEXT: st1w { z6.s }, p0, [x8]
-; CHECK-NEXT: cntw x8, all, mul #9
+; CHECK-NEXT: ld1w { z6.s }, p0/z, [x12]
+; CHECK-NEXT: addvl x12, sp, #31
+; CHECK-NEXT: addvl x12, x12, #10
+; CHECK-NEXT: add x11, x12, x11
+; CHECK-NEXT: st1w { z6.s }, p0, [x11]
+; CHECK-NEXT: add x11, x13, x8
+; CHECK-NEXT: str z0, [x11, #1, mul vl]
+; CHECK-NEXT: add x11, x13, x10
+; CHECK-NEXT: str z1, [x11, #1, mul vl]
+; CHECK-NEXT: cntw x11, all, mul #9
; CHECK-NEXT: ldr z6, [sp, #43, mul vl]
-; CHECK-NEXT: str z1, [sp, #20, mul vl]
-; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: str z2, [sp, #19, mul vl]
-; CHECK-NEXT: addvl x9, sp, #6
-; CHECK-NEXT: str z0, [sp, #18, mul vl]
+; CHECK-NEXT: add x11, x13, x11
; CHECK-NEXT: str z3, [sp, #17, mul vl]
+; CHECK-NEXT: addvl x13, sp, #6
; CHECK-NEXT: str z4, [sp, #16, mul vl]
; CHECK-NEXT: str z5, [sp, #15, mul vl]
; CHECK-NEXT: str z6, [sp, #6, mul vl]
-; CHECK-NEXT: ld1w { z6.s }, p0/z, [x8]
-; CHECK-NEXT: cntw x8
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: addvl x9, sp, #7
-; CHECK-NEXT: st1w { z6.s }, p0, [x8]
-; CHECK-NEXT: cntw x8, all, mul #15
-; CHECK-NEXT: str z1, [sp, #12, mul vl]
-; CHECK-NEXT: ldr z1, [sp, #42, mul vl]
-; CHECK-NEXT: add x9, x9, x8
-; CHECK-NEXT: str z0, [sp, #10, mul vl]
+; CHECK-NEXT: ld1w { z6.s }, p0/z, [x11]
+; CHECK-NEXT: cntw x11
+; CHECK-NEXT: add x11, x13, x11
+; CHECK-NEXT: addvl x13, sp, #7
+; CHECK-NEXT: st1w { z6.s }, p0, [x11]
+; CHECK-NEXT: add x11, x13, x8
+; CHECK-NEXT: str z0, [x11, #1, mul vl]
+; CHECK-NEXT: add x11, x13, x10
+; CHECK-NEXT: str z1, [x11, #1, mul vl]
+; CHECK-NEXT: add x11, x12, x10
+; CHECK-NEXT: cntw x12, all, mul #15
+; CHECK-NEXT: ldr z0, [x11, #1, mul vl]
+; CHECK-NEXT: addvl x11, sp, #31
+; CHECK-NEXT: add x13, x13, x12
+; CHECK-NEXT: addvl x11, x11, #14
+; CHECK-NEXT: add x10, x11, x10
+; CHECK-NEXT: add x12, x11, x12
+; CHECK-NEXT: add x8, x11, x8
+; CHECK-NEXT: mov z0.s, p0/m, z1.s
+; CHECK-NEXT: add x11, x0, x9
+; CHECK-NEXT: str z0, [x10, #1, mul vl]
+; CHECK-NEXT: ldr z0, [sp, #42, mul vl]
; CHECK-NEXT: str z2, [sp, #11, mul vl]
-; CHECK-NEXT: str z1, [sp, #46, mul vl]
-; CHECK-NEXT: ldr z1, [sp, #41, mul vl]
; CHECK-NEXT: str z3, [sp, #9, mul vl]
; CHECK-NEXT: str z4, [sp, #8, mul vl]
-; CHECK-NEXT: str z1, [sp, #45, mul vl]
-; CHECK-NEXT: ldr z1, [sp, #44, mul vl]
+; CHECK-NEXT: str z0, [sp, #46, mul vl]
+; CHECK-NEXT: ldr z0, [sp, #41, mul vl]
; CHECK-NEXT: str z5, [sp, #7, mul vl]
-; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s
-; CHECK-NEXT: str z0, [sp, #48, mul vl]
+; CHECK-NEXT: str z0, [sp, #45, mul vl]
; CHECK-NEXT: ldr z0, [sp, #6, mul vl]
; CHECK-NEXT: str z0, [sp, #47, mul vl]
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x9]
-; CHECK-NEXT: addvl x9, sp, #31
-; CHECK-NEXT: addvl x9, x9, #14
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: st1w { z0.s }, p0, [x8]
-; CHECK-NEXT: add x8, x9, x10
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x13]
+; CHECK-NEXT: st1w { z0.s }, p0, [x12]
; CHECK-NEXT: ld1w { z0.d }, p1/z, [x8]
-; CHECK-NEXT: ldr z1, [sp, #48, mul vl]
+; CHECK-NEXT: ldr z1, [x10, #1, mul vl]
; CHECK-NEXT: ldr z2, [sp, #47, mul vl]
; CHECK-NEXT: ldr z3, [sp, #45, mul vl]
; CHECK-NEXT: ldr z4, [sp, #46, mul vl]
-; CHECK-NEXT: add x8, x0, x10
-; CHECK-NEXT: st1w { z0.d }, p1, [x8]
-; CHECK-NEXT: str z1, [x0, #3, mul vl]
-; CHECK-NEXT: str z2, [x0, #2, mul vl]
+; CHECK-NEXT: add x8, x11, x9
+; CHECK-NEXT: add x9, x8, x9
+; CHECK-NEXT: st1w { z0.d }, p1, [x9, #2, mul vl]
+; CHECK-NEXT: str z1, [x8, #1, mul vl]
+; CHECK-NEXT: str z2, [x11, #1, mul vl]
; CHECK-NEXT: str z4, [x0, #1, mul vl]
; CHECK-NEXT: str z3, [x0]
; CHECK-NEXT: addvl sp, sp, #31
@@ -321,164 +339,191 @@ define void @interleave8_v3fp32(ptr %dst, <vscale x 3 x float> %a, <vscale x 3
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: zip2 z24.s, z3.s, z7.s
; CHECK-NEXT: zip2 z25.s, z1.s, z5.s
-; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: addvl x12, sp, #31
; CHECK-NEXT: zip2 z26.s, z2.s, z6.s
; CHECK-NEXT: zip2 z27.s, z0.s, z4.s
-; CHECK-NEXT: lsr x8, x8, #4
-; CHECK-NEXT: zip1 z3.s, z3.s, z7.s
-; CHECK-NEXT: zip1 z5.s, z1.s, z5.s
-; CHECK-NEXT: mov w9, #3 // =0x3
-; CHECK-NEXT: zip1 z6.s, z2.s, z6.s
+; CHECK-NEXT: rdvl x9, #4
+; CHECK-NEXT: addvl x12, x12, #13
; CHECK-NEXT: zip1 z4.s, z0.s, z4.s
-; CHECK-NEXT: mul x9, x8, x9
+; CHECK-NEXT: rdvl x10, #2
; CHECK-NEXT: zip2 z28.s, z25.s, z24.s
+; CHECK-NEXT: zip1 z24.s, z25.s, z24.s
+; CHECK-NEXT: add x13, x12, x9
; CHECK-NEXT: zip2 z29.s, z27.s, z26.s
-; CHECK-NEXT: addvl x10, sp, #31
-; CHECK-NEXT: zip1 z7.s, z25.s, z24.s
-; CHECK-NEXT: zip1 z24.s, z27.s, z26.s
-; CHECK-NEXT: addvl x10, x10, #13
-; CHECK-NEXT: zip2 z25.s, z5.s, z3.s
-; CHECK-NEXT: zip2 z26.s, z4.s, z6.s
-; CHECK-NEXT: addvl x11, sp, #20
-; CHECK-NEXT: zip1 z27.s, z5.s, z3.s
-; CHECK-NEXT: zip1 z4.s, z4.s, z6.s
+; CHECK-NEXT: zip1 z25.s, z27.s, z26.s
+; CHECK-NEXT: zip1 z5.s, z1.s, z5.s
+; CHECK-NEXT: zip1 z6.s, z2.s, z6.s
+; CHECK-NEXT: add x8, x13, x10
+; CHECK-NEXT: zip1 z7.s, z3.s, z7.s
+; CHECK-NEXT: add x14, x12, x10
; CHECK-NEXT: zip2 z0.s, z29.s, z28.s
; CHECK-NEXT: zip1 z1.s, z29.s, z28.s
-; CHECK-NEXT: zip2 z2.s, z24.s, z7.s
-; CHECK-NEXT: zip1 z3.s, z24.s, z7.s
-; CHECK-NEXT: zip2 z5.s, z26.s, z25.s
-; CHECK-NEXT: zip1 z7.s, z26.s, z25.s
-; CHECK-NEXT: whilelo p0.s, xzr, x9
-; CHECK-NEXT: cntw x9, all, mul #3
-; CHECK-NEXT: str z0, [sp, #51, mul vl]
-; CHECK-NEXT: zip2 z6.s, z4.s, z27.s
-; CHECK-NEXT: str z1, [sp, #50, mul vl]
-; CHECK-NEXT: add x10, x10, x9
-; CHECK-NEXT: zip1 z4.s, z4.s, z27.s
-; CHECK-NEXT: str z2, [sp, #49, mul vl]
-; CHECK-NEXT: str z3, [sp, #48, mul vl]
-; CHECK-NEXT: str z5, [sp, #47, mul vl]
+; CHECK-NEXT: zip2 z2.s, z25.s, z24.s
+; CHECK-NEXT: zip2 z26.s, z5.s, z7.s
+; CHECK-NEXT: zip2 z27.s, z4.s, z6.s
+; CHECK-NEXT: zip1 z28.s, z5.s, z7.s
+; CHECK-NEXT: zip1 z4.s, z4.s, z6.s
+; CHECK-NEXT: zip1 z6.s, z25.s, z24.s
+; CHECK-NEXT: str z0, [x8, #1, mul vl]
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: lsr x11, x8, #4
+; CHECK-NEXT: str z1, [x13, #2, mul vl]
+; CHECK-NEXT: zip2 z3.s, z27.s, z26.s
+; CHECK-NEXT: str z2, [x13, #1, mul vl]
+; CHECK-NEXT: mov w13, #3 // =0x3
+; CHECK-NEXT: zip1 z7.s, z27.s, z26.s
+; CHECK-NEXT: mul x13, x11, x13
+; CHECK-NEXT: zip2 z5.s, z4.s, z28.s
+; CHECK-NEXT: zip1 z4.s, z4.s, z28.s
+; CHECK-NEXT: str z3, [x14, #1, mul vl]
+; CHECK-NEXT: addvl x14, sp, #31
+; CHECK-NEXT: str z6, [sp, #48, mul vl]
+; CHECK-NEXT: addvl x14, x14, #5
; CHECK-NEXT: str z7, [sp, #46, mul vl]
-; CHECK-NEXT: str z6, [sp, #45, mul vl]
+; CHECK-NEXT: str z5, [sp, #45, mul vl]
+; CHECK-NEXT: whilelo p0.s, xzr, x13
+; CHECK-NEXT: cntw x13, all, mul #3
; CHECK-NEXT: str z4, [sp, #44, mul vl]
; CHECK-NEXT: str z4, [sp, #52, mul vl]
-; CHECK-NEXT: ld1w { z24.s }, p0/z, [x10]
-; CHECK-NEXT: addvl x10, sp, #31
-; CHECK-NEXT: addvl x10, x10, #21
-; CHECK-NEXT: add x9, x10, x9
-; CHECK-NEXT: addvl x10, sp, #31
-; CHECK-NEXT: st1w { z24.s }, p0, [x9]
-; CHECK-NEXT: cnth x9, all, mul #3
-; CHECK-NEXT: addvl x10, x10, #5
+; CHECK-NEXT: add x12, x12, x13
+; CHECK-NEXT: ld1w { z24.s }, p0/z, [x12]
+; CHECK-NEXT: addvl x12, sp, #31
+; CHECK-NEXT: addvl x12, x12, #21
+; CHECK-NEXT: add x12, x12, x13
+; CHECK-NEXT: add x13, x14, x9
+; CHECK-NEXT: st1w { z24.s }, p0, [x12]
+; CHECK-NEXT: add x12, x13, x10
+; CHECK-NEXT: str z0, [x12, #1, mul vl]
+; CHECK-NEXT: add x12, x14, x10
+; CHECK-NEXT: str z1, [x13, #2, mul vl]
+; CHECK-NEXT: str z2, [x13, #1, mul vl]
+; CHECK-NEXT: str z3, [x12, #1, mul vl]
+; CHECK-NEXT: cnth x12, all, mul #3
; CHECK-NEXT: ldr z24, [sp, #53, mul vl]
-; CHECK-NEXT: str z0, [sp, #43, mul vl]
-; CHECK-NEXT: add x10, x10, x9
-; CHECK-NEXT: str z1, [sp, #42, mul vl]
-; CHECK-NEXT: str z2, [sp, #41, mul vl]
+; CHECK-NEXT: str z6, [sp, #40, mul vl]
+; CHECK-NEXT: add x13, x14, x12
+; CHECK-NEXT: str z7, [sp, #38, mul vl]
+; CHECK-NEXT: addvl x14, sp, #28
+; CHECK-NEXT: str z5, [sp, #37, mul vl]
+; CHECK-NEXT: add x15, x14, x9
; CHECK-NEXT: str z24, [sp, #55, mul vl]
; CHECK-NEXT: ldr z24, [sp, #52, mul vl]
-; CHECK-NEXT: str z3, [sp, #40, mul vl]
-; CHECK-NEXT: str z5, [sp, #39, mul vl]
-; CHECK-NEXT: str z7, [sp, #38, mul vl]
-; CHECK-NEXT: str z6, [sp, #37, mul vl]
; CHECK-NEXT: str z4, [sp, #36, mul vl]
; CHECK-NEXT: str z24, [sp, #54, mul vl]
-; CHECK-NEXT: ld1w { z24.s }, p0/z, [x10]
-; CHECK-NEXT: addvl x10, sp, #31
-; CHECK-NEXT: addvl x10, x10, #23
-; CHECK-NEXT: add x9, x10, x9
-; CHECK-NEXT: addvl x10, sp, #28
-; CHECK-NEXT: st1w { z24.s }, p0, [x9]
-; CHECK-NEXT: cntw x9, all, mul #9
+; CHECK-NEXT: ld1w { z24.s }, p0/z, [x13]
+; CHECK-NEXT: addvl x13, sp, #31
+; CHECK-NEXT: addvl x13, x13, #23
+; CHECK-NEXT: add x12, x13, x12
+; CHECK-NEXT: add x13, x13, x10
+; CHECK-NEXT: st1w { z24.s }, p0, [x12]
+; CHECK-NEXT: add x12, x15, x10
+; CHECK-NEXT: str z0, [x12, #1, mul vl]
+; CHECK-NEXT: add x12, x14, x10
+; CHECK-NEXT: str z1, [x15, #2, mul vl]
+; CHECK-NEXT: str z2, [x15, #1, mul vl]
+; CHECK-NEXT: addvl x15, sp, #20
+; CHECK-NEXT: str z3, [x12, #1, mul vl]
+; CHECK-NEXT: cntw x12, all, mul #9
+; CHECK-NEXT: add x16, x15, x9
; CHECK-NEXT: ldr z24, [sp, #56, mul vl]
-; CHECK-NEXT: str z0, [sp, #35, mul vl]
-; CHECK-NEXT: add x9, x10, x9
-; CHECK-NEXT: str z1, [sp, #34, mul vl]
-; CHECK-NEXT: addvl x10, sp, #3
-; CHECK-NEXT: str z2, [sp, #33, mul vl]
-; CHECK-NEXT: str z3, [sp, #32, mul vl]
-; CHECK-NEXT: str z5, [sp, #31, mul vl]
+; CHECK-NEXT: str z6, [sp, #32, mul vl]
+; CHECK-NEXT: add x12, x14, x12
; CHECK-NEXT: str z7, [sp, #30, mul vl]
-; CHECK-NEXT: str z6, [sp, #29, mul vl]
+; CHECK-NEXT: addvl x14, sp, #3
+; CHECK-NEXT: add x17, x16, x10
+; CHECK-NEXT: str z5, [sp, #29, mul vl]
; CHECK-NEXT: str z4, [sp, #28, mul vl]
; CHECK-NEXT: str z24, [sp, #3, mul vl]
-; CHECK-NEXT: ld1w { z24.s }, p0/z, [x9]
-; CHECK-NEXT: cntw x9
-; CHECK-NEXT: add x10, x10, x9
-; CHECK-NEXT: st1w { z24.s }, p0, [x10]
-; CHECK-NEXT: cntw x10, all, mul #15
+; CHECK-NEXT: ld1w { z24.s }, p0/z, [x12]
+; CHECK-NEXT: cntw x12
+; CHECK-NEXT: add x14, x14, x12
+; CHECK-NEXT: st1w { z24.s }, p0, [x14]
+; CHECK-NEXT: add x14, x15, x10
+; CHECK-NEXT: str z0, [x17, #1, mul vl]
+; CHECK-NEXT: str z1, [x16, #2, mul vl]
+; CHECK-NEXT: str z2, [x16, #1, mul vl]
+; CHECK-NEXT: cntw x16, all, mul #15
+; CHECK-NEXT: str z3, [x14, #1, mul vl]
+; CHECK-NEXT: addvl x14, sp, #31
+; CHECK-NEXT: add x15, x15, x16
+; CHECK-NEXT: ldr z24, [x13, #1, mul vl]
+; CHECK-NEXT: addvl x14, x14, #27
+; CHECK-NEXT: add x13, x14, x10
+; CHECK-NEXT: add x16, x14, x16
+; CHECK-NEXT: add x14, x14, x9
+; CHECK-NEXT: mov z24.s, p0/m, z3.s
+; CHECK-NEXT: str z24, [x13, #1, mul vl]
; CHECK-NEXT: ldr z24, [sp, #55, mul vl]
-; CHECK-NEXT: str z0, [sp, #27, mul vl]
-; CHECK-NEXT: add x11, x11, x10
-; CHECK-NEXT: str z1, [sp, #26, mul vl]
-; CHECK-NEXT: str z2, [sp, #25, mul vl]
+; CHECK-NEXT: str z6, [sp, #24, mul vl]
+; CHECK-NEXT: str z7, [sp, #22, mul vl]
+; CHECK-NEXT: str z5, [sp, #21, mul vl]
; CHECK-NEXT: str z24, [sp, #59, mul vl]
; CHECK-NEXT: ldr z24, [sp, #54, mul vl]
-; CHECK-NEXT: str z3, [sp, #24, mul vl]
-; CHECK-NEXT: str z5, [sp, #23, mul vl]
-; CHECK-NEXT: str z24, [sp, #58, mul vl]
-; CHECK-NEXT: ldr z24, [sp, #57, mul vl]
-; CHECK-NEXT: str z7, [sp, #22, mul vl]
-; CHECK-NEXT: str z6, [sp, #21, mul vl]
-; CHECK-NEXT: mov z24.s, p0/m, z5.s
; CHECK-NEXT: str z4, [sp, #20, mul vl]
-; CHECK-NEXT: str z24, [sp, #61, mul vl]
+; CHECK-NEXT: str z24, [sp, #58, mul vl]
; CHECK-NEXT: ldr z24, [sp, #3, mul vl]
; CHECK-NEXT: str z24, [sp, #60, mul vl]
-; CHECK-NEXT: ld1w { z24.s }, p0/z, [x11]
-; CHECK-NEXT: addvl x11, sp, #31
-; CHECK-NEXT: addvl x11, x11, #27
-; CHECK-NEXT: add x10, x11, x10
-; CHECK-NEXT: addvl x11, sp, #12
-; CHECK-NEXT: st1w { z24.s }, p0, [x10]
-; CHECK-NEXT: cnth x10, all, mul #9
-; CHECK-NEXT: ldr z24, [sp, #63, mul vl]
-; CHECK-NEXT: str z0, [sp, #19, mul vl]
-; CHECK-NEXT: add x10, x11, x10
-; CHECK-NEXT: str z1, [sp, #18, mul vl]
-; CHECK-NEXT: addvl x11, sp, #1
-; CHECK-NEXT: str z2, [sp, #17, mul vl]
-; CHECK-NEXT: str z24, [sp, #2, mul vl]
-; CHECK-NEXT: ldr z24, [sp, #62, mul vl]
-; CHECK-NEXT: str z3, [sp, #16, mul vl]
-; CHECK-NEXT: str z5, [sp, #15, mul vl]
+; CHECK-NEXT: ld1w { z24.s }, p0/z, [x15]
+; CHECK-NEXT: addvl x15, sp, #12
+; CHECK-NEXT: add x17, x15, x9
+; CHECK-NEXT: st1w { z24.s }, p0, [x16]
+; CHECK-NEXT: add x16, x17, x10
+; CHECK-NEXT: str z0, [x16, #1, mul vl]
+; CHECK-NEXT: add x16, x15, x10
+; CHECK-NEXT: str z1, [x17, #2, mul vl]
+; CHECK-NEXT: str z2, [x17, #1, mul vl]
+; CHECK-NEXT: str z3, [x16, #1, mul vl]
+; CHECK-NEXT: addvl x16, sp, #4
+; CHECK-NEXT: str z6, [sp, #16, mul vl]
+; CHECK-NEXT: add x9, x16, x9
; CHECK-NEXT: str z7, [sp, #14, mul vl]
-; CHECK-NEXT: str z6, [sp, #13, mul vl]
+; CHECK-NEXT: str z5, [sp, #13, mul vl]
; CHECK-NEXT: str z4, [sp, #12, mul vl]
+; CHECK-NEXT: ldr z24, [x14, #1, mul vl]
+; CHECK-NEXT: cnth x14, all, mul #9
+; CHECK-NEXT: add x14, x15, x14
+; CHECK-NEXT: addvl x15, sp, #1
+; CHECK-NEXT: str z24, [sp, #2, mul vl]
+; CHECK-NEXT: ldr z24, [sp, #62, mul vl]
; CHECK-NEXT: str z24, [sp, #1, mul vl]
-; CHECK-NEXT: ld1w { z24.s }, p0/z, [x10]
-; CHECK-NEXT: cnth x10
-; CHECK-NEXT: add x10, x11, x10
-; CHECK-NEXT: addvl x11, sp, #4
-; CHECK-NEXT: st1w { z24.s }, p0, [x10]
-; CHECK-NEXT: mov w10, #84 // =0x54
-; CHECK-NEXT: madd x8, x8, x10, x11
-; CHECK-NEXT: str z0, [sp, #11, mul vl]
+; CHECK-NEXT: ld1w { z24.s }, p0/z, [x14]
+; CHECK-NEXT: cnth x14
+; CHECK-NEXT: add x14, x15, x14
+; CHECK-NEXT: add x15, x9, x10
+; CHECK-NEXT: add x10, x16, x10
+; CHECK-NEXT: st1w { z24.s }, p0, [x14]
+; CHECK-NEXT: str z0, [x15, #1, mul vl]
+; CHECK-NEXT: str z1, [x9, #2, mul vl]
+; CHECK-NEXT: str z2, [x9, #1, mul vl]
+; CHECK-NEXT: mov w9, #84 // =0x54
+; CHECK-NEXT: str z3, [x10, #1, mul vl]
+; CHECK-NEXT: madd x9, x11, x9, x16
; CHECK-NEXT: ldr z0, [sp, #2, mul vl]
-; CHECK-NEXT: str z1, [sp, #10, mul vl]
-; CHECK-NEXT: str z2, [sp, #9, mul vl]
-; CHECK-NEXT: str z3, [sp, #8, mul vl]
-; CHECK-NEXT: str z5, [sp, #7, mul vl]
+; CHECK-NEXT: str z6, [sp, #8, mul vl]
; CHECK-NEXT: str z7, [sp, #6, mul vl]
-; CHECK-NEXT: str z6, [sp, #5, mul vl]
+; CHECK-NEXT: str z5, [sp, #5, mul vl]
; CHECK-NEXT: str z4, [sp, #4, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8]
-; CHECK-NEXT: mov x8, sp
-; CHECK-NEXT: add x8, x8, x9
-; CHECK-NEXT: st1w { z0.s }, p0, [x8]
-; CHECK-NEXT: ldr z0, [sp, #61, mul vl]
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x9]
+; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: add x9, x9, x12
+; CHECK-NEXT: st1w { z0.s }, p0, [x9]
+; CHECK-NEXT: add x9, x0, x8
+; CHECK-NEXT: ldr z0, [x13, #1, mul vl]
; CHECK-NEXT: ldr z1, [sp, #60, mul vl]
+; CHECK-NEXT: add x10, x9, x8
; CHECK-NEXT: ldr z2, [sp]
; CHECK-NEXT: ldr z3, [sp, #59, mul vl]
; CHECK-NEXT: ldr z4, [sp, #1, mul vl]
; CHECK-NEXT: ldr z5, [sp, #58, mul vl]
-; CHECK-NEXT: str z0, [x0, #3, mul vl]
-; CHECK-NEXT: str z1, [x0, #2, mul vl]
+; CHECK-NEXT: str z0, [x10, #1, mul vl]
+; CHECK-NEXT: str z1, [x9, #1, mul vl]
+; CHECK-NEXT: add x9, x10, x8
+; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: str z3, [x0, #1, mul vl]
; CHECK-NEXT: str z5, [x0]
-; CHECK-NEXT: str z4, [x0, #4, mul vl]
-; CHECK-NEXT: str z2, [x0, #5, mul vl]
+; CHECK-NEXT: str z4, [x9, #1, mul vl]
+; CHECK-NEXT: str z2, [x8, #1, mul vl]
; CHECK-NEXT: addvl sp, sp, #31
; CHECK-NEXT: addvl sp, sp, #31
; CHECK-NEXT: addvl sp, sp, #4
diff --git a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
index b066abd0a19d5..1556c5f6ec8fc 100644
--- a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
@@ -1963,24 +1963,28 @@ define void @zext_v16i8_to_v16i32_in_loop_scalable_vectors(ptr %src, ptr %dst) {
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: ptrue p0.s
; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: cnth x9
+; CHECK-NEXT: rdvl x10, #2
; CHECK-NEXT: LBB19_1: ; %loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: add x11, x0, x8
; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, x8]
-; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: ld1b { z1.s }, p0/z, [x9, #2, mul vl]
-; CHECK-NEXT: ld1b { z2.s }, p0/z, [x9, #3, mul vl]
-; CHECK-NEXT: ld1b { z3.s }, p0/z, [x9, #1, mul vl]
-; CHECK-NEXT: add x9, x1, x8, lsl #2
+; CHECK-NEXT: ld1b { z1.s }, p0/z, [x11, #2, mul vl]
+; CHECK-NEXT: ld1b { z2.s }, p0/z, [x11, #1, mul vl]
+; CHECK-NEXT: add x11, x11, x9
+; CHECK-NEXT: ld1b { z3.s }, p0/z, [x11, #1, mul vl]
+; CHECK-NEXT: add x11, x1, x8, lsl #2
; CHECK-NEXT: add z0.s, z0.s, z0.s
; CHECK-NEXT: add z1.s, z1.s, z1.s
; CHECK-NEXT: add z2.s, z2.s, z2.s
; CHECK-NEXT: add z3.s, z3.s, z3.s
; CHECK-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]
; CHECK-NEXT: add x8, x8, #16
+; CHECK-NEXT: str z1, [x11, #2, mul vl]
; CHECK-NEXT: cmp x8, #128
-; CHECK-NEXT: str z1, [x9, #2, mul vl]
-; CHECK-NEXT: str z2, [x9, #3, mul vl]
-; CHECK-NEXT: str z3, [x9, #1, mul vl]
+; CHECK-NEXT: str z2, [x11, #1, mul vl]
+; CHECK-NEXT: add x11, x11, x10
+; CHECK-NEXT: str z3, [x11, #1, mul vl]
; CHECK-NEXT: b.ne LBB19_1
; CHECK-NEXT: ; %bb.2: ; %exit
; CHECK-NEXT: ret
@@ -1989,24 +1993,28 @@ define void @zext_v16i8_to_v16i32_in_loop_scalable_vectors(ptr %src, ptr %dst) {
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ptrue p0.s
; CHECK-BE-NEXT: mov x8, xzr
+; CHECK-BE-NEXT: cnth x9
+; CHECK-BE-NEXT: rdvl x10, #2
; CHECK-BE-NEXT: .LBB19_1: // %loop
; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-BE-NEXT: add x11, x0, x8
; CHECK-BE-NEXT: ld1b { z0.s }, p0/z, [x0, x8]
-; CHECK-BE-NEXT: add x9, x0, x8
-; CHECK-BE-NEXT: ld1b { z1.s }, p0/z, [x9, #2, mul vl]
-; CHECK-BE-NEXT: ld1b { z2.s }, p0/z, [x9, #3, mul vl]
-; CHECK-BE-NEXT: ld1b { z3.s }, p0/z, [x9, #1, mul vl]
-; CHECK-BE-NEXT: add x9, x1, x8, lsl #2
+; CHECK-BE-NEXT: ld1b { z1.s }, p0/z, [x11, #2, mul vl]
+; CHECK-BE-NEXT: ld1b { z2.s }, p0/z, [x11, #1, mul vl]
+; CHECK-BE-NEXT: add x11, x11, x9
+; CHECK-BE-NEXT: ld1b { z3.s }, p0/z, [x11, #1, mul vl]
+; CHECK-BE-NEXT: add x11, x1, x8, lsl #2
; CHECK-BE-NEXT: add z0.s, z0.s, z0.s
; CHECK-BE-NEXT: add z1.s, z1.s, z1.s
; CHECK-BE-NEXT: add z2.s, z2.s, z2.s
; CHECK-BE-NEXT: add z3.s, z3.s, z3.s
; CHECK-BE-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]
; CHECK-BE-NEXT: add x8, x8, #16
+; CHECK-BE-NEXT: st1w { z1.s }, p0, [x11, #2, mul vl]
; CHECK-BE-NEXT: cmp x8, #128
-; CHECK-BE-NEXT: st1w { z1.s }, p0, [x9, #2, mul vl]
-; CHECK-BE-NEXT: st1w { z2.s }, p0, [x9, #3, mul vl]
-; CHECK-BE-NEXT: st1w { z3.s }, p0, [x9, #1, mul vl]
+; CHECK-BE-NEXT: st1w { z2.s }, p0, [x11, #1, mul vl]
+; CHECK-BE-NEXT: add x11, x11, x10
+; CHECK-BE-NEXT: st1w { z3.s }, p0, [x11, #1, mul vl]
; CHECK-BE-NEXT: b.ne .LBB19_1
; CHECK-BE-NEXT: // %bb.2: // %exit
; CHECK-BE-NEXT: ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/calling-conv-fastcc.ll b/llvm/test/CodeGen/RISCV/rvv/calling-conv-fastcc.ll
index e7610de30ef10..83fb78c12bfc1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/calling-conv-fastcc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/calling-conv-fastcc.ll
@@ -70,23 +70,22 @@ define fastcc <vscale x 64 x i32> @ret_split_nxv64i32(ptr %x) {
; CHECK-LABEL: ret_split_nxv64i32:
; CHECK: # %bb.0:
; CHECK-NEXT: csrr a2, vlenb
-; CHECK-NEXT: slli a3, a2, 3
-; CHECK-NEXT: slli a2, a2, 4
-; CHECK-NEXT: add a4, a2, a3
-; CHECK-NEXT: add a5, a1, a4
+; CHECK-NEXT: slli a3, a2, 4
+; CHECK-NEXT: slli a2, a2, 3
+; CHECK-NEXT: add a4, a1, a3
+; CHECK-NEXT: add a5, a4, a2
; CHECK-NEXT: vl8re32.v v8, (a5)
-; CHECK-NEXT: add a5, a1, a3
-; CHECK-NEXT: vl8re32.v v16, (a5)
; CHECK-NEXT: add a5, a1, a2
-; CHECK-NEXT: vl8re32.v v24, (a5)
+; CHECK-NEXT: vl8re32.v v16, (a5)
+; CHECK-NEXT: vl8re32.v v24, (a4)
; CHECK-NEXT: vl8re32.v v0, (a1)
; CHECK-NEXT: vs8r.v v0, (a0)
-; CHECK-NEXT: add a2, a0, a2
-; CHECK-NEXT: vs8r.v v24, (a2)
; CHECK-NEXT: add a3, a0, a3
-; CHECK-NEXT: vs8r.v v16, (a3)
-; CHECK-NEXT: add a0, a0, a4
-; CHECK-NEXT: vs8r.v v8, (a0)
+; CHECK-NEXT: vs8r.v v24, (a3)
+; CHECK-NEXT: add a0, a0, a2
+; CHECK-NEXT: vs8r.v v16, (a0)
+; CHECK-NEXT: add a2, a3, a2
+; CHECK-NEXT: vs8r.v v8, (a2)
; CHECK-NEXT: ret
%v = load <vscale x 64 x i32>, ptr %x
ret <vscale x 64 x i32> %v
@@ -103,80 +102,75 @@ define fastcc <vscale x 128 x i32> @ret_split_nxv128i32(ptr %x) {
; CHECK-NEXT: sub sp, sp, a2
; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
; CHECK-NEXT: csrr a2, vlenb
-; CHECK-NEXT: slli a3, a2, 3
+; CHECK-NEXT: slli a3, a2, 5
; CHECK-NEXT: slli a4, a2, 4
-; CHECK-NEXT: add a5, a4, a3
-; CHECK-NEXT: add a6, a1, a5
-; CHECK-NEXT: vl8re32.v v8, (a6)
-; CHECK-NEXT: csrr a6, vlenb
-; CHECK-NEXT: li a7, 24
-; CHECK-NEXT: mul a6, a6, a7
-; CHECK-NEXT: add a6, sp, a6
-; CHECK-NEXT: addi a6, a6, 16
-; CHECK-NEXT: vs8r.v v8, (a6) # vscale x 64-byte Folded Spill
-; CHECK-NEXT: slli a6, a2, 5
-; CHECK-NEXT: add a7, a6, a3
-; CHECK-NEXT: add t0, a1, a7
+; CHECK-NEXT: add a5, a1, a3
+; CHECK-NEXT: slli a2, a2, 3
+; CHECK-NEXT: add a6, a5, a4
+; CHECK-NEXT: add a7, a6, a2
+; CHECK-NEXT: vl8re32.v v8, (a7)
+; CHECK-NEXT: csrr a7, vlenb
+; CHECK-NEXT: li t0, 24
+; CHECK-NEXT: mul a7, a7, t0
+; CHECK-NEXT: add a7, sp, a7
+; CHECK-NEXT: addi a7, a7, 16
+; CHECK-NEXT: vs8r.v v8, (a7) # vscale x 64-byte Folded Spill
+; CHECK-NEXT: add a7, a1, a4
+; CHECK-NEXT: add t0, a7, a2
; CHECK-NEXT: vl8re32.v v8, (t0)
; CHECK-NEXT: csrr t0, vlenb
; CHECK-NEXT: slli t0, t0, 4
; CHECK-NEXT: add t0, sp, t0
; CHECK-NEXT: addi t0, t0, 16
; CHECK-NEXT: vs8r.v v8, (t0) # vscale x 64-byte Folded Spill
-; CHECK-NEXT: add t0, a6, a4
-; CHECK-NEXT: add t1, a1, t0
-; CHECK-NEXT: vl8re32.v v8, (t1)
-; CHECK-NEXT: csrr t1, vlenb
-; CHECK-NEXT: slli t1, t1, 3
-; CHECK-NEXT: add t1, sp, t1
-; CHECK-NEXT: addi t1, t1, 16
-; CHECK-NEXT: vs8r.v v8, (t1) # vscale x 64-byte Folded Spill
-; CHECK-NEXT: slli a2, a2, 6
-; CHECK-NEXT: sub a2, a2, a3
-; CHECK-NEXT: add t1, a1, a2
-; CHECK-NEXT: vl8re32.v v8, (t1)
-; CHECK-NEXT: addi t1, sp, 16
-; CHECK-NEXT: vs8r.v v8, (t1) # vscale x 64-byte Folded Spill
-; CHECK-NEXT: add t1, a1, a3
-; CHECK-NEXT: vl8re32.v v0, (t1)
-; CHECK-NEXT: add t1, a1, a4
-; CHECK-NEXT: vl8re32.v v24, (t1)
-; CHECK-NEXT: add t1, a1, a6
-; CHECK-NEXT: vl8re32.v v16, (t1)
+; CHECK-NEXT: add t0, a5, a2
+; CHECK-NEXT: vl8re32.v v8, (t0)
+; CHECK-NEXT: csrr t0, vlenb
+; CHECK-NEXT: slli t0, t0, 3
+; CHECK-NEXT: add t0, sp, t0
+; CHECK-NEXT: addi t0, t0, 16
+; CHECK-NEXT: vs8r.v v8, (t0) # vscale x 64-byte Folded Spill
+; CHECK-NEXT: vl8re32.v v8, (a6)
+; CHECK-NEXT: addi a6, sp, 16
+; CHECK-NEXT: vs8r.v v8, (a6) # vscale x 64-byte Folded Spill
+; CHECK-NEXT: add a6, a1, a2
+; CHECK-NEXT: vl8re32.v v0, (a6)
+; CHECK-NEXT: vl8re32.v v24, (a7)
+; CHECK-NEXT: vl8re32.v v16, (a5)
; CHECK-NEXT: vl8re32.v v8, (a1)
; CHECK-NEXT: vs8r.v v8, (a0)
-; CHECK-NEXT: add a6, a0, a6
-; CHECK-NEXT: vs8r.v v16, (a6)
-; CHECK-NEXT: add a4, a0, a4
-; CHECK-NEXT: vs8r.v v24, (a4)
; CHECK-NEXT: add a3, a0, a3
-; CHECK-NEXT: vs8r.v v0, (a3)
-; CHECK-NEXT: add a2, a0, a2
-; CHECK-NEXT: addi a1, sp, 16
-; CHECK-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
+; CHECK-NEXT: vs8r.v v16, (a3)
+; CHECK-NEXT: add a1, a0, a4
+; CHECK-NEXT: vs8r.v v24, (a1)
+; CHECK-NEXT: add a0, a0, a2
+; CHECK-NEXT: vs8r.v v0, (a0)
+; CHECK-NEXT: add a4, a3, a4
+; CHECK-NEXT: addi a0, sp, 16
+; CHECK-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
+; CHECK-NEXT: vs8r.v v8, (a4)
+; CHECK-NEXT: add a3, a3, a2
+; CHECK-NEXT: csrr a0, vlenb
+; CHECK-NEXT: slli a0, a0, 3
+; CHECK-NEXT: add a0, sp, a0
+; CHECK-NEXT: addi a0, a0, 16
+; CHECK-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
+; CHECK-NEXT: vs8r.v v8, (a3)
+; CHECK-NEXT: add a1, a1, a2
+; CHECK-NEXT: csrr a0, vlenb
+; CHECK-NEXT: slli a0, a0, 4
+; CHECK-NEXT: add a0, sp, a0
+; CHECK-NEXT: addi a0, a0, 16
+; CHECK-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
+; CHECK-NEXT: vs8r.v v8, (a1)
+; CHECK-NEXT: add a2, a4, a2
+; CHECK-NEXT: csrr a0, vlenb
+; CHECK-NEXT: li a1, 24
+; CHECK-NEXT: mul a0, a0, a1
+; CHECK-NEXT: add a0, sp, a0
+; CHECK-NEXT: addi a0, a0, 16
+; CHECK-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
; CHECK-NEXT: vs8r.v v8, (a2)
-; CHECK-NEXT: add t0, a0, t0
-; CHECK-NEXT: csrr a1, vlenb
-; CHECK-NEXT: slli a1, a1, 3
-; CHECK-NEXT: add a1, sp, a1
-; CHECK-NEXT: addi a1, a1, 16
-; CHECK-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
-; CHECK-NEXT: vs8r.v v8, (t0)
-; CHECK-NEXT: add a7, a0, a7
-; CHECK-NEXT: csrr a1, vlenb
-; CHECK-NEXT: slli a1, a1, 4
-; CHECK-NEXT: add a1, sp, a1
-; CHECK-NEXT: addi a1, a1, 16
-; CHECK-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
-; CHECK-NEXT: vs8r.v v8, (a7)
-; CHECK-NEXT: add a0, a0, a5
-; CHECK-NEXT: csrr a1, vlenb
-; CHECK-NEXT: li a2, 24
-; CHECK-NEXT: mul a1, a1, a2
-; CHECK-NEXT: add a1, sp, a1
-; CHECK-NEXT: addi a1, a1, 16
-; CHECK-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
-; CHECK-NEXT: vs8r.v v8, (a0)
; CHECK-NEXT: csrr a0, vlenb
; CHECK-NEXT: slli a0, a0, 5
; CHECK-NEXT: add sp, sp, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/lrint-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/lrint-sdnode.ll
index 2f005debc88e5..d68f2763104fb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/lrint-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/lrint-sdnode.ll
@@ -537,100 +537,32 @@ define <vscale x 32 x iXLen> @lrint_nxv32bf16(<vscale x 32 x bfloat> %x) {
;
; RV64-i64-LABEL: lrint_nxv32bf16:
; RV64-i64: # %bb.0:
-; RV64-i64-NEXT: addi sp, sp, -64
-; RV64-i64-NEXT: .cfi_def_cfa_offset 64
-; RV64-i64-NEXT: sd ra, 56(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT: sd s0, 48(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT: sd s1, 40(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT: .cfi_offset ra, -8
-; RV64-i64-NEXT: .cfi_offset s0, -16
-; RV64-i64-NEXT: .cfi_offset s1, -24
; RV64-i64-NEXT: csrr a1, vlenb
-; RV64-i64-NEXT: slli a1, a1, 5
-; RV64-i64-NEXT: sub sp, sp, a1
-; RV64-i64-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xc0, 0x00, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 64 + 32 * vlenb
-; RV64-i64-NEXT: vsetvli a1, zero, e16, m2, ta, ma
-; RV64-i64-NEXT: vfwcvtbf16.f.f.v v16, v8
+; RV64-i64-NEXT: vsetvli a2, zero, e16, m2, ta, ma
+; RV64-i64-NEXT: vfwcvtbf16.f.f.v v24, v12
; RV64-i64-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; RV64-i64-NEXT: vfwcvt.x.f.v v24, v16
-; RV64-i64-NEXT: csrr a1, vlenb
-; RV64-i64-NEXT: slli a1, a1, 3
-; RV64-i64-NEXT: mv a2, a1
-; RV64-i64-NEXT: slli a1, a1, 1
-; RV64-i64-NEXT: add a1, a1, a2
-; RV64-i64-NEXT: add a1, sp, a1
-; RV64-i64-NEXT: addi a1, a1, 32
-; RV64-i64-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; RV64-i64-NEXT: vfwcvt.x.f.v v16, v24
; RV64-i64-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; RV64-i64-NEXT: vfwcvtbf16.f.f.v v16, v10
+; RV64-i64-NEXT: vfwcvtbf16.f.f.v v4, v14
; RV64-i64-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; RV64-i64-NEXT: vfwcvt.x.f.v v24, v16
-; RV64-i64-NEXT: csrr a1, vlenb
-; RV64-i64-NEXT: slli a1, a1, 4
-; RV64-i64-NEXT: add a1, sp, a1
-; RV64-i64-NEXT: addi a1, a1, 32
-; RV64-i64-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; RV64-i64-NEXT: vfwcvt.x.f.v v24, v4
+; RV64-i64-NEXT: slli a2, a1, 4
+; RV64-i64-NEXT: add a2, a0, a2
+; RV64-i64-NEXT: slli a1, a1, 3
+; RV64-i64-NEXT: add a3, a2, a1
+; RV64-i64-NEXT: vs8r.v v24, (a3)
; RV64-i64-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; RV64-i64-NEXT: vfwcvtbf16.f.f.v v8, v12
+; RV64-i64-NEXT: vfwcvtbf16.f.f.v v12, v10
; RV64-i64-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; RV64-i64-NEXT: vfwcvt.x.f.v v16, v8
-; RV64-i64-NEXT: csrr a1, vlenb
-; RV64-i64-NEXT: slli a1, a1, 3
-; RV64-i64-NEXT: add a1, sp, a1
-; RV64-i64-NEXT: addi a1, a1, 32
-; RV64-i64-NEXT: vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
+; RV64-i64-NEXT: vfwcvt.x.f.v v24, v12
+; RV64-i64-NEXT: vs8r.v v16, (a2)
; RV64-i64-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; RV64-i64-NEXT: vfwcvtbf16.f.f.v v8, v14
+; RV64-i64-NEXT: vfwcvtbf16.f.f.v v16, v8
+; RV64-i64-NEXT: add a1, a0, a1
+; RV64-i64-NEXT: vs8r.v v24, (a1)
; RV64-i64-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; RV64-i64-NEXT: vfwcvt.x.f.v v16, v8
-; RV64-i64-NEXT: addi a1, sp, 32
-; RV64-i64-NEXT: vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
-; RV64-i64-NEXT: mv s0, a0
-; RV64-i64-NEXT: csrr s1, vlenb
-; RV64-i64-NEXT: li a1, 24
-; RV64-i64-NEXT: mv a0, s1
-; RV64-i64-NEXT: call __muldi3
-; RV64-i64-NEXT: add a0, s0, a0
-; RV64-i64-NEXT: addi a1, sp, 32
-; RV64-i64-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
-; RV64-i64-NEXT: vs8r.v v8, (a0)
-; RV64-i64-NEXT: slli a0, s1, 4
-; RV64-i64-NEXT: add a0, s0, a0
-; RV64-i64-NEXT: csrr a1, vlenb
-; RV64-i64-NEXT: slli a1, a1, 3
-; RV64-i64-NEXT: add a1, sp, a1
-; RV64-i64-NEXT: addi a1, a1, 32
-; RV64-i64-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
+; RV64-i64-NEXT: vfwcvt.x.f.v v8, v16
; RV64-i64-NEXT: vs8r.v v8, (a0)
-; RV64-i64-NEXT: slli s1, s1, 3
-; RV64-i64-NEXT: add s1, s0, s1
-; RV64-i64-NEXT: csrr a0, vlenb
-; RV64-i64-NEXT: slli a0, a0, 4
-; RV64-i64-NEXT: add a0, sp, a0
-; RV64-i64-NEXT: addi a0, a0, 32
-; RV64-i64-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
-; RV64-i64-NEXT: vs8r.v v8, (s1)
-; RV64-i64-NEXT: csrr a0, vlenb
-; RV64-i64-NEXT: slli a0, a0, 3
-; RV64-i64-NEXT: mv a1, a0
-; RV64-i64-NEXT: slli a0, a0, 1
-; RV64-i64-NEXT: add a0, a0, a1
-; RV64-i64-NEXT: add a0, sp, a0
-; RV64-i64-NEXT: addi a0, a0, 32
-; RV64-i64-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
-; RV64-i64-NEXT: vs8r.v v8, (s0)
-; RV64-i64-NEXT: csrr a0, vlenb
-; RV64-i64-NEXT: slli a0, a0, 5
-; RV64-i64-NEXT: add sp, sp, a0
-; RV64-i64-NEXT: .cfi_def_cfa sp, 64
-; RV64-i64-NEXT: ld ra, 56(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT: ld s0, 48(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT: ld s1, 40(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT: .cfi_restore ra
-; RV64-i64-NEXT: .cfi_restore s0
-; RV64-i64-NEXT: .cfi_restore s1
-; RV64-i64-NEXT: addi sp, sp, 64
-; RV64-i64-NEXT: .cfi_def_cfa_offset 0
; RV64-i64-NEXT: ret
%a = call <vscale x 32 x iXLen> @llvm.lrint.nxv32iXLen.nxv32bf16(<vscale x 32 x bfloat> %x)
ret <vscale x 32 x iXLen> %a
diff --git a/llvm/test/CodeGen/RISCV/rvv/lround-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/lround-sdnode.ll
index f1096a7e47ae9..37eb3cc0856ff 100644
--- a/llvm/test/CodeGen/RISCV/rvv/lround-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/lround-sdnode.ll
@@ -655,102 +655,34 @@ define <vscale x 32 x iXLen> @lround_nxv32bf16(<vscale x 32 x bfloat> %x) {
;
; RV64-i64-LABEL: lround_nxv32bf16:
; RV64-i64: # %bb.0:
-; RV64-i64-NEXT: addi sp, sp, -64
-; RV64-i64-NEXT: .cfi_def_cfa_offset 64
-; RV64-i64-NEXT: sd ra, 56(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT: sd s0, 48(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT: sd s1, 40(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT: .cfi_offset ra, -8
-; RV64-i64-NEXT: .cfi_offset s0, -16
-; RV64-i64-NEXT: .cfi_offset s1, -24
-; RV64-i64-NEXT: csrr a1, vlenb
-; RV64-i64-NEXT: slli a1, a1, 5
-; RV64-i64-NEXT: sub sp, sp, a1
-; RV64-i64-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xc0, 0x00, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 64 + 32 * vlenb
-; RV64-i64-NEXT: fsrmi a2, 4
-; RV64-i64-NEXT: vsetvli a1, zero, e16, m2, ta, ma
-; RV64-i64-NEXT: vfwcvtbf16.f.f.v v16, v8
+; RV64-i64-NEXT: fsrmi a1, 4
+; RV64-i64-NEXT: vsetvli a2, zero, e16, m2, ta, ma
+; RV64-i64-NEXT: vfwcvtbf16.f.f.v v24, v12
; RV64-i64-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; RV64-i64-NEXT: vfwcvt.x.f.v v24, v16
-; RV64-i64-NEXT: csrr a1, vlenb
-; RV64-i64-NEXT: slli a1, a1, 3
-; RV64-i64-NEXT: mv a3, a1
-; RV64-i64-NEXT: slli a1, a1, 1
-; RV64-i64-NEXT: add a1, a1, a3
-; RV64-i64-NEXT: add a1, sp, a1
-; RV64-i64-NEXT: addi a1, a1, 32
-; RV64-i64-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; RV64-i64-NEXT: vfwcvt.x.f.v v16, v24
; RV64-i64-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; RV64-i64-NEXT: vfwcvtbf16.f.f.v v16, v10
+; RV64-i64-NEXT: vfwcvtbf16.f.f.v v4, v14
; RV64-i64-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; RV64-i64-NEXT: vfwcvt.x.f.v v24, v16
-; RV64-i64-NEXT: csrr a1, vlenb
-; RV64-i64-NEXT: slli a1, a1, 4
-; RV64-i64-NEXT: add a1, sp, a1
-; RV64-i64-NEXT: addi a1, a1, 32
-; RV64-i64-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; RV64-i64-NEXT: vfwcvt.x.f.v v24, v4
+; RV64-i64-NEXT: csrr a2, vlenb
+; RV64-i64-NEXT: slli a3, a2, 4
+; RV64-i64-NEXT: add a3, a0, a3
+; RV64-i64-NEXT: slli a2, a2, 3
+; RV64-i64-NEXT: add a4, a3, a2
+; RV64-i64-NEXT: vs8r.v v24, (a4)
; RV64-i64-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; RV64-i64-NEXT: vfwcvtbf16.f.f.v v8, v12
+; RV64-i64-NEXT: vfwcvtbf16.f.f.v v12, v10
; RV64-i64-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; RV64-i64-NEXT: vfwcvt.x.f.v v16, v8
-; RV64-i64-NEXT: csrr a1, vlenb
-; RV64-i64-NEXT: slli a1, a1, 3
-; RV64-i64-NEXT: add a1, sp, a1
-; RV64-i64-NEXT: addi a1, a1, 32
-; RV64-i64-NEXT: vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
+; RV64-i64-NEXT: vfwcvt.x.f.v v24, v12
+; RV64-i64-NEXT: vs8r.v v16, (a3)
; RV64-i64-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; RV64-i64-NEXT: vfwcvtbf16.f.f.v v8, v14
+; RV64-i64-NEXT: vfwcvtbf16.f.f.v v16, v8
+; RV64-i64-NEXT: add a2, a0, a2
+; RV64-i64-NEXT: vs8r.v v24, (a2)
; RV64-i64-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; RV64-i64-NEXT: vfwcvt.x.f.v v16, v8
-; RV64-i64-NEXT: addi a1, sp, 32
-; RV64-i64-NEXT: vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
-; RV64-i64-NEXT: mv s0, a0
-; RV64-i64-NEXT: csrr s1, vlenb
-; RV64-i64-NEXT: li a1, 24
-; RV64-i64-NEXT: mv a0, s1
-; RV64-i64-NEXT: fsrm a2
-; RV64-i64-NEXT: call __muldi3
-; RV64-i64-NEXT: add a0, s0, a0
-; RV64-i64-NEXT: addi a1, sp, 32
-; RV64-i64-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
-; RV64-i64-NEXT: vs8r.v v8, (a0)
-; RV64-i64-NEXT: slli a0, s1, 4
-; RV64-i64-NEXT: add a0, s0, a0
-; RV64-i64-NEXT: csrr a1, vlenb
-; RV64-i64-NEXT: slli a1, a1, 3
-; RV64-i64-NEXT: add a1, sp, a1
-; RV64-i64-NEXT: addi a1, a1, 32
-; RV64-i64-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
+; RV64-i64-NEXT: vfwcvt.x.f.v v8, v16
+; RV64-i64-NEXT: fsrm a1
; RV64-i64-NEXT: vs8r.v v8, (a0)
-; RV64-i64-NEXT: slli s1, s1, 3
-; RV64-i64-NEXT: add s1, s0, s1
-; RV64-i64-NEXT: csrr a0, vlenb
-; RV64-i64-NEXT: slli a0, a0, 4
-; RV64-i64-NEXT: add a0, sp, a0
-; RV64-i64-NEXT: addi a0, a0, 32
-; RV64-i64-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
-; RV64-i64-NEXT: vs8r.v v8, (s1)
-; RV64-i64-NEXT: csrr a0, vlenb
-; RV64-i64-NEXT: slli a0, a0, 3
-; RV64-i64-NEXT: mv a1, a0
-; RV64-i64-NEXT: slli a0, a0, 1
-; RV64-i64-NEXT: add a0, a0, a1
-; RV64-i64-NEXT: add a0, sp, a0
-; RV64-i64-NEXT: addi a0, a0, 32
-; RV64-i64-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
-; RV64-i64-NEXT: vs8r.v v8, (s0)
-; RV64-i64-NEXT: csrr a0, vlenb
-; RV64-i64-NEXT: slli a0, a0, 5
-; RV64-i64-NEXT: add sp, sp, a0
-; RV64-i64-NEXT: .cfi_def_cfa sp, 64
-; RV64-i64-NEXT: ld ra, 56(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT: ld s0, 48(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT: ld s1, 40(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT: .cfi_restore ra
-; RV64-i64-NEXT: .cfi_restore s0
-; RV64-i64-NEXT: .cfi_restore s1
-; RV64-i64-NEXT: addi sp, sp, 64
-; RV64-i64-NEXT: .cfi_def_cfa_offset 0
; RV64-i64-NEXT: ret
%a = call <vscale x 32 x iXLen> @llvm.lround.nxv32iXLen.nxv32bf16(<vscale x 32 x bfloat> %x)
ret <vscale x 32 x iXLen> %a
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
index 6fb9b73783d43..8282c31afbf8e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
@@ -3941,20 +3941,19 @@ define <vscale x 112 x i8> @vector_interleave_nxv112i8_nxv16i8(<vscale x 16 x i8
; RV32-NEXT: vl1r.v v20, (a3)
; RV32-NEXT: add a3, a3, a0
; RV32-NEXT: vl1r.v v21, (a3)
-; RV32-NEXT: slli a1, a0, 2
-; RV32-NEXT: slli a0, a0, 3
-; RV32-NEXT: add a1, a0, a1
+; RV32-NEXT: slli a1, a0, 3
; RV32-NEXT: csrr a2, vlenb
; RV32-NEXT: li a3, 14
; RV32-NEXT: mul a2, a2, a3
; RV32-NEXT: add a2, sp, a2
; RV32-NEXT: addi a2, a2, 64
; RV32-NEXT: add a1, a2, a1
-; RV32-NEXT: vs2r.v v20, (a1)
-; RV32-NEXT: add a0, a2, a0
-; RV32-NEXT: vs4r.v v16, (a0)
+; RV32-NEXT: slli a0, a0, 2
+; RV32-NEXT: add a0, a1, a0
+; RV32-NEXT: vs2r.v v20, (a0)
+; RV32-NEXT: vs4r.v v16, (a1)
; RV32-NEXT: vs8r.v v8, (a2)
-; RV32-NEXT: vl8r.v v16, (a0)
+; RV32-NEXT: vl8r.v v16, (a1)
; RV32-NEXT: vl8r.v v8, (a2)
; RV32-NEXT: addi sp, s0, -80
; RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -4022,20 +4021,19 @@ define <vscale x 112 x i8> @vector_interleave_nxv112i8_nxv16i8(<vscale x 16 x i8
; RV64-NEXT: vl1r.v v20, (a3)
; RV64-NEXT: add a3, a3, a0
; RV64-NEXT: vl1r.v v21, (a3)
-; RV64-NEXT: slli a1, a0, 2
-; RV64-NEXT: slli a0, a0, 3
-; RV64-NEXT: add a1, a0, a1
+; RV64-NEXT: slli a1, a0, 3
; RV64-NEXT: csrr a2, vlenb
; RV64-NEXT: li a3, 14
; RV64-NEXT: mul a2, a2, a3
; RV64-NEXT: add a2, sp, a2
; RV64-NEXT: addi a2, a2, 64
; RV64-NEXT: add a1, a2, a1
-; RV64-NEXT: vs2r.v v20, (a1)
-; RV64-NEXT: add a0, a2, a0
-; RV64-NEXT: vs4r.v v16, (a0)
+; RV64-NEXT: slli a0, a0, 2
+; RV64-NEXT: add a0, a1, a0
+; RV64-NEXT: vs2r.v v20, (a0)
+; RV64-NEXT: vs4r.v v16, (a1)
; RV64-NEXT: vs8r.v v8, (a2)
-; RV64-NEXT: vl8r.v v16, (a0)
+; RV64-NEXT: vl8r.v v16, (a1)
; RV64-NEXT: vl8r.v v8, (a2)
; RV64-NEXT: addi sp, s0, -80
; RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -4103,20 +4101,19 @@ define <vscale x 112 x i8> @vector_interleave_nxv112i8_nxv16i8(<vscale x 16 x i8
; ZVBB-RV32-NEXT: vl1r.v v20, (a3)
; ZVBB-RV32-NEXT: add a3, a3, a0
; ZVBB-RV32-NEXT: vl1r.v v21, (a3)
-; ZVBB-RV32-NEXT: slli a1, a0, 2
-; ZVBB-RV32-NEXT: slli a0, a0, 3
-; ZVBB-RV32-NEXT: add a1, a0, a1
+; ZVBB-RV32-NEXT: slli a1, a0, 3
; ZVBB-RV32-NEXT: csrr a2, vlenb
; ZVBB-RV32-NEXT: li a3, 14
; ZVBB-RV32-NEXT: mul a2, a2, a3
; ZVBB-RV32-NEXT: add a2, sp, a2
; ZVBB-RV32-NEXT: addi a2, a2, 64
; ZVBB-RV32-NEXT: add a1, a2, a1
-; ZVBB-RV32-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV32-NEXT: add a0, a2, a0
-; ZVBB-RV32-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV32-NEXT: slli a0, a0, 2
+; ZVBB-RV32-NEXT: add a0, a1, a0
+; ZVBB-RV32-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV32-NEXT: vs4r.v v16, (a1)
; ZVBB-RV32-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV32-NEXT: vl8r.v v16, (a0)
+; ZVBB-RV32-NEXT: vl8r.v v16, (a1)
; ZVBB-RV32-NEXT: vl8r.v v8, (a2)
; ZVBB-RV32-NEXT: addi sp, s0, -80
; ZVBB-RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -4184,20 +4181,19 @@ define <vscale x 112 x i8> @vector_interleave_nxv112i8_nxv16i8(<vscale x 16 x i8
; ZVBB-RV64-NEXT: vl1r.v v20, (a3)
; ZVBB-RV64-NEXT: add a3, a3, a0
; ZVBB-RV64-NEXT: vl1r.v v21, (a3)
-; ZVBB-RV64-NEXT: slli a1, a0, 2
-; ZVBB-RV64-NEXT: slli a0, a0, 3
-; ZVBB-RV64-NEXT: add a1, a0, a1
+; ZVBB-RV64-NEXT: slli a1, a0, 3
; ZVBB-RV64-NEXT: csrr a2, vlenb
; ZVBB-RV64-NEXT: li a3, 14
; ZVBB-RV64-NEXT: mul a2, a2, a3
; ZVBB-RV64-NEXT: add a2, sp, a2
; ZVBB-RV64-NEXT: addi a2, a2, 64
; ZVBB-RV64-NEXT: add a1, a2, a1
-; ZVBB-RV64-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV64-NEXT: add a0, a2, a0
-; ZVBB-RV64-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV64-NEXT: slli a0, a0, 2
+; ZVBB-RV64-NEXT: add a0, a1, a0
+; ZVBB-RV64-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV64-NEXT: vs4r.v v16, (a1)
; ZVBB-RV64-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV64-NEXT: vl8r.v v16, (a0)
+; ZVBB-RV64-NEXT: vl8r.v v16, (a1)
; ZVBB-RV64-NEXT: vl8r.v v8, (a2)
; ZVBB-RV64-NEXT: addi sp, s0, -80
; ZVBB-RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -4265,20 +4261,19 @@ define <vscale x 112 x i8> @vector_interleave_nxv112i8_nxv16i8(<vscale x 16 x i8
; ZVZIP-NEXT: vl1r.v v20, (a3)
; ZVZIP-NEXT: add a3, a3, a0
; ZVZIP-NEXT: vl1r.v v21, (a3)
-; ZVZIP-NEXT: slli a1, a0, 2
-; ZVZIP-NEXT: slli a0, a0, 3
-; ZVZIP-NEXT: add a1, a0, a1
+; ZVZIP-NEXT: slli a1, a0, 3
; ZVZIP-NEXT: csrr a2, vlenb
; ZVZIP-NEXT: li a3, 14
; ZVZIP-NEXT: mul a2, a2, a3
; ZVZIP-NEXT: add a2, sp, a2
; ZVZIP-NEXT: addi a2, a2, 64
; ZVZIP-NEXT: add a1, a2, a1
-; ZVZIP-NEXT: vs2r.v v20, (a1)
-; ZVZIP-NEXT: add a0, a2, a0
-; ZVZIP-NEXT: vs4r.v v16, (a0)
+; ZVZIP-NEXT: slli a0, a0, 2
+; ZVZIP-NEXT: add a0, a1, a0
+; ZVZIP-NEXT: vs2r.v v20, (a0)
+; ZVZIP-NEXT: vs4r.v v16, (a1)
; ZVZIP-NEXT: vs8r.v v8, (a2)
-; ZVZIP-NEXT: vl8r.v v16, (a0)
+; ZVZIP-NEXT: vl8r.v v16, (a1)
; ZVZIP-NEXT: vl8r.v v8, (a2)
; ZVZIP-NEXT: addi sp, s0, -80
; ZVZIP-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -4352,20 +4347,19 @@ define <vscale x 56 x i16> @vector_interleave_nxv56i16_nxv8i16(<vscale x 8 x i16
; RV32-NEXT: vl1re16.v v20, (a3)
; RV32-NEXT: add a3, a3, a0
; RV32-NEXT: vl1re16.v v21, (a3)
-; RV32-NEXT: slli a1, a0, 2
-; RV32-NEXT: slli a0, a0, 3
-; RV32-NEXT: add a1, a0, a1
+; RV32-NEXT: slli a1, a0, 3
; RV32-NEXT: csrr a2, vlenb
; RV32-NEXT: li a3, 14
; RV32-NEXT: mul a2, a2, a3
; RV32-NEXT: add a2, sp, a2
; RV32-NEXT: addi a2, a2, 64
; RV32-NEXT: add a1, a2, a1
-; RV32-NEXT: vs2r.v v20, (a1)
-; RV32-NEXT: add a0, a2, a0
-; RV32-NEXT: vs4r.v v16, (a0)
+; RV32-NEXT: slli a0, a0, 2
+; RV32-NEXT: add a0, a1, a0
+; RV32-NEXT: vs2r.v v20, (a0)
+; RV32-NEXT: vs4r.v v16, (a1)
; RV32-NEXT: vs8r.v v8, (a2)
-; RV32-NEXT: vl8re16.v v16, (a0)
+; RV32-NEXT: vl8re16.v v16, (a1)
; RV32-NEXT: vl8re16.v v8, (a2)
; RV32-NEXT: addi sp, s0, -80
; RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -4433,20 +4427,19 @@ define <vscale x 56 x i16> @vector_interleave_nxv56i16_nxv8i16(<vscale x 8 x i16
; RV64-NEXT: vl1re16.v v20, (a3)
; RV64-NEXT: add a3, a3, a0
; RV64-NEXT: vl1re16.v v21, (a3)
-; RV64-NEXT: slli a1, a0, 2
-; RV64-NEXT: slli a0, a0, 3
-; RV64-NEXT: add a1, a0, a1
+; RV64-NEXT: slli a1, a0, 3
; RV64-NEXT: csrr a2, vlenb
; RV64-NEXT: li a3, 14
; RV64-NEXT: mul a2, a2, a3
; RV64-NEXT: add a2, sp, a2
; RV64-NEXT: addi a2, a2, 64
; RV64-NEXT: add a1, a2, a1
-; RV64-NEXT: vs2r.v v20, (a1)
-; RV64-NEXT: add a0, a2, a0
-; RV64-NEXT: vs4r.v v16, (a0)
+; RV64-NEXT: slli a0, a0, 2
+; RV64-NEXT: add a0, a1, a0
+; RV64-NEXT: vs2r.v v20, (a0)
+; RV64-NEXT: vs4r.v v16, (a1)
; RV64-NEXT: vs8r.v v8, (a2)
-; RV64-NEXT: vl8re16.v v16, (a0)
+; RV64-NEXT: vl8re16.v v16, (a1)
; RV64-NEXT: vl8re16.v v8, (a2)
; RV64-NEXT: addi sp, s0, -80
; RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -4514,20 +4507,19 @@ define <vscale x 56 x i16> @vector_interleave_nxv56i16_nxv8i16(<vscale x 8 x i16
; ZVBB-RV32-NEXT: vl1re16.v v20, (a3)
; ZVBB-RV32-NEXT: add a3, a3, a0
; ZVBB-RV32-NEXT: vl1re16.v v21, (a3)
-; ZVBB-RV32-NEXT: slli a1, a0, 2
-; ZVBB-RV32-NEXT: slli a0, a0, 3
-; ZVBB-RV32-NEXT: add a1, a0, a1
+; ZVBB-RV32-NEXT: slli a1, a0, 3
; ZVBB-RV32-NEXT: csrr a2, vlenb
; ZVBB-RV32-NEXT: li a3, 14
; ZVBB-RV32-NEXT: mul a2, a2, a3
; ZVBB-RV32-NEXT: add a2, sp, a2
; ZVBB-RV32-NEXT: addi a2, a2, 64
; ZVBB-RV32-NEXT: add a1, a2, a1
-; ZVBB-RV32-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV32-NEXT: add a0, a2, a0
-; ZVBB-RV32-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV32-NEXT: slli a0, a0, 2
+; ZVBB-RV32-NEXT: add a0, a1, a0
+; ZVBB-RV32-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV32-NEXT: vs4r.v v16, (a1)
; ZVBB-RV32-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV32-NEXT: vl8re16.v v16, (a0)
+; ZVBB-RV32-NEXT: vl8re16.v v16, (a1)
; ZVBB-RV32-NEXT: vl8re16.v v8, (a2)
; ZVBB-RV32-NEXT: addi sp, s0, -80
; ZVBB-RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -4595,20 +4587,19 @@ define <vscale x 56 x i16> @vector_interleave_nxv56i16_nxv8i16(<vscale x 8 x i16
; ZVBB-RV64-NEXT: vl1re16.v v20, (a3)
; ZVBB-RV64-NEXT: add a3, a3, a0
; ZVBB-RV64-NEXT: vl1re16.v v21, (a3)
-; ZVBB-RV64-NEXT: slli a1, a0, 2
-; ZVBB-RV64-NEXT: slli a0, a0, 3
-; ZVBB-RV64-NEXT: add a1, a0, a1
+; ZVBB-RV64-NEXT: slli a1, a0, 3
; ZVBB-RV64-NEXT: csrr a2, vlenb
; ZVBB-RV64-NEXT: li a3, 14
; ZVBB-RV64-NEXT: mul a2, a2, a3
; ZVBB-RV64-NEXT: add a2, sp, a2
; ZVBB-RV64-NEXT: addi a2, a2, 64
; ZVBB-RV64-NEXT: add a1, a2, a1
-; ZVBB-RV64-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV64-NEXT: add a0, a2, a0
-; ZVBB-RV64-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV64-NEXT: slli a0, a0, 2
+; ZVBB-RV64-NEXT: add a0, a1, a0
+; ZVBB-RV64-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV64-NEXT: vs4r.v v16, (a1)
; ZVBB-RV64-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV64-NEXT: vl8re16.v v16, (a0)
+; ZVBB-RV64-NEXT: vl8re16.v v16, (a1)
; ZVBB-RV64-NEXT: vl8re16.v v8, (a2)
; ZVBB-RV64-NEXT: addi sp, s0, -80
; ZVBB-RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -4676,20 +4667,19 @@ define <vscale x 56 x i16> @vector_interleave_nxv56i16_nxv8i16(<vscale x 8 x i16
; ZVZIP-NEXT: vl1re16.v v20, (a3)
; ZVZIP-NEXT: add a3, a3, a0
; ZVZIP-NEXT: vl1re16.v v21, (a3)
-; ZVZIP-NEXT: slli a1, a0, 2
-; ZVZIP-NEXT: slli a0, a0, 3
-; ZVZIP-NEXT: add a1, a0, a1
+; ZVZIP-NEXT: slli a1, a0, 3
; ZVZIP-NEXT: csrr a2, vlenb
; ZVZIP-NEXT: li a3, 14
; ZVZIP-NEXT: mul a2, a2, a3
; ZVZIP-NEXT: add a2, sp, a2
; ZVZIP-NEXT: addi a2, a2, 64
; ZVZIP-NEXT: add a1, a2, a1
-; ZVZIP-NEXT: vs2r.v v20, (a1)
-; ZVZIP-NEXT: add a0, a2, a0
-; ZVZIP-NEXT: vs4r.v v16, (a0)
+; ZVZIP-NEXT: slli a0, a0, 2
+; ZVZIP-NEXT: add a0, a1, a0
+; ZVZIP-NEXT: vs2r.v v20, (a0)
+; ZVZIP-NEXT: vs4r.v v16, (a1)
; ZVZIP-NEXT: vs8r.v v8, (a2)
-; ZVZIP-NEXT: vl8re16.v v16, (a0)
+; ZVZIP-NEXT: vl8re16.v v16, (a1)
; ZVZIP-NEXT: vl8re16.v v8, (a2)
; ZVZIP-NEXT: addi sp, s0, -80
; ZVZIP-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -4763,20 +4753,19 @@ define <vscale x 28 x i32> @vector_interleave_nxv28i32_nxv4i32(<vscale x 4 x i32
; RV32-NEXT: vl1re32.v v20, (a3)
; RV32-NEXT: add a3, a3, a0
; RV32-NEXT: vl1re32.v v21, (a3)
-; RV32-NEXT: slli a1, a0, 2
-; RV32-NEXT: slli a0, a0, 3
-; RV32-NEXT: add a1, a0, a1
+; RV32-NEXT: slli a1, a0, 3
; RV32-NEXT: csrr a2, vlenb
; RV32-NEXT: li a3, 14
; RV32-NEXT: mul a2, a2, a3
; RV32-NEXT: add a2, sp, a2
; RV32-NEXT: addi a2, a2, 64
; RV32-NEXT: add a1, a2, a1
-; RV32-NEXT: vs2r.v v20, (a1)
-; RV32-NEXT: add a0, a2, a0
-; RV32-NEXT: vs4r.v v16, (a0)
+; RV32-NEXT: slli a0, a0, 2
+; RV32-NEXT: add a0, a1, a0
+; RV32-NEXT: vs2r.v v20, (a0)
+; RV32-NEXT: vs4r.v v16, (a1)
; RV32-NEXT: vs8r.v v8, (a2)
-; RV32-NEXT: vl8re32.v v16, (a0)
+; RV32-NEXT: vl8re32.v v16, (a1)
; RV32-NEXT: vl8re32.v v8, (a2)
; RV32-NEXT: addi sp, s0, -80
; RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -4844,20 +4833,19 @@ define <vscale x 28 x i32> @vector_interleave_nxv28i32_nxv4i32(<vscale x 4 x i32
; RV64-NEXT: vl1re32.v v20, (a3)
; RV64-NEXT: add a3, a3, a0
; RV64-NEXT: vl1re32.v v21, (a3)
-; RV64-NEXT: slli a1, a0, 2
-; RV64-NEXT: slli a0, a0, 3
-; RV64-NEXT: add a1, a0, a1
+; RV64-NEXT: slli a1, a0, 3
; RV64-NEXT: csrr a2, vlenb
; RV64-NEXT: li a3, 14
; RV64-NEXT: mul a2, a2, a3
; RV64-NEXT: add a2, sp, a2
; RV64-NEXT: addi a2, a2, 64
; RV64-NEXT: add a1, a2, a1
-; RV64-NEXT: vs2r.v v20, (a1)
-; RV64-NEXT: add a0, a2, a0
-; RV64-NEXT: vs4r.v v16, (a0)
+; RV64-NEXT: slli a0, a0, 2
+; RV64-NEXT: add a0, a1, a0
+; RV64-NEXT: vs2r.v v20, (a0)
+; RV64-NEXT: vs4r.v v16, (a1)
; RV64-NEXT: vs8r.v v8, (a2)
-; RV64-NEXT: vl8re32.v v16, (a0)
+; RV64-NEXT: vl8re32.v v16, (a1)
; RV64-NEXT: vl8re32.v v8, (a2)
; RV64-NEXT: addi sp, s0, -80
; RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -4925,20 +4913,19 @@ define <vscale x 28 x i32> @vector_interleave_nxv28i32_nxv4i32(<vscale x 4 x i32
; ZVBB-RV32-NEXT: vl1re32.v v20, (a3)
; ZVBB-RV32-NEXT: add a3, a3, a0
; ZVBB-RV32-NEXT: vl1re32.v v21, (a3)
-; ZVBB-RV32-NEXT: slli a1, a0, 2
-; ZVBB-RV32-NEXT: slli a0, a0, 3
-; ZVBB-RV32-NEXT: add a1, a0, a1
+; ZVBB-RV32-NEXT: slli a1, a0, 3
; ZVBB-RV32-NEXT: csrr a2, vlenb
; ZVBB-RV32-NEXT: li a3, 14
; ZVBB-RV32-NEXT: mul a2, a2, a3
; ZVBB-RV32-NEXT: add a2, sp, a2
; ZVBB-RV32-NEXT: addi a2, a2, 64
; ZVBB-RV32-NEXT: add a1, a2, a1
-; ZVBB-RV32-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV32-NEXT: add a0, a2, a0
-; ZVBB-RV32-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV32-NEXT: slli a0, a0, 2
+; ZVBB-RV32-NEXT: add a0, a1, a0
+; ZVBB-RV32-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV32-NEXT: vs4r.v v16, (a1)
; ZVBB-RV32-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV32-NEXT: vl8re32.v v16, (a0)
+; ZVBB-RV32-NEXT: vl8re32.v v16, (a1)
; ZVBB-RV32-NEXT: vl8re32.v v8, (a2)
; ZVBB-RV32-NEXT: addi sp, s0, -80
; ZVBB-RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -5006,20 +4993,19 @@ define <vscale x 28 x i32> @vector_interleave_nxv28i32_nxv4i32(<vscale x 4 x i32
; ZVBB-RV64-NEXT: vl1re32.v v20, (a3)
; ZVBB-RV64-NEXT: add a3, a3, a0
; ZVBB-RV64-NEXT: vl1re32.v v21, (a3)
-; ZVBB-RV64-NEXT: slli a1, a0, 2
-; ZVBB-RV64-NEXT: slli a0, a0, 3
-; ZVBB-RV64-NEXT: add a1, a0, a1
+; ZVBB-RV64-NEXT: slli a1, a0, 3
; ZVBB-RV64-NEXT: csrr a2, vlenb
; ZVBB-RV64-NEXT: li a3, 14
; ZVBB-RV64-NEXT: mul a2, a2, a3
; ZVBB-RV64-NEXT: add a2, sp, a2
; ZVBB-RV64-NEXT: addi a2, a2, 64
; ZVBB-RV64-NEXT: add a1, a2, a1
-; ZVBB-RV64-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV64-NEXT: add a0, a2, a0
-; ZVBB-RV64-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV64-NEXT: slli a0, a0, 2
+; ZVBB-RV64-NEXT: add a0, a1, a0
+; ZVBB-RV64-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV64-NEXT: vs4r.v v16, (a1)
; ZVBB-RV64-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV64-NEXT: vl8re32.v v16, (a0)
+; ZVBB-RV64-NEXT: vl8re32.v v16, (a1)
; ZVBB-RV64-NEXT: vl8re32.v v8, (a2)
; ZVBB-RV64-NEXT: addi sp, s0, -80
; ZVBB-RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -5087,20 +5073,19 @@ define <vscale x 28 x i32> @vector_interleave_nxv28i32_nxv4i32(<vscale x 4 x i32
; ZVZIP-NEXT: vl1re32.v v20, (a3)
; ZVZIP-NEXT: add a3, a3, a0
; ZVZIP-NEXT: vl1re32.v v21, (a3)
-; ZVZIP-NEXT: slli a1, a0, 2
-; ZVZIP-NEXT: slli a0, a0, 3
-; ZVZIP-NEXT: add a1, a0, a1
+; ZVZIP-NEXT: slli a1, a0, 3
; ZVZIP-NEXT: csrr a2, vlenb
; ZVZIP-NEXT: li a3, 14
; ZVZIP-NEXT: mul a2, a2, a3
; ZVZIP-NEXT: add a2, sp, a2
; ZVZIP-NEXT: addi a2, a2, 64
; ZVZIP-NEXT: add a1, a2, a1
-; ZVZIP-NEXT: vs2r.v v20, (a1)
-; ZVZIP-NEXT: add a0, a2, a0
-; ZVZIP-NEXT: vs4r.v v16, (a0)
+; ZVZIP-NEXT: slli a0, a0, 2
+; ZVZIP-NEXT: add a0, a1, a0
+; ZVZIP-NEXT: vs2r.v v20, (a0)
+; ZVZIP-NEXT: vs4r.v v16, (a1)
; ZVZIP-NEXT: vs8r.v v8, (a2)
-; ZVZIP-NEXT: vl8re32.v v16, (a0)
+; ZVZIP-NEXT: vl8re32.v v16, (a1)
; ZVZIP-NEXT: vl8re32.v v8, (a2)
; ZVZIP-NEXT: addi sp, s0, -80
; ZVZIP-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -5173,20 +5158,19 @@ define <vscale x 14 x i64> @vector_interleave_nxv14i64_nxv2i64(<vscale x 2 x i64
; RV32-NEXT: vl1re64.v v20, (a3)
; RV32-NEXT: add a3, a3, a0
; RV32-NEXT: vl1re64.v v21, (a3)
-; RV32-NEXT: slli a1, a0, 2
-; RV32-NEXT: slli a0, a0, 3
-; RV32-NEXT: add a1, a0, a1
+; RV32-NEXT: slli a1, a0, 3
; RV32-NEXT: csrr a2, vlenb
; RV32-NEXT: li a3, 14
; RV32-NEXT: mul a2, a2, a3
; RV32-NEXT: add a2, sp, a2
; RV32-NEXT: addi a2, a2, 64
; RV32-NEXT: add a1, a2, a1
-; RV32-NEXT: vs2r.v v20, (a1)
-; RV32-NEXT: add a0, a2, a0
-; RV32-NEXT: vs4r.v v16, (a0)
+; RV32-NEXT: slli a0, a0, 2
+; RV32-NEXT: add a0, a1, a0
+; RV32-NEXT: vs2r.v v20, (a0)
+; RV32-NEXT: vs4r.v v16, (a1)
; RV32-NEXT: vs8r.v v8, (a2)
-; RV32-NEXT: vl8re64.v v16, (a0)
+; RV32-NEXT: vl8re64.v v16, (a1)
; RV32-NEXT: vl8re64.v v8, (a2)
; RV32-NEXT: addi sp, s0, -80
; RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -5254,20 +5238,19 @@ define <vscale x 14 x i64> @vector_interleave_nxv14i64_nxv2i64(<vscale x 2 x i64
; RV64-NEXT: vl1re64.v v20, (a3)
; RV64-NEXT: add a3, a3, a0
; RV64-NEXT: vl1re64.v v21, (a3)
-; RV64-NEXT: slli a1, a0, 2
-; RV64-NEXT: slli a0, a0, 3
-; RV64-NEXT: add a1, a0, a1
+; RV64-NEXT: slli a1, a0, 3
; RV64-NEXT: csrr a2, vlenb
; RV64-NEXT: li a3, 14
; RV64-NEXT: mul a2, a2, a3
; RV64-NEXT: add a2, sp, a2
; RV64-NEXT: addi a2, a2, 64
; RV64-NEXT: add a1, a2, a1
-; RV64-NEXT: vs2r.v v20, (a1)
-; RV64-NEXT: add a0, a2, a0
-; RV64-NEXT: vs4r.v v16, (a0)
+; RV64-NEXT: slli a0, a0, 2
+; RV64-NEXT: add a0, a1, a0
+; RV64-NEXT: vs2r.v v20, (a0)
+; RV64-NEXT: vs4r.v v16, (a1)
; RV64-NEXT: vs8r.v v8, (a2)
-; RV64-NEXT: vl8re64.v v16, (a0)
+; RV64-NEXT: vl8re64.v v16, (a1)
; RV64-NEXT: vl8re64.v v8, (a2)
; RV64-NEXT: addi sp, s0, -80
; RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -5335,20 +5318,19 @@ define <vscale x 14 x i64> @vector_interleave_nxv14i64_nxv2i64(<vscale x 2 x i64
; ZVBB-RV32-NEXT: vl1re64.v v20, (a3)
; ZVBB-RV32-NEXT: add a3, a3, a0
; ZVBB-RV32-NEXT: vl1re64.v v21, (a3)
-; ZVBB-RV32-NEXT: slli a1, a0, 2
-; ZVBB-RV32-NEXT: slli a0, a0, 3
-; ZVBB-RV32-NEXT: add a1, a0, a1
+; ZVBB-RV32-NEXT: slli a1, a0, 3
; ZVBB-RV32-NEXT: csrr a2, vlenb
; ZVBB-RV32-NEXT: li a3, 14
; ZVBB-RV32-NEXT: mul a2, a2, a3
; ZVBB-RV32-NEXT: add a2, sp, a2
; ZVBB-RV32-NEXT: addi a2, a2, 64
; ZVBB-RV32-NEXT: add a1, a2, a1
-; ZVBB-RV32-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV32-NEXT: add a0, a2, a0
-; ZVBB-RV32-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV32-NEXT: slli a0, a0, 2
+; ZVBB-RV32-NEXT: add a0, a1, a0
+; ZVBB-RV32-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV32-NEXT: vs4r.v v16, (a1)
; ZVBB-RV32-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV32-NEXT: vl8re64.v v16, (a0)
+; ZVBB-RV32-NEXT: vl8re64.v v16, (a1)
; ZVBB-RV32-NEXT: vl8re64.v v8, (a2)
; ZVBB-RV32-NEXT: addi sp, s0, -80
; ZVBB-RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -5416,20 +5398,19 @@ define <vscale x 14 x i64> @vector_interleave_nxv14i64_nxv2i64(<vscale x 2 x i64
; ZVBB-RV64-NEXT: vl1re64.v v20, (a3)
; ZVBB-RV64-NEXT: add a3, a3, a0
; ZVBB-RV64-NEXT: vl1re64.v v21, (a3)
-; ZVBB-RV64-NEXT: slli a1, a0, 2
-; ZVBB-RV64-NEXT: slli a0, a0, 3
-; ZVBB-RV64-NEXT: add a1, a0, a1
+; ZVBB-RV64-NEXT: slli a1, a0, 3
; ZVBB-RV64-NEXT: csrr a2, vlenb
; ZVBB-RV64-NEXT: li a3, 14
; ZVBB-RV64-NEXT: mul a2, a2, a3
; ZVBB-RV64-NEXT: add a2, sp, a2
; ZVBB-RV64-NEXT: addi a2, a2, 64
; ZVBB-RV64-NEXT: add a1, a2, a1
-; ZVBB-RV64-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV64-NEXT: add a0, a2, a0
-; ZVBB-RV64-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV64-NEXT: slli a0, a0, 2
+; ZVBB-RV64-NEXT: add a0, a1, a0
+; ZVBB-RV64-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV64-NEXT: vs4r.v v16, (a1)
; ZVBB-RV64-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV64-NEXT: vl8re64.v v16, (a0)
+; ZVBB-RV64-NEXT: vl8re64.v v16, (a1)
; ZVBB-RV64-NEXT: vl8re64.v v8, (a2)
; ZVBB-RV64-NEXT: addi sp, s0, -80
; ZVBB-RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -5497,20 +5478,19 @@ define <vscale x 14 x i64> @vector_interleave_nxv14i64_nxv2i64(<vscale x 2 x i64
; ZVZIP-NEXT: vl1re64.v v20, (a3)
; ZVZIP-NEXT: add a3, a3, a0
; ZVZIP-NEXT: vl1re64.v v21, (a3)
-; ZVZIP-NEXT: slli a1, a0, 2
-; ZVZIP-NEXT: slli a0, a0, 3
-; ZVZIP-NEXT: add a1, a0, a1
+; ZVZIP-NEXT: slli a1, a0, 3
; ZVZIP-NEXT: csrr a2, vlenb
; ZVZIP-NEXT: li a3, 14
; ZVZIP-NEXT: mul a2, a2, a3
; ZVZIP-NEXT: add a2, sp, a2
; ZVZIP-NEXT: addi a2, a2, 64
; ZVZIP-NEXT: add a1, a2, a1
-; ZVZIP-NEXT: vs2r.v v20, (a1)
-; ZVZIP-NEXT: add a0, a2, a0
-; ZVZIP-NEXT: vs4r.v v16, (a0)
+; ZVZIP-NEXT: slli a0, a0, 2
+; ZVZIP-NEXT: add a0, a1, a0
+; ZVZIP-NEXT: vs2r.v v20, (a0)
+; ZVZIP-NEXT: vs4r.v v16, (a1)
; ZVZIP-NEXT: vs8r.v v8, (a2)
-; ZVZIP-NEXT: vl8re64.v v16, (a0)
+; ZVZIP-NEXT: vl8re64.v v16, (a1)
; ZVZIP-NEXT: vl8re64.v v8, (a2)
; ZVZIP-NEXT: addi sp, s0, -80
; ZVZIP-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -12150,20 +12130,19 @@ define <vscale x 56 x half> @vector_interleave_nxv56f16_nxv8f16(<vscale x 8 x ha
; RV32-NEXT: vl1re16.v v20, (a3)
; RV32-NEXT: add a3, a3, a0
; RV32-NEXT: vl1re16.v v21, (a3)
-; RV32-NEXT: slli a1, a0, 2
-; RV32-NEXT: slli a0, a0, 3
-; RV32-NEXT: add a1, a0, a1
+; RV32-NEXT: slli a1, a0, 3
; RV32-NEXT: csrr a2, vlenb
; RV32-NEXT: li a3, 14
; RV32-NEXT: mul a2, a2, a3
; RV32-NEXT: add a2, sp, a2
; RV32-NEXT: addi a2, a2, 64
; RV32-NEXT: add a1, a2, a1
-; RV32-NEXT: vs2r.v v20, (a1)
-; RV32-NEXT: add a0, a2, a0
-; RV32-NEXT: vs4r.v v16, (a0)
+; RV32-NEXT: slli a0, a0, 2
+; RV32-NEXT: add a0, a1, a0
+; RV32-NEXT: vs2r.v v20, (a0)
+; RV32-NEXT: vs4r.v v16, (a1)
; RV32-NEXT: vs8r.v v8, (a2)
-; RV32-NEXT: vl8re16.v v16, (a0)
+; RV32-NEXT: vl8re16.v v16, (a1)
; RV32-NEXT: vl8re16.v v8, (a2)
; RV32-NEXT: addi sp, s0, -80
; RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -12231,20 +12210,19 @@ define <vscale x 56 x half> @vector_interleave_nxv56f16_nxv8f16(<vscale x 8 x ha
; RV64-NEXT: vl1re16.v v20, (a3)
; RV64-NEXT: add a3, a3, a0
; RV64-NEXT: vl1re16.v v21, (a3)
-; RV64-NEXT: slli a1, a0, 2
-; RV64-NEXT: slli a0, a0, 3
-; RV64-NEXT: add a1, a0, a1
+; RV64-NEXT: slli a1, a0, 3
; RV64-NEXT: csrr a2, vlenb
; RV64-NEXT: li a3, 14
; RV64-NEXT: mul a2, a2, a3
; RV64-NEXT: add a2, sp, a2
; RV64-NEXT: addi a2, a2, 64
; RV64-NEXT: add a1, a2, a1
-; RV64-NEXT: vs2r.v v20, (a1)
-; RV64-NEXT: add a0, a2, a0
-; RV64-NEXT: vs4r.v v16, (a0)
+; RV64-NEXT: slli a0, a0, 2
+; RV64-NEXT: add a0, a1, a0
+; RV64-NEXT: vs2r.v v20, (a0)
+; RV64-NEXT: vs4r.v v16, (a1)
; RV64-NEXT: vs8r.v v8, (a2)
-; RV64-NEXT: vl8re16.v v16, (a0)
+; RV64-NEXT: vl8re16.v v16, (a1)
; RV64-NEXT: vl8re16.v v8, (a2)
; RV64-NEXT: addi sp, s0, -80
; RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -12312,20 +12290,19 @@ define <vscale x 56 x half> @vector_interleave_nxv56f16_nxv8f16(<vscale x 8 x ha
; ZVBB-RV32-NEXT: vl1re16.v v20, (a3)
; ZVBB-RV32-NEXT: add a3, a3, a0
; ZVBB-RV32-NEXT: vl1re16.v v21, (a3)
-; ZVBB-RV32-NEXT: slli a1, a0, 2
-; ZVBB-RV32-NEXT: slli a0, a0, 3
-; ZVBB-RV32-NEXT: add a1, a0, a1
+; ZVBB-RV32-NEXT: slli a1, a0, 3
; ZVBB-RV32-NEXT: csrr a2, vlenb
; ZVBB-RV32-NEXT: li a3, 14
; ZVBB-RV32-NEXT: mul a2, a2, a3
; ZVBB-RV32-NEXT: add a2, sp, a2
; ZVBB-RV32-NEXT: addi a2, a2, 64
; ZVBB-RV32-NEXT: add a1, a2, a1
-; ZVBB-RV32-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV32-NEXT: add a0, a2, a0
-; ZVBB-RV32-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV32-NEXT: slli a0, a0, 2
+; ZVBB-RV32-NEXT: add a0, a1, a0
+; ZVBB-RV32-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV32-NEXT: vs4r.v v16, (a1)
; ZVBB-RV32-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV32-NEXT: vl8re16.v v16, (a0)
+; ZVBB-RV32-NEXT: vl8re16.v v16, (a1)
; ZVBB-RV32-NEXT: vl8re16.v v8, (a2)
; ZVBB-RV32-NEXT: addi sp, s0, -80
; ZVBB-RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -12393,20 +12370,19 @@ define <vscale x 56 x half> @vector_interleave_nxv56f16_nxv8f16(<vscale x 8 x ha
; ZVBB-RV64-NEXT: vl1re16.v v20, (a3)
; ZVBB-RV64-NEXT: add a3, a3, a0
; ZVBB-RV64-NEXT: vl1re16.v v21, (a3)
-; ZVBB-RV64-NEXT: slli a1, a0, 2
-; ZVBB-RV64-NEXT: slli a0, a0, 3
-; ZVBB-RV64-NEXT: add a1, a0, a1
+; ZVBB-RV64-NEXT: slli a1, a0, 3
; ZVBB-RV64-NEXT: csrr a2, vlenb
; ZVBB-RV64-NEXT: li a3, 14
; ZVBB-RV64-NEXT: mul a2, a2, a3
; ZVBB-RV64-NEXT: add a2, sp, a2
; ZVBB-RV64-NEXT: addi a2, a2, 64
; ZVBB-RV64-NEXT: add a1, a2, a1
-; ZVBB-RV64-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV64-NEXT: add a0, a2, a0
-; ZVBB-RV64-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV64-NEXT: slli a0, a0, 2
+; ZVBB-RV64-NEXT: add a0, a1, a0
+; ZVBB-RV64-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV64-NEXT: vs4r.v v16, (a1)
; ZVBB-RV64-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV64-NEXT: vl8re16.v v16, (a0)
+; ZVBB-RV64-NEXT: vl8re16.v v16, (a1)
; ZVBB-RV64-NEXT: vl8re16.v v8, (a2)
; ZVBB-RV64-NEXT: addi sp, s0, -80
; ZVBB-RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -12474,20 +12450,19 @@ define <vscale x 56 x half> @vector_interleave_nxv56f16_nxv8f16(<vscale x 8 x ha
; ZVZIP-NEXT: vl1re16.v v20, (a3)
; ZVZIP-NEXT: add a3, a3, a0
; ZVZIP-NEXT: vl1re16.v v21, (a3)
-; ZVZIP-NEXT: slli a1, a0, 2
-; ZVZIP-NEXT: slli a0, a0, 3
-; ZVZIP-NEXT: add a1, a0, a1
+; ZVZIP-NEXT: slli a1, a0, 3
; ZVZIP-NEXT: csrr a2, vlenb
; ZVZIP-NEXT: li a3, 14
; ZVZIP-NEXT: mul a2, a2, a3
; ZVZIP-NEXT: add a2, sp, a2
; ZVZIP-NEXT: addi a2, a2, 64
; ZVZIP-NEXT: add a1, a2, a1
-; ZVZIP-NEXT: vs2r.v v20, (a1)
-; ZVZIP-NEXT: add a0, a2, a0
-; ZVZIP-NEXT: vs4r.v v16, (a0)
+; ZVZIP-NEXT: slli a0, a0, 2
+; ZVZIP-NEXT: add a0, a1, a0
+; ZVZIP-NEXT: vs2r.v v20, (a0)
+; ZVZIP-NEXT: vs4r.v v16, (a1)
; ZVZIP-NEXT: vs8r.v v8, (a2)
-; ZVZIP-NEXT: vl8re16.v v16, (a0)
+; ZVZIP-NEXT: vl8re16.v v16, (a1)
; ZVZIP-NEXT: vl8re16.v v8, (a2)
; ZVZIP-NEXT: addi sp, s0, -80
; ZVZIP-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -12707,20 +12682,19 @@ define <vscale x 56 x bfloat> @vector_interleave_nxv56bf16_nxv8bf16(<vscale x 8
; RV32-NEXT: vl1re16.v v20, (a3)
; RV32-NEXT: add a3, a3, a0
; RV32-NEXT: vl1re16.v v21, (a3)
-; RV32-NEXT: slli a1, a0, 2
-; RV32-NEXT: slli a0, a0, 3
-; RV32-NEXT: add a1, a0, a1
+; RV32-NEXT: slli a1, a0, 3
; RV32-NEXT: csrr a2, vlenb
; RV32-NEXT: li a3, 14
; RV32-NEXT: mul a2, a2, a3
; RV32-NEXT: add a2, sp, a2
; RV32-NEXT: addi a2, a2, 64
; RV32-NEXT: add a1, a2, a1
-; RV32-NEXT: vs2r.v v20, (a1)
-; RV32-NEXT: add a0, a2, a0
-; RV32-NEXT: vs4r.v v16, (a0)
+; RV32-NEXT: slli a0, a0, 2
+; RV32-NEXT: add a0, a1, a0
+; RV32-NEXT: vs2r.v v20, (a0)
+; RV32-NEXT: vs4r.v v16, (a1)
; RV32-NEXT: vs8r.v v8, (a2)
-; RV32-NEXT: vl8re16.v v16, (a0)
+; RV32-NEXT: vl8re16.v v16, (a1)
; RV32-NEXT: vl8re16.v v8, (a2)
; RV32-NEXT: addi sp, s0, -80
; RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -12788,20 +12762,19 @@ define <vscale x 56 x bfloat> @vector_interleave_nxv56bf16_nxv8bf16(<vscale x 8
; RV64-NEXT: vl1re16.v v20, (a3)
; RV64-NEXT: add a3, a3, a0
; RV64-NEXT: vl1re16.v v21, (a3)
-; RV64-NEXT: slli a1, a0, 2
-; RV64-NEXT: slli a0, a0, 3
-; RV64-NEXT: add a1, a0, a1
+; RV64-NEXT: slli a1, a0, 3
; RV64-NEXT: csrr a2, vlenb
; RV64-NEXT: li a3, 14
; RV64-NEXT: mul a2, a2, a3
; RV64-NEXT: add a2, sp, a2
; RV64-NEXT: addi a2, a2, 64
; RV64-NEXT: add a1, a2, a1
-; RV64-NEXT: vs2r.v v20, (a1)
-; RV64-NEXT: add a0, a2, a0
-; RV64-NEXT: vs4r.v v16, (a0)
+; RV64-NEXT: slli a0, a0, 2
+; RV64-NEXT: add a0, a1, a0
+; RV64-NEXT: vs2r.v v20, (a0)
+; RV64-NEXT: vs4r.v v16, (a1)
; RV64-NEXT: vs8r.v v8, (a2)
-; RV64-NEXT: vl8re16.v v16, (a0)
+; RV64-NEXT: vl8re16.v v16, (a1)
; RV64-NEXT: vl8re16.v v8, (a2)
; RV64-NEXT: addi sp, s0, -80
; RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -12869,20 +12842,19 @@ define <vscale x 56 x bfloat> @vector_interleave_nxv56bf16_nxv8bf16(<vscale x 8
; ZVBB-RV32-NEXT: vl1re16.v v20, (a3)
; ZVBB-RV32-NEXT: add a3, a3, a0
; ZVBB-RV32-NEXT: vl1re16.v v21, (a3)
-; ZVBB-RV32-NEXT: slli a1, a0, 2
-; ZVBB-RV32-NEXT: slli a0, a0, 3
-; ZVBB-RV32-NEXT: add a1, a0, a1
+; ZVBB-RV32-NEXT: slli a1, a0, 3
; ZVBB-RV32-NEXT: csrr a2, vlenb
; ZVBB-RV32-NEXT: li a3, 14
; ZVBB-RV32-NEXT: mul a2, a2, a3
; ZVBB-RV32-NEXT: add a2, sp, a2
; ZVBB-RV32-NEXT: addi a2, a2, 64
; ZVBB-RV32-NEXT: add a1, a2, a1
-; ZVBB-RV32-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV32-NEXT: add a0, a2, a0
-; ZVBB-RV32-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV32-NEXT: slli a0, a0, 2
+; ZVBB-RV32-NEXT: add a0, a1, a0
+; ZVBB-RV32-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV32-NEXT: vs4r.v v16, (a1)
; ZVBB-RV32-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV32-NEXT: vl8re16.v v16, (a0)
+; ZVBB-RV32-NEXT: vl8re16.v v16, (a1)
; ZVBB-RV32-NEXT: vl8re16.v v8, (a2)
; ZVBB-RV32-NEXT: addi sp, s0, -80
; ZVBB-RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -12950,20 +12922,19 @@ define <vscale x 56 x bfloat> @vector_interleave_nxv56bf16_nxv8bf16(<vscale x 8
; ZVBB-RV64-NEXT: vl1re16.v v20, (a3)
; ZVBB-RV64-NEXT: add a3, a3, a0
; ZVBB-RV64-NEXT: vl1re16.v v21, (a3)
-; ZVBB-RV64-NEXT: slli a1, a0, 2
-; ZVBB-RV64-NEXT: slli a0, a0, 3
-; ZVBB-RV64-NEXT: add a1, a0, a1
+; ZVBB-RV64-NEXT: slli a1, a0, 3
; ZVBB-RV64-NEXT: csrr a2, vlenb
; ZVBB-RV64-NEXT: li a3, 14
; ZVBB-RV64-NEXT: mul a2, a2, a3
; ZVBB-RV64-NEXT: add a2, sp, a2
; ZVBB-RV64-NEXT: addi a2, a2, 64
; ZVBB-RV64-NEXT: add a1, a2, a1
-; ZVBB-RV64-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV64-NEXT: add a0, a2, a0
-; ZVBB-RV64-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV64-NEXT: slli a0, a0, 2
+; ZVBB-RV64-NEXT: add a0, a1, a0
+; ZVBB-RV64-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV64-NEXT: vs4r.v v16, (a1)
; ZVBB-RV64-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV64-NEXT: vl8re16.v v16, (a0)
+; ZVBB-RV64-NEXT: vl8re16.v v16, (a1)
; ZVBB-RV64-NEXT: vl8re16.v v8, (a2)
; ZVBB-RV64-NEXT: addi sp, s0, -80
; ZVBB-RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -13031,20 +13002,19 @@ define <vscale x 56 x bfloat> @vector_interleave_nxv56bf16_nxv8bf16(<vscale x 8
; ZVZIP-NEXT: vl1re16.v v20, (a3)
; ZVZIP-NEXT: add a3, a3, a0
; ZVZIP-NEXT: vl1re16.v v21, (a3)
-; ZVZIP-NEXT: slli a1, a0, 2
-; ZVZIP-NEXT: slli a0, a0, 3
-; ZVZIP-NEXT: add a1, a0, a1
+; ZVZIP-NEXT: slli a1, a0, 3
; ZVZIP-NEXT: csrr a2, vlenb
; ZVZIP-NEXT: li a3, 14
; ZVZIP-NEXT: mul a2, a2, a3
; ZVZIP-NEXT: add a2, sp, a2
; ZVZIP-NEXT: addi a2, a2, 64
; ZVZIP-NEXT: add a1, a2, a1
-; ZVZIP-NEXT: vs2r.v v20, (a1)
-; ZVZIP-NEXT: add a0, a2, a0
-; ZVZIP-NEXT: vs4r.v v16, (a0)
+; ZVZIP-NEXT: slli a0, a0, 2
+; ZVZIP-NEXT: add a0, a1, a0
+; ZVZIP-NEXT: vs2r.v v20, (a0)
+; ZVZIP-NEXT: vs4r.v v16, (a1)
; ZVZIP-NEXT: vs8r.v v8, (a2)
-; ZVZIP-NEXT: vl8re16.v v16, (a0)
+; ZVZIP-NEXT: vl8re16.v v16, (a1)
; ZVZIP-NEXT: vl8re16.v v8, (a2)
; ZVZIP-NEXT: addi sp, s0, -80
; ZVZIP-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -13264,20 +13234,19 @@ define <vscale x 28 x float> @vector_interleave_nxv28f32_nxv4f32(<vscale x 4 x f
; RV32-NEXT: vl1re32.v v20, (a3)
; RV32-NEXT: add a3, a3, a0
; RV32-NEXT: vl1re32.v v21, (a3)
-; RV32-NEXT: slli a1, a0, 2
-; RV32-NEXT: slli a0, a0, 3
-; RV32-NEXT: add a1, a0, a1
+; RV32-NEXT: slli a1, a0, 3
; RV32-NEXT: csrr a2, vlenb
; RV32-NEXT: li a3, 14
; RV32-NEXT: mul a2, a2, a3
; RV32-NEXT: add a2, sp, a2
; RV32-NEXT: addi a2, a2, 64
; RV32-NEXT: add a1, a2, a1
-; RV32-NEXT: vs2r.v v20, (a1)
-; RV32-NEXT: add a0, a2, a0
-; RV32-NEXT: vs4r.v v16, (a0)
+; RV32-NEXT: slli a0, a0, 2
+; RV32-NEXT: add a0, a1, a0
+; RV32-NEXT: vs2r.v v20, (a0)
+; RV32-NEXT: vs4r.v v16, (a1)
; RV32-NEXT: vs8r.v v8, (a2)
-; RV32-NEXT: vl8re32.v v16, (a0)
+; RV32-NEXT: vl8re32.v v16, (a1)
; RV32-NEXT: vl8re32.v v8, (a2)
; RV32-NEXT: addi sp, s0, -80
; RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -13345,20 +13314,19 @@ define <vscale x 28 x float> @vector_interleave_nxv28f32_nxv4f32(<vscale x 4 x f
; RV64-NEXT: vl1re32.v v20, (a3)
; RV64-NEXT: add a3, a3, a0
; RV64-NEXT: vl1re32.v v21, (a3)
-; RV64-NEXT: slli a1, a0, 2
-; RV64-NEXT: slli a0, a0, 3
-; RV64-NEXT: add a1, a0, a1
+; RV64-NEXT: slli a1, a0, 3
; RV64-NEXT: csrr a2, vlenb
; RV64-NEXT: li a3, 14
; RV64-NEXT: mul a2, a2, a3
; RV64-NEXT: add a2, sp, a2
; RV64-NEXT: addi a2, a2, 64
; RV64-NEXT: add a1, a2, a1
-; RV64-NEXT: vs2r.v v20, (a1)
-; RV64-NEXT: add a0, a2, a0
-; RV64-NEXT: vs4r.v v16, (a0)
+; RV64-NEXT: slli a0, a0, 2
+; RV64-NEXT: add a0, a1, a0
+; RV64-NEXT: vs2r.v v20, (a0)
+; RV64-NEXT: vs4r.v v16, (a1)
; RV64-NEXT: vs8r.v v8, (a2)
-; RV64-NEXT: vl8re32.v v16, (a0)
+; RV64-NEXT: vl8re32.v v16, (a1)
; RV64-NEXT: vl8re32.v v8, (a2)
; RV64-NEXT: addi sp, s0, -80
; RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -13426,20 +13394,19 @@ define <vscale x 28 x float> @vector_interleave_nxv28f32_nxv4f32(<vscale x 4 x f
; ZVBB-RV32-NEXT: vl1re32.v v20, (a3)
; ZVBB-RV32-NEXT: add a3, a3, a0
; ZVBB-RV32-NEXT: vl1re32.v v21, (a3)
-; ZVBB-RV32-NEXT: slli a1, a0, 2
-; ZVBB-RV32-NEXT: slli a0, a0, 3
-; ZVBB-RV32-NEXT: add a1, a0, a1
+; ZVBB-RV32-NEXT: slli a1, a0, 3
; ZVBB-RV32-NEXT: csrr a2, vlenb
; ZVBB-RV32-NEXT: li a3, 14
; ZVBB-RV32-NEXT: mul a2, a2, a3
; ZVBB-RV32-NEXT: add a2, sp, a2
; ZVBB-RV32-NEXT: addi a2, a2, 64
; ZVBB-RV32-NEXT: add a1, a2, a1
-; ZVBB-RV32-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV32-NEXT: add a0, a2, a0
-; ZVBB-RV32-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV32-NEXT: slli a0, a0, 2
+; ZVBB-RV32-NEXT: add a0, a1, a0
+; ZVBB-RV32-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV32-NEXT: vs4r.v v16, (a1)
; ZVBB-RV32-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV32-NEXT: vl8re32.v v16, (a0)
+; ZVBB-RV32-NEXT: vl8re32.v v16, (a1)
; ZVBB-RV32-NEXT: vl8re32.v v8, (a2)
; ZVBB-RV32-NEXT: addi sp, s0, -80
; ZVBB-RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -13507,20 +13474,19 @@ define <vscale x 28 x float> @vector_interleave_nxv28f32_nxv4f32(<vscale x 4 x f
; ZVBB-RV64-NEXT: vl1re32.v v20, (a3)
; ZVBB-RV64-NEXT: add a3, a3, a0
; ZVBB-RV64-NEXT: vl1re32.v v21, (a3)
-; ZVBB-RV64-NEXT: slli a1, a0, 2
-; ZVBB-RV64-NEXT: slli a0, a0, 3
-; ZVBB-RV64-NEXT: add a1, a0, a1
+; ZVBB-RV64-NEXT: slli a1, a0, 3
; ZVBB-RV64-NEXT: csrr a2, vlenb
; ZVBB-RV64-NEXT: li a3, 14
; ZVBB-RV64-NEXT: mul a2, a2, a3
; ZVBB-RV64-NEXT: add a2, sp, a2
; ZVBB-RV64-NEXT: addi a2, a2, 64
; ZVBB-RV64-NEXT: add a1, a2, a1
-; ZVBB-RV64-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV64-NEXT: add a0, a2, a0
-; ZVBB-RV64-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV64-NEXT: slli a0, a0, 2
+; ZVBB-RV64-NEXT: add a0, a1, a0
+; ZVBB-RV64-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV64-NEXT: vs4r.v v16, (a1)
; ZVBB-RV64-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV64-NEXT: vl8re32.v v16, (a0)
+; ZVBB-RV64-NEXT: vl8re32.v v16, (a1)
; ZVBB-RV64-NEXT: vl8re32.v v8, (a2)
; ZVBB-RV64-NEXT: addi sp, s0, -80
; ZVBB-RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -13588,20 +13554,19 @@ define <vscale x 28 x float> @vector_interleave_nxv28f32_nxv4f32(<vscale x 4 x f
; ZVZIP-NEXT: vl1re32.v v20, (a3)
; ZVZIP-NEXT: add a3, a3, a0
; ZVZIP-NEXT: vl1re32.v v21, (a3)
-; ZVZIP-NEXT: slli a1, a0, 2
-; ZVZIP-NEXT: slli a0, a0, 3
-; ZVZIP-NEXT: add a1, a0, a1
+; ZVZIP-NEXT: slli a1, a0, 3
; ZVZIP-NEXT: csrr a2, vlenb
; ZVZIP-NEXT: li a3, 14
; ZVZIP-NEXT: mul a2, a2, a3
; ZVZIP-NEXT: add a2, sp, a2
; ZVZIP-NEXT: addi a2, a2, 64
; ZVZIP-NEXT: add a1, a2, a1
-; ZVZIP-NEXT: vs2r.v v20, (a1)
-; ZVZIP-NEXT: add a0, a2, a0
-; ZVZIP-NEXT: vs4r.v v16, (a0)
+; ZVZIP-NEXT: slli a0, a0, 2
+; ZVZIP-NEXT: add a0, a1, a0
+; ZVZIP-NEXT: vs2r.v v20, (a0)
+; ZVZIP-NEXT: vs4r.v v16, (a1)
; ZVZIP-NEXT: vs8r.v v8, (a2)
-; ZVZIP-NEXT: vl8re32.v v16, (a0)
+; ZVZIP-NEXT: vl8re32.v v16, (a1)
; ZVZIP-NEXT: vl8re32.v v8, (a2)
; ZVZIP-NEXT: addi sp, s0, -80
; ZVZIP-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -13739,20 +13704,19 @@ define <vscale x 14 x double> @vector_interleave_nxv14f64_nxv2f64(<vscale x 2 x
; RV32-NEXT: vl1re64.v v20, (a3)
; RV32-NEXT: add a3, a3, a0
; RV32-NEXT: vl1re64.v v21, (a3)
-; RV32-NEXT: slli a1, a0, 2
-; RV32-NEXT: slli a0, a0, 3
-; RV32-NEXT: add a1, a0, a1
+; RV32-NEXT: slli a1, a0, 3
; RV32-NEXT: csrr a2, vlenb
; RV32-NEXT: li a3, 14
; RV32-NEXT: mul a2, a2, a3
; RV32-NEXT: add a2, sp, a2
; RV32-NEXT: addi a2, a2, 64
; RV32-NEXT: add a1, a2, a1
-; RV32-NEXT: vs2r.v v20, (a1)
-; RV32-NEXT: add a0, a2, a0
-; RV32-NEXT: vs4r.v v16, (a0)
+; RV32-NEXT: slli a0, a0, 2
+; RV32-NEXT: add a0, a1, a0
+; RV32-NEXT: vs2r.v v20, (a0)
+; RV32-NEXT: vs4r.v v16, (a1)
; RV32-NEXT: vs8r.v v8, (a2)
-; RV32-NEXT: vl8re64.v v16, (a0)
+; RV32-NEXT: vl8re64.v v16, (a1)
; RV32-NEXT: vl8re64.v v8, (a2)
; RV32-NEXT: addi sp, s0, -80
; RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -13820,20 +13784,19 @@ define <vscale x 14 x double> @vector_interleave_nxv14f64_nxv2f64(<vscale x 2 x
; RV64-NEXT: vl1re64.v v20, (a3)
; RV64-NEXT: add a3, a3, a0
; RV64-NEXT: vl1re64.v v21, (a3)
-; RV64-NEXT: slli a1, a0, 2
-; RV64-NEXT: slli a0, a0, 3
-; RV64-NEXT: add a1, a0, a1
+; RV64-NEXT: slli a1, a0, 3
; RV64-NEXT: csrr a2, vlenb
; RV64-NEXT: li a3, 14
; RV64-NEXT: mul a2, a2, a3
; RV64-NEXT: add a2, sp, a2
; RV64-NEXT: addi a2, a2, 64
; RV64-NEXT: add a1, a2, a1
-; RV64-NEXT: vs2r.v v20, (a1)
-; RV64-NEXT: add a0, a2, a0
-; RV64-NEXT: vs4r.v v16, (a0)
+; RV64-NEXT: slli a0, a0, 2
+; RV64-NEXT: add a0, a1, a0
+; RV64-NEXT: vs2r.v v20, (a0)
+; RV64-NEXT: vs4r.v v16, (a1)
; RV64-NEXT: vs8r.v v8, (a2)
-; RV64-NEXT: vl8re64.v v16, (a0)
+; RV64-NEXT: vl8re64.v v16, (a1)
; RV64-NEXT: vl8re64.v v8, (a2)
; RV64-NEXT: addi sp, s0, -80
; RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -13901,20 +13864,19 @@ define <vscale x 14 x double> @vector_interleave_nxv14f64_nxv2f64(<vscale x 2 x
; ZVBB-RV32-NEXT: vl1re64.v v20, (a3)
; ZVBB-RV32-NEXT: add a3, a3, a0
; ZVBB-RV32-NEXT: vl1re64.v v21, (a3)
-; ZVBB-RV32-NEXT: slli a1, a0, 2
-; ZVBB-RV32-NEXT: slli a0, a0, 3
-; ZVBB-RV32-NEXT: add a1, a0, a1
+; ZVBB-RV32-NEXT: slli a1, a0, 3
; ZVBB-RV32-NEXT: csrr a2, vlenb
; ZVBB-RV32-NEXT: li a3, 14
; ZVBB-RV32-NEXT: mul a2, a2, a3
; ZVBB-RV32-NEXT: add a2, sp, a2
; ZVBB-RV32-NEXT: addi a2, a2, 64
; ZVBB-RV32-NEXT: add a1, a2, a1
-; ZVBB-RV32-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV32-NEXT: add a0, a2, a0
-; ZVBB-RV32-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV32-NEXT: slli a0, a0, 2
+; ZVBB-RV32-NEXT: add a0, a1, a0
+; ZVBB-RV32-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV32-NEXT: vs4r.v v16, (a1)
; ZVBB-RV32-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV32-NEXT: vl8re64.v v16, (a0)
+; ZVBB-RV32-NEXT: vl8re64.v v16, (a1)
; ZVBB-RV32-NEXT: vl8re64.v v8, (a2)
; ZVBB-RV32-NEXT: addi sp, s0, -80
; ZVBB-RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
@@ -13982,20 +13944,19 @@ define <vscale x 14 x double> @vector_interleave_nxv14f64_nxv2f64(<vscale x 2 x
; ZVBB-RV64-NEXT: vl1re64.v v20, (a3)
; ZVBB-RV64-NEXT: add a3, a3, a0
; ZVBB-RV64-NEXT: vl1re64.v v21, (a3)
-; ZVBB-RV64-NEXT: slli a1, a0, 2
-; ZVBB-RV64-NEXT: slli a0, a0, 3
-; ZVBB-RV64-NEXT: add a1, a0, a1
+; ZVBB-RV64-NEXT: slli a1, a0, 3
; ZVBB-RV64-NEXT: csrr a2, vlenb
; ZVBB-RV64-NEXT: li a3, 14
; ZVBB-RV64-NEXT: mul a2, a2, a3
; ZVBB-RV64-NEXT: add a2, sp, a2
; ZVBB-RV64-NEXT: addi a2, a2, 64
; ZVBB-RV64-NEXT: add a1, a2, a1
-; ZVBB-RV64-NEXT: vs2r.v v20, (a1)
-; ZVBB-RV64-NEXT: add a0, a2, a0
-; ZVBB-RV64-NEXT: vs4r.v v16, (a0)
+; ZVBB-RV64-NEXT: slli a0, a0, 2
+; ZVBB-RV64-NEXT: add a0, a1, a0
+; ZVBB-RV64-NEXT: vs2r.v v20, (a0)
+; ZVBB-RV64-NEXT: vs4r.v v16, (a1)
; ZVBB-RV64-NEXT: vs8r.v v8, (a2)
-; ZVBB-RV64-NEXT: vl8re64.v v16, (a0)
+; ZVBB-RV64-NEXT: vl8re64.v v16, (a1)
; ZVBB-RV64-NEXT: vl8re64.v v8, (a2)
; ZVBB-RV64-NEXT: addi sp, s0, -80
; ZVBB-RV64-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
@@ -14063,20 +14024,19 @@ define <vscale x 14 x double> @vector_interleave_nxv14f64_nxv2f64(<vscale x 2 x
; ZVZIP-NEXT: vl1re64.v v20, (a3)
; ZVZIP-NEXT: add a3, a3, a0
; ZVZIP-NEXT: vl1re64.v v21, (a3)
-; ZVZIP-NEXT: slli a1, a0, 2
-; ZVZIP-NEXT: slli a0, a0, 3
-; ZVZIP-NEXT: add a1, a0, a1
+; ZVZIP-NEXT: slli a1, a0, 3
; ZVZIP-NEXT: csrr a2, vlenb
; ZVZIP-NEXT: li a3, 14
; ZVZIP-NEXT: mul a2, a2, a3
; ZVZIP-NEXT: add a2, sp, a2
; ZVZIP-NEXT: addi a2, a2, 64
; ZVZIP-NEXT: add a1, a2, a1
-; ZVZIP-NEXT: vs2r.v v20, (a1)
-; ZVZIP-NEXT: add a0, a2, a0
-; ZVZIP-NEXT: vs4r.v v16, (a0)
+; ZVZIP-NEXT: slli a0, a0, 2
+; ZVZIP-NEXT: add a0, a1, a0
+; ZVZIP-NEXT: vs2r.v v20, (a0)
+; ZVZIP-NEXT: vs4r.v v16, (a1)
; ZVZIP-NEXT: vs8r.v v8, (a2)
-; ZVZIP-NEXT: vl8re64.v v16, (a0)
+; ZVZIP-NEXT: vl8re64.v v16, (a1)
; ZVZIP-NEXT: vl8re64.v v8, (a2)
; ZVZIP-NEXT: addi sp, s0, -80
; ZVZIP-NEXT: ld ra, 72(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vsub-sdnode.ll
index e1dbc031b4e15..8c5dc172a23b7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsub-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsub-sdnode.ll
@@ -927,19 +927,17 @@ define <vscale x 8 x i32> @vsub_vi_mask_nxv8i32(<vscale x 8 x i32> %va, <vscale
define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x 64 x i64> %vb) {
; RV32-LABEL: vsub_vv_nxv64i64:
; RV32: # %bb.0:
-; RV32-NEXT: addi sp, sp, -80
-; RV32-NEXT: .cfi_def_cfa_offset 80
-; RV32-NEXT: sw ra, 76(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s0, 72(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s1, 68(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s2, 64(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s3, 60(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s4, 56(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s5, 52(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s6, 48(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s7, 44(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s8, 40(sp) # 4-byte Folded Spill
-; RV32-NEXT: sw s9, 36(sp) # 4-byte Folded Spill
+; RV32-NEXT: addi sp, sp, -64
+; RV32-NEXT: .cfi_def_cfa_offset 64
+; RV32-NEXT: sw ra, 60(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s0, 56(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s1, 52(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s2, 48(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s3, 44(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s4, 40(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s5, 36(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s6, 32(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s7, 28(sp) # 4-byte Folded Spill
; RV32-NEXT: .cfi_offset ra, -4
; RV32-NEXT: .cfi_offset s0, -8
; RV32-NEXT: .cfi_offset s1, -12
@@ -949,8 +947,6 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV32-NEXT: .cfi_offset s5, -28
; RV32-NEXT: .cfi_offset s6, -32
; RV32-NEXT: .cfi_offset s7, -36
-; RV32-NEXT: .cfi_offset s8, -40
-; RV32-NEXT: .cfi_offset s9, -44
; RV32-NEXT: csrr a2, vlenb
; RV32-NEXT: slli a2, a2, 3
; RV32-NEXT: mv a3, a2
@@ -959,7 +955,7 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV32-NEXT: slli a2, a2, 1
; RV32-NEXT: add a2, a2, a3
; RV32-NEXT: sub sp, sp, a2
-; RV32-NEXT: .cfi_escape 0x0f, 0x0f, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0xe8, 0x00, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 104 * vlenb
+; RV32-NEXT: .cfi_escape 0x0f, 0x0f, 0x72, 0x00, 0x11, 0xc0, 0x00, 0x22, 0x11, 0xe8, 0x00, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 64 + 104 * vlenb
; RV32-NEXT: mv s2, a7
; RV32-NEXT: mv s1, a1
; RV32-NEXT: csrr a1, vlenb
@@ -968,7 +964,7 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV32-NEXT: slli a1, a1, 1
; RV32-NEXT: add a1, a1, a2
; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
+; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 3
@@ -976,11 +972,11 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV32-NEXT: slli a1, a1, 3
; RV32-NEXT: add a1, a1, a2
; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
+; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV32-NEXT: csrr s6, vlenb
-; RV32-NEXT: slli s8, s6, 4
-; RV32-NEXT: add a1, a7, s8
+; RV32-NEXT: csrr s3, vlenb
+; RV32-NEXT: slli s6, s3, 4
+; RV32-NEXT: add a1, a7, s6
; RV32-NEXT: vl8re64.v v8, (a1)
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 3
@@ -990,10 +986,10 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV32-NEXT: slli a1, a1, 2
; RV32-NEXT: add a1, a1, a2
; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
+; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV32-NEXT: slli s7, s6, 3
-; RV32-NEXT: add a1, a7, s7
+; RV32-NEXT: slli s5, s3, 3
+; RV32-NEXT: add a1, a7, s5
; RV32-NEXT: vl8re64.v v8, (a1)
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 4
@@ -1001,25 +997,21 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV32-NEXT: slli a1, a1, 2
; RV32-NEXT: add a1, a1, a2
; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
+; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
; RV32-NEXT: mv s0, a0
-; RV32-NEXT: li a1, 56
-; RV32-NEXT: mv a0, s6
+; RV32-NEXT: li a1, 48
+; RV32-NEXT: mv a0, s3
; RV32-NEXT: call __mulsi3
-; RV32-NEXT: mv s3, a0
+; RV32-NEXT: slli s7, s3, 5
; RV32-NEXT: add a0, s2, a0
; RV32-NEXT: vl8re64.v v8, (a0)
; RV32-NEXT: csrr a0, vlenb
; RV32-NEXT: slli a0, a0, 6
; RV32-NEXT: add a0, sp, a0
-; RV32-NEXT: addi a0, a0, 32
+; RV32-NEXT: addi a0, a0, 16
; RV32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; RV32-NEXT: li a1, 40
-; RV32-NEXT: mv a0, s6
-; RV32-NEXT: call __mulsi3
-; RV32-NEXT: mv s4, a0
-; RV32-NEXT: add a0, s1, a0
+; RV32-NEXT: add a0, s1, s7
; RV32-NEXT: vl8re64.v v8, (a0)
; RV32-NEXT: csrr a0, vlenb
; RV32-NEXT: slli a0, a0, 3
@@ -1029,67 +1021,43 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV32-NEXT: slli a0, a0, 1
; RV32-NEXT: add a0, a0, a1
; RV32-NEXT: add a0, sp, a0
-; RV32-NEXT: addi a0, a0, 32
+; RV32-NEXT: addi a0, a0, 16
; RV32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; RV32-NEXT: li a1, 48
-; RV32-NEXT: mv a0, s6
+; RV32-NEXT: li a1, 40
+; RV32-NEXT: mv a0, s3
; RV32-NEXT: call __mulsi3
-; RV32-NEXT: mv s5, a0
-; RV32-NEXT: slli s9, s6, 5
+; RV32-NEXT: mv s4, a0
; RV32-NEXT: add a0, s2, a0
; RV32-NEXT: vl8re64.v v8, (a0)
; RV32-NEXT: csrr a0, vlenb
-; RV32-NEXT: slli a0, a0, 3
-; RV32-NEXT: mv a1, a0
-; RV32-NEXT: slli a0, a0, 2
-; RV32-NEXT: add a0, a0, a1
-; RV32-NEXT: add a0, sp, a0
-; RV32-NEXT: addi a0, a0, 32
-; RV32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; RV32-NEXT: add a0, s1, s9
-; RV32-NEXT: vl8re64.v v8, (a0)
-; RV32-NEXT: csrr a0, vlenb
-; RV32-NEXT: slli a0, a0, 5
-; RV32-NEXT: add a0, sp, a0
-; RV32-NEXT: addi a0, a0, 32
-; RV32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; RV32-NEXT: add a0, s2, s4
-; RV32-NEXT: vl8re64.v v8, (a0)
-; RV32-NEXT: csrr a0, vlenb
; RV32-NEXT: slli a0, a0, 4
; RV32-NEXT: mv a1, a0
; RV32-NEXT: slli a0, a0, 1
; RV32-NEXT: add a0, a0, a1
; RV32-NEXT: add a0, sp, a0
-; RV32-NEXT: addi a0, a0, 32
+; RV32-NEXT: addi a0, a0, 16
; RV32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
; RV32-NEXT: li a1, 24
-; RV32-NEXT: mv a0, s6
+; RV32-NEXT: mv a0, s3
; RV32-NEXT: call __mulsi3
+; RV32-NEXT: add a1, s2, s7
+; RV32-NEXT: add a2, s1, a0
+; RV32-NEXT: vl8re64.v v8, (a2)
+; RV32-NEXT: csrr a2, vlenb
+; RV32-NEXT: slli a2, a2, 5
+; RV32-NEXT: add a2, sp, a2
+; RV32-NEXT: addi a2, a2, 16
+; RV32-NEXT: vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
+; RV32-NEXT: vl8re64.v v8, (a1)
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 3
; RV32-NEXT: mv a2, a1
; RV32-NEXT: slli a1, a1, 2
; RV32-NEXT: add a1, a1, a2
; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
-; RV32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
-; RV32-NEXT: csrr a1, vlenb
-; RV32-NEXT: slli a1, a1, 5
-; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
-; RV32-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; RV32-NEXT: vsetvli a1, zero, e64, m8, ta, ma
-; RV32-NEXT: vsub.vv v8, v16, v8
-; RV32-NEXT: csrr a1, vlenb
-; RV32-NEXT: slli a1, a1, 3
-; RV32-NEXT: mv a2, a1
-; RV32-NEXT: slli a1, a1, 2
-; RV32-NEXT: add a1, a1, a2
-; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
+; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV32-NEXT: add a1, s1, a0
+; RV32-NEXT: add a1, s1, s6
; RV32-NEXT: vl8re64.v v8, (a1)
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 3
@@ -1097,147 +1065,167 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV32-NEXT: slli a1, a1, 1
; RV32-NEXT: add a1, a1, a2
; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
-; RV32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV32-NEXT: add a1, s2, s9
-; RV32-NEXT: vl8re64.v v8, (a1)
-; RV32-NEXT: csrr a1, vlenb
-; RV32-NEXT: slli a1, a1, 5
-; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
-; RV32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV32-NEXT: add a1, s1, s8
-; RV32-NEXT: vl8re64.v v8, (a1)
-; RV32-NEXT: csrr a1, vlenb
-; RV32-NEXT: slli a1, a1, 3
-; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
+; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV32-NEXT: add a1, s2, a0
-; RV32-NEXT: vl8re64.v v8, (a1)
+; RV32-NEXT: add a0, s2, a0
+; RV32-NEXT: vl8re64.v v8, (a0)
+; RV32-NEXT: csrr a0, vlenb
+; RV32-NEXT: slli a0, a0, 4
+; RV32-NEXT: add a0, sp, a0
+; RV32-NEXT: addi a0, a0, 16
+; RV32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; RV32-NEXT: add a0, s1, s5
+; RV32-NEXT: vl8re64.v v8, (a0)
+; RV32-NEXT: csrr a0, vlenb
+; RV32-NEXT: slli a0, a0, 3
+; RV32-NEXT: add a0, sp, a0
+; RV32-NEXT: addi a0, a0, 16
+; RV32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; RV32-NEXT: li a1, 56
+; RV32-NEXT: mv a0, s3
+; RV32-NEXT: call __mulsi3
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 4
; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
-; RV32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV32-NEXT: add a1, s1, s7
-; RV32-NEXT: vl8re64.v v8, (a1)
-; RV32-NEXT: addi a1, sp, 32
-; RV32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV32-NEXT: vl8re64.v v8, (s2)
+; RV32-NEXT: addi a1, a1, 16
+; RV32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 3
-; RV32-NEXT: mv a2, a1
-; RV32-NEXT: slli a1, a1, 3
-; RV32-NEXT: add a1, a1, a2
; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
+; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
+; RV32-NEXT: vsetvli a1, zero, e64, m8, ta, ma
; RV32-NEXT: vsub.vv v8, v16, v8
-; RV32-NEXT: vl8re64.v v24, (s1)
-; RV32-NEXT: csrr a1, vlenb
-; RV32-NEXT: slli a1, a1, 6
-; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
-; RV32-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; RV32-NEXT: csrr a1, vlenb
-; RV32-NEXT: slli a1, a1, 3
-; RV32-NEXT: mv a2, a1
-; RV32-NEXT: slli a1, a1, 1
-; RV32-NEXT: add a2, a2, a1
-; RV32-NEXT: slli a1, a1, 1
-; RV32-NEXT: add a1, a1, a2
-; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
-; RV32-NEXT: vl8r.v v0, (a1) # vscale x 64-byte Folded Reload
-; RV32-NEXT: vsub.vv v16, v0, v16
+; RV32-NEXT: addi a1, sp, 16
+; RV32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; RV32-NEXT: add a0, s2, a0
+; RV32-NEXT: vl8re64.v v8, (a0)
+; RV32-NEXT: csrr a0, vlenb
+; RV32-NEXT: slli a0, a0, 4
+; RV32-NEXT: add a0, sp, a0
+; RV32-NEXT: addi a0, a0, 16
+; RV32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; RV32-NEXT: add s4, s1, s4
+; RV32-NEXT: vl8re64.v v8, (s4)
+; RV32-NEXT: csrr a0, vlenb
+; RV32-NEXT: slli a0, a0, 3
+; RV32-NEXT: add a0, sp, a0
+; RV32-NEXT: addi a0, a0, 16
+; RV32-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; RV32-NEXT: vl8re64.v v0, (s2)
+; RV32-NEXT: csrr a0, vlenb
+; RV32-NEXT: slli a0, a0, 3
+; RV32-NEXT: mv a1, a0
+; RV32-NEXT: slli a0, a0, 3
+; RV32-NEXT: add a0, a0, a1
+; RV32-NEXT: add a0, sp, a0
+; RV32-NEXT: addi a0, a0, 16
+; RV32-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
+; RV32-NEXT: vsub.vv v8, v8, v0
+; RV32-NEXT: vl8re64.v v0, (s1)
+; RV32-NEXT: csrr a0, vlenb
+; RV32-NEXT: slli a0, a0, 6
+; RV32-NEXT: add a0, sp, a0
+; RV32-NEXT: addi a0, a0, 16
+; RV32-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload
+; RV32-NEXT: csrr a0, vlenb
+; RV32-NEXT: slli a0, a0, 3
+; RV32-NEXT: mv a1, a0
+; RV32-NEXT: slli a0, a0, 1
+; RV32-NEXT: add a1, a1, a0
+; RV32-NEXT: slli a0, a0, 1
+; RV32-NEXT: add a0, a0, a1
+; RV32-NEXT: add a0, sp, a0
+; RV32-NEXT: addi a0, a0, 16
+; RV32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; RV32-NEXT: vsub.vv v16, v24, v16
; RV32-NEXT: vs8r.v v8, (s0)
-; RV32-NEXT: add s3, s0, s3
-; RV32-NEXT: vs8r.v v16, (s3)
-; RV32-NEXT: add s5, s0, s5
+; RV32-NEXT: add s7, s0, s7
+; RV32-NEXT: add a0, s7, s6
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 4
; RV32-NEXT: mv a2, a1
; RV32-NEXT: slli a1, a1, 1
; RV32-NEXT: add a1, a1, a2
; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
+; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
; RV32-NEXT: csrr a1, vlenb
-; RV32-NEXT: slli a1, a1, 3
+; RV32-NEXT: slli a1, a1, 5
+; RV32-NEXT: add a1, sp, a1
+; RV32-NEXT: addi a1, a1, 16
+; RV32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; RV32-NEXT: vsub.vv v8, v24, v8
+; RV32-NEXT: vs8r.v v16, (a0)
+; RV32-NEXT: add a1, s7, s5
+; RV32-NEXT: csrr a2, vlenb
+; RV32-NEXT: slli a2, a2, 3
+; RV32-NEXT: mv a3, a2
+; RV32-NEXT: slli a2, a2, 2
+; RV32-NEXT: add a2, a2, a3
+; RV32-NEXT: add a2, sp, a2
+; RV32-NEXT: addi a2, a2, 16
+; RV32-NEXT: vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
+; RV32-NEXT: csrr a2, vlenb
+; RV32-NEXT: slli a2, a2, 3
+; RV32-NEXT: mv a3, a2
+; RV32-NEXT: slli a2, a2, 1
+; RV32-NEXT: add a2, a2, a3
+; RV32-NEXT: add a2, sp, a2
+; RV32-NEXT: addi a2, a2, 16
+; RV32-NEXT: vl8r.v v24, (a2) # vscale x 64-byte Folded Reload
+; RV32-NEXT: vsub.vv v16, v24, v16
+; RV32-NEXT: vs8r.v v8, (a1)
+; RV32-NEXT: vs8r.v v16, (s7)
+; RV32-NEXT: add s6, s0, s6
+; RV32-NEXT: add a1, s6, s5
+; RV32-NEXT: csrr a2, vlenb
+; RV32-NEXT: slli a2, a2, 3
+; RV32-NEXT: mv a3, a2
+; RV32-NEXT: slli a2, a2, 1
+; RV32-NEXT: add a3, a3, a2
+; RV32-NEXT: slli a2, a2, 2
+; RV32-NEXT: add a2, a2, a3
+; RV32-NEXT: add a2, sp, a2
+; RV32-NEXT: addi a2, a2, 16
+; RV32-NEXT: vl8r.v v8, (a2) # vscale x 64-byte Folded Reload
+; RV32-NEXT: vsub.vv v8, v0, v8
+; RV32-NEXT: addi a2, sp, 16
+; RV32-NEXT: vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
+; RV32-NEXT: vs8r.v v16, (a1)
+; RV32-NEXT: csrr a1, vlenb
+; RV32-NEXT: slli a1, a1, 5
; RV32-NEXT: mv a2, a1
; RV32-NEXT: slli a1, a1, 1
; RV32-NEXT: add a1, a1, a2
; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
+; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; RV32-NEXT: vsub.vv v8, v16, v8
; RV32-NEXT: csrr a1, vlenb
-; RV32-NEXT: slli a1, a1, 3
+; RV32-NEXT: slli a1, a1, 4
; RV32-NEXT: mv a2, a1
; RV32-NEXT: slli a1, a1, 2
; RV32-NEXT: add a1, a1, a2
; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
-; RV32-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; RV32-NEXT: vs8r.v v16, (s5)
-; RV32-NEXT: add s4, s0, s4
-; RV32-NEXT: csrr a1, vlenb
-; RV32-NEXT: slli a1, a1, 5
-; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
-; RV32-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; RV32-NEXT: csrr a1, vlenb
-; RV32-NEXT: slli a1, a1, 3
-; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
-; RV32-NEXT: vl8r.v v0, (a1) # vscale x 64-byte Folded Reload
-; RV32-NEXT: vsub.vv v16, v0, v16
-; RV32-NEXT: vs8r.v v8, (s4)
-; RV32-NEXT: add s9, s0, s9
+; RV32-NEXT: addi a1, a1, 16
+; RV32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; RV32-NEXT: vsub.vv v16, v16, v24
+; RV32-NEXT: vs8r.v v8, (s6)
+; RV32-NEXT: add s0, s0, s5
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 4
; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
+; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
-; RV32-NEXT: addi a1, sp, 32
-; RV32-NEXT: vl8r.v v0, (a1) # vscale x 64-byte Folded Reload
-; RV32-NEXT: vsub.vv v8, v0, v8
-; RV32-NEXT: vs8r.v v16, (s9)
-; RV32-NEXT: add a0, s0, a0
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 3
-; RV32-NEXT: mv a2, a1
-; RV32-NEXT: slli a1, a1, 1
-; RV32-NEXT: add a2, a2, a1
-; RV32-NEXT: slli a1, a1, 2
-; RV32-NEXT: add a1, a1, a2
; RV32-NEXT: add a1, sp, a1
-; RV32-NEXT: addi a1, a1, 32
-; RV32-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; RV32-NEXT: vsub.vv v16, v24, v16
+; RV32-NEXT: addi a1, a1, 16
+; RV32-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; RV32-NEXT: vsub.vv v8, v24, v8
+; RV32-NEXT: vs8r.v v16, (s0)
+; RV32-NEXT: add a0, a0, s5
; RV32-NEXT: vs8r.v v8, (a0)
-; RV32-NEXT: add s8, s0, s8
-; RV32-NEXT: csrr a0, vlenb
-; RV32-NEXT: slli a0, a0, 5
-; RV32-NEXT: mv a1, a0
-; RV32-NEXT: slli a0, a0, 1
-; RV32-NEXT: add a0, a0, a1
-; RV32-NEXT: add a0, sp, a0
-; RV32-NEXT: addi a0, a0, 32
-; RV32-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
-; RV32-NEXT: csrr a0, vlenb
-; RV32-NEXT: slli a0, a0, 4
-; RV32-NEXT: mv a1, a0
-; RV32-NEXT: slli a0, a0, 2
-; RV32-NEXT: add a0, a0, a1
-; RV32-NEXT: add a0, sp, a0
-; RV32-NEXT: addi a0, a0, 32
-; RV32-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-; RV32-NEXT: vsub.vv v8, v8, v24
-; RV32-NEXT: vs8r.v v16, (s8)
-; RV32-NEXT: add s0, s0, s7
-; RV32-NEXT: vs8r.v v8, (s0)
; RV32-NEXT: csrr a0, vlenb
; RV32-NEXT: slli a0, a0, 3
; RV32-NEXT: mv a1, a0
@@ -1246,18 +1234,16 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV32-NEXT: slli a0, a0, 1
; RV32-NEXT: add a0, a0, a1
; RV32-NEXT: add sp, sp, a0
-; RV32-NEXT: .cfi_def_cfa sp, 80
-; RV32-NEXT: lw ra, 76(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s0, 72(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s1, 68(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s2, 64(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s3, 60(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s4, 56(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s5, 52(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s6, 48(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s7, 44(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s8, 40(sp) # 4-byte Folded Reload
-; RV32-NEXT: lw s9, 36(sp) # 4-byte Folded Reload
+; RV32-NEXT: .cfi_def_cfa sp, 64
+; RV32-NEXT: lw ra, 60(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s0, 56(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s1, 52(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s2, 48(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s3, 44(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s4, 40(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s5, 36(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s6, 32(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s7, 28(sp) # 4-byte Folded Reload
; RV32-NEXT: .cfi_restore ra
; RV32-NEXT: .cfi_restore s0
; RV32-NEXT: .cfi_restore s1
@@ -1267,27 +1253,23 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV32-NEXT: .cfi_restore s5
; RV32-NEXT: .cfi_restore s6
; RV32-NEXT: .cfi_restore s7
-; RV32-NEXT: .cfi_restore s8
-; RV32-NEXT: .cfi_restore s9
-; RV32-NEXT: addi sp, sp, 80
+; RV32-NEXT: addi sp, sp, 64
; RV32-NEXT: .cfi_def_cfa_offset 0
; RV32-NEXT: ret
;
; RV64-LABEL: vsub_vv_nxv64i64:
; RV64: # %bb.0:
-; RV64-NEXT: addi sp, sp, -128
-; RV64-NEXT: .cfi_def_cfa_offset 128
-; RV64-NEXT: sd ra, 120(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s0, 112(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s1, 104(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s2, 96(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s3, 88(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s4, 80(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s5, 72(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s6, 64(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s7, 56(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s8, 48(sp) # 8-byte Folded Spill
-; RV64-NEXT: sd s9, 40(sp) # 8-byte Folded Spill
+; RV64-NEXT: addi sp, sp, -112
+; RV64-NEXT: .cfi_def_cfa_offset 112
+; RV64-NEXT: sd ra, 104(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s0, 96(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s1, 88(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s2, 80(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s3, 72(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s4, 64(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s5, 56(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s6, 48(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s7, 40(sp) # 8-byte Folded Spill
; RV64-NEXT: .cfi_offset ra, -8
; RV64-NEXT: .cfi_offset s0, -16
; RV64-NEXT: .cfi_offset s1, -24
@@ -1297,8 +1279,6 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV64-NEXT: .cfi_offset s5, -56
; RV64-NEXT: .cfi_offset s6, -64
; RV64-NEXT: .cfi_offset s7, -72
-; RV64-NEXT: .cfi_offset s8, -80
-; RV64-NEXT: .cfi_offset s9, -88
; RV64-NEXT: csrr a2, vlenb
; RV64-NEXT: slli a2, a2, 3
; RV64-NEXT: mv a3, a2
@@ -1307,7 +1287,7 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV64-NEXT: slli a2, a2, 1
; RV64-NEXT: add a2, a2, a3
; RV64-NEXT: sub sp, sp, a2
-; RV64-NEXT: .cfi_escape 0x0f, 0x0f, 0x72, 0x00, 0x11, 0x80, 0x01, 0x22, 0x11, 0xe8, 0x00, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 128 + 104 * vlenb
+; RV64-NEXT: .cfi_escape 0x0f, 0x0f, 0x72, 0x00, 0x11, 0xf0, 0x00, 0x22, 0x11, 0xe8, 0x00, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 112 + 104 * vlenb
; RV64-NEXT: mv s2, a7
; RV64-NEXT: mv s1, a1
; RV64-NEXT: csrr a1, vlenb
@@ -1326,9 +1306,9 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV64-NEXT: add a1, sp, a1
; RV64-NEXT: addi a1, a1, 32
; RV64-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV64-NEXT: csrr s6, vlenb
-; RV64-NEXT: slli s8, s6, 4
-; RV64-NEXT: add a1, a7, s8
+; RV64-NEXT: csrr s3, vlenb
+; RV64-NEXT: slli s6, s3, 4
+; RV64-NEXT: add a1, a7, s6
; RV64-NEXT: vl8re64.v v8, (a1)
; RV64-NEXT: csrr a1, vlenb
; RV64-NEXT: slli a1, a1, 3
@@ -1340,8 +1320,8 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV64-NEXT: add a1, sp, a1
; RV64-NEXT: addi a1, a1, 32
; RV64-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV64-NEXT: slli s7, s6, 3
-; RV64-NEXT: add a1, a7, s7
+; RV64-NEXT: slli s5, s3, 3
+; RV64-NEXT: add a1, a7, s5
; RV64-NEXT: vl8re64.v v8, (a1)
; RV64-NEXT: csrr a1, vlenb
; RV64-NEXT: slli a1, a1, 4
@@ -1352,10 +1332,10 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV64-NEXT: addi a1, a1, 32
; RV64-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
; RV64-NEXT: mv s0, a0
-; RV64-NEXT: li a1, 56
-; RV64-NEXT: mv a0, s6
+; RV64-NEXT: li a1, 48
+; RV64-NEXT: mv a0, s3
; RV64-NEXT: call __muldi3
-; RV64-NEXT: mv s3, a0
+; RV64-NEXT: slli s7, s3, 5
; RV64-NEXT: add a0, s2, a0
; RV64-NEXT: vl8re64.v v8, (a0)
; RV64-NEXT: csrr a0, vlenb
@@ -1363,11 +1343,7 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV64-NEXT: add a0, sp, a0
; RV64-NEXT: addi a0, a0, 32
; RV64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; RV64-NEXT: li a1, 40
-; RV64-NEXT: mv a0, s6
-; RV64-NEXT: call __muldi3
-; RV64-NEXT: mv s4, a0
-; RV64-NEXT: add a0, s1, a0
+; RV64-NEXT: add a0, s1, s7
; RV64-NEXT: vl8re64.v v8, (a0)
; RV64-NEXT: csrr a0, vlenb
; RV64-NEXT: slli a0, a0, 3
@@ -1379,31 +1355,13 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV64-NEXT: add a0, sp, a0
; RV64-NEXT: addi a0, a0, 32
; RV64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; RV64-NEXT: li a1, 48
-; RV64-NEXT: mv a0, s6
+; RV64-NEXT: li a1, 40
+; RV64-NEXT: mv a0, s3
; RV64-NEXT: call __muldi3
-; RV64-NEXT: mv s5, a0
-; RV64-NEXT: slli s9, s6, 5
+; RV64-NEXT: mv s4, a0
; RV64-NEXT: add a0, s2, a0
; RV64-NEXT: vl8re64.v v8, (a0)
; RV64-NEXT: csrr a0, vlenb
-; RV64-NEXT: slli a0, a0, 3
-; RV64-NEXT: mv a1, a0
-; RV64-NEXT: slli a0, a0, 2
-; RV64-NEXT: add a0, a0, a1
-; RV64-NEXT: add a0, sp, a0
-; RV64-NEXT: addi a0, a0, 32
-; RV64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; RV64-NEXT: add a0, s1, s9
-; RV64-NEXT: vl8re64.v v8, (a0)
-; RV64-NEXT: csrr a0, vlenb
-; RV64-NEXT: slli a0, a0, 5
-; RV64-NEXT: add a0, sp, a0
-; RV64-NEXT: addi a0, a0, 32
-; RV64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; RV64-NEXT: add a0, s2, s4
-; RV64-NEXT: vl8re64.v v8, (a0)
-; RV64-NEXT: csrr a0, vlenb
; RV64-NEXT: slli a0, a0, 4
; RV64-NEXT: mv a1, a0
; RV64-NEXT: slli a0, a0, 1
@@ -1412,23 +1370,17 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV64-NEXT: addi a0, a0, 32
; RV64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
; RV64-NEXT: li a1, 24
-; RV64-NEXT: mv a0, s6
+; RV64-NEXT: mv a0, s3
; RV64-NEXT: call __muldi3
-; RV64-NEXT: csrr a1, vlenb
-; RV64-NEXT: slli a1, a1, 3
-; RV64-NEXT: mv a2, a1
-; RV64-NEXT: slli a1, a1, 2
-; RV64-NEXT: add a1, a1, a2
-; RV64-NEXT: add a1, sp, a1
-; RV64-NEXT: addi a1, a1, 32
-; RV64-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
-; RV64-NEXT: csrr a1, vlenb
-; RV64-NEXT: slli a1, a1, 5
-; RV64-NEXT: add a1, sp, a1
-; RV64-NEXT: addi a1, a1, 32
-; RV64-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; RV64-NEXT: vsetvli a1, zero, e64, m8, ta, ma
-; RV64-NEXT: vsub.vv v8, v16, v8
+; RV64-NEXT: add a1, s2, s7
+; RV64-NEXT: add a2, s1, a0
+; RV64-NEXT: vl8re64.v v8, (a2)
+; RV64-NEXT: csrr a2, vlenb
+; RV64-NEXT: slli a2, a2, 5
+; RV64-NEXT: add a2, sp, a2
+; RV64-NEXT: addi a2, a2, 32
+; RV64-NEXT: vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
+; RV64-NEXT: vl8re64.v v8, (a1)
; RV64-NEXT: csrr a1, vlenb
; RV64-NEXT: slli a1, a1, 3
; RV64-NEXT: mv a2, a1
@@ -1437,7 +1389,7 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV64-NEXT: add a1, sp, a1
; RV64-NEXT: addi a1, a1, 32
; RV64-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV64-NEXT: add a1, s1, a0
+; RV64-NEXT: add a1, s1, s6
; RV64-NEXT: vl8re64.v v8, (a1)
; RV64-NEXT: csrr a1, vlenb
; RV64-NEXT: slli a1, a1, 3
@@ -1447,62 +1399,81 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV64-NEXT: add a1, sp, a1
; RV64-NEXT: addi a1, a1, 32
; RV64-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV64-NEXT: add a1, s2, s9
-; RV64-NEXT: vl8re64.v v8, (a1)
-; RV64-NEXT: csrr a1, vlenb
-; RV64-NEXT: slli a1, a1, 5
-; RV64-NEXT: add a1, sp, a1
-; RV64-NEXT: addi a1, a1, 32
-; RV64-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV64-NEXT: add a1, s1, s8
-; RV64-NEXT: vl8re64.v v8, (a1)
-; RV64-NEXT: csrr a1, vlenb
-; RV64-NEXT: slli a1, a1, 3
-; RV64-NEXT: add a1, sp, a1
-; RV64-NEXT: addi a1, a1, 32
-; RV64-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV64-NEXT: add a1, s2, a0
-; RV64-NEXT: vl8re64.v v8, (a1)
+; RV64-NEXT: add a0, s2, a0
+; RV64-NEXT: vl8re64.v v8, (a0)
+; RV64-NEXT: csrr a0, vlenb
+; RV64-NEXT: slli a0, a0, 4
+; RV64-NEXT: add a0, sp, a0
+; RV64-NEXT: addi a0, a0, 32
+; RV64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; RV64-NEXT: add a0, s1, s5
+; RV64-NEXT: vl8re64.v v8, (a0)
+; RV64-NEXT: csrr a0, vlenb
+; RV64-NEXT: slli a0, a0, 3
+; RV64-NEXT: add a0, sp, a0
+; RV64-NEXT: addi a0, a0, 32
+; RV64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; RV64-NEXT: li a1, 56
+; RV64-NEXT: mv a0, s3
+; RV64-NEXT: call __muldi3
; RV64-NEXT: csrr a1, vlenb
; RV64-NEXT: slli a1, a1, 4
; RV64-NEXT: add a1, sp, a1
; RV64-NEXT: addi a1, a1, 32
-; RV64-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV64-NEXT: add a1, s1, s7
-; RV64-NEXT: vl8re64.v v8, (a1)
-; RV64-NEXT: addi a1, sp, 32
-; RV64-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; RV64-NEXT: vl8re64.v v8, (s2)
+; RV64-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
; RV64-NEXT: csrr a1, vlenb
; RV64-NEXT: slli a1, a1, 3
-; RV64-NEXT: mv a2, a1
-; RV64-NEXT: slli a1, a1, 3
-; RV64-NEXT: add a1, a1, a2
; RV64-NEXT: add a1, sp, a1
; RV64-NEXT: addi a1, a1, 32
; RV64-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
+; RV64-NEXT: vsetvli a1, zero, e64, m8, ta, ma
; RV64-NEXT: vsub.vv v8, v16, v8
-; RV64-NEXT: vl8re64.v v24, (s1)
-; RV64-NEXT: csrr a1, vlenb
-; RV64-NEXT: slli a1, a1, 6
-; RV64-NEXT: add a1, sp, a1
-; RV64-NEXT: addi a1, a1, 32
-; RV64-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; RV64-NEXT: csrr a1, vlenb
-; RV64-NEXT: slli a1, a1, 3
-; RV64-NEXT: mv a2, a1
-; RV64-NEXT: slli a1, a1, 1
-; RV64-NEXT: add a2, a2, a1
-; RV64-NEXT: slli a1, a1, 1
-; RV64-NEXT: add a1, a1, a2
-; RV64-NEXT: add a1, sp, a1
-; RV64-NEXT: addi a1, a1, 32
-; RV64-NEXT: vl8r.v v0, (a1) # vscale x 64-byte Folded Reload
-; RV64-NEXT: vsub.vv v16, v0, v16
+; RV64-NEXT: addi a1, sp, 32
+; RV64-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; RV64-NEXT: add a0, s2, a0
+; RV64-NEXT: vl8re64.v v8, (a0)
+; RV64-NEXT: csrr a0, vlenb
+; RV64-NEXT: slli a0, a0, 4
+; RV64-NEXT: add a0, sp, a0
+; RV64-NEXT: addi a0, a0, 32
+; RV64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; RV64-NEXT: add s4, s1, s4
+; RV64-NEXT: vl8re64.v v8, (s4)
+; RV64-NEXT: csrr a0, vlenb
+; RV64-NEXT: slli a0, a0, 3
+; RV64-NEXT: add a0, sp, a0
+; RV64-NEXT: addi a0, a0, 32
+; RV64-NEXT: vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
+; RV64-NEXT: vl8re64.v v0, (s2)
+; RV64-NEXT: csrr a0, vlenb
+; RV64-NEXT: slli a0, a0, 3
+; RV64-NEXT: mv a1, a0
+; RV64-NEXT: slli a0, a0, 3
+; RV64-NEXT: add a0, a0, a1
+; RV64-NEXT: add a0, sp, a0
+; RV64-NEXT: addi a0, a0, 32
+; RV64-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
+; RV64-NEXT: vsub.vv v8, v8, v0
+; RV64-NEXT: vl8re64.v v0, (s1)
+; RV64-NEXT: csrr a0, vlenb
+; RV64-NEXT: slli a0, a0, 6
+; RV64-NEXT: add a0, sp, a0
+; RV64-NEXT: addi a0, a0, 32
+; RV64-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload
+; RV64-NEXT: csrr a0, vlenb
+; RV64-NEXT: slli a0, a0, 3
+; RV64-NEXT: mv a1, a0
+; RV64-NEXT: slli a0, a0, 1
+; RV64-NEXT: add a1, a1, a0
+; RV64-NEXT: slli a0, a0, 1
+; RV64-NEXT: add a0, a0, a1
+; RV64-NEXT: add a0, sp, a0
+; RV64-NEXT: addi a0, a0, 32
+; RV64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; RV64-NEXT: vsub.vv v16, v24, v16
; RV64-NEXT: vs8r.v v8, (s0)
-; RV64-NEXT: add s3, s0, s3
-; RV64-NEXT: vs8r.v v16, (s3)
-; RV64-NEXT: add s5, s0, s5
+; RV64-NEXT: add s7, s0, s7
+; RV64-NEXT: add a0, s7, s6
; RV64-NEXT: csrr a1, vlenb
; RV64-NEXT: slli a1, a1, 4
; RV64-NEXT: mv a2, a1
@@ -1512,80 +1483,81 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV64-NEXT: addi a1, a1, 32
; RV64-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
; RV64-NEXT: csrr a1, vlenb
-; RV64-NEXT: slli a1, a1, 3
+; RV64-NEXT: slli a1, a1, 5
+; RV64-NEXT: add a1, sp, a1
+; RV64-NEXT: addi a1, a1, 32
+; RV64-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; RV64-NEXT: vsub.vv v8, v24, v8
+; RV64-NEXT: vs8r.v v16, (a0)
+; RV64-NEXT: add a1, s7, s5
+; RV64-NEXT: csrr a2, vlenb
+; RV64-NEXT: slli a2, a2, 3
+; RV64-NEXT: mv a3, a2
+; RV64-NEXT: slli a2, a2, 2
+; RV64-NEXT: add a2, a2, a3
+; RV64-NEXT: add a2, sp, a2
+; RV64-NEXT: addi a2, a2, 32
+; RV64-NEXT: vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
+; RV64-NEXT: csrr a2, vlenb
+; RV64-NEXT: slli a2, a2, 3
+; RV64-NEXT: mv a3, a2
+; RV64-NEXT: slli a2, a2, 1
+; RV64-NEXT: add a2, a2, a3
+; RV64-NEXT: add a2, sp, a2
+; RV64-NEXT: addi a2, a2, 32
+; RV64-NEXT: vl8r.v v24, (a2) # vscale x 64-byte Folded Reload
+; RV64-NEXT: vsub.vv v16, v24, v16
+; RV64-NEXT: vs8r.v v8, (a1)
+; RV64-NEXT: vs8r.v v16, (s7)
+; RV64-NEXT: add s6, s0, s6
+; RV64-NEXT: add a1, s6, s5
+; RV64-NEXT: csrr a2, vlenb
+; RV64-NEXT: slli a2, a2, 3
+; RV64-NEXT: mv a3, a2
+; RV64-NEXT: slli a2, a2, 1
+; RV64-NEXT: add a3, a3, a2
+; RV64-NEXT: slli a2, a2, 2
+; RV64-NEXT: add a2, a2, a3
+; RV64-NEXT: add a2, sp, a2
+; RV64-NEXT: addi a2, a2, 32
+; RV64-NEXT: vl8r.v v8, (a2) # vscale x 64-byte Folded Reload
+; RV64-NEXT: vsub.vv v8, v0, v8
+; RV64-NEXT: addi a2, sp, 32
+; RV64-NEXT: vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
+; RV64-NEXT: vs8r.v v16, (a1)
+; RV64-NEXT: csrr a1, vlenb
+; RV64-NEXT: slli a1, a1, 5
; RV64-NEXT: mv a2, a1
; RV64-NEXT: slli a1, a1, 1
; RV64-NEXT: add a1, a1, a2
; RV64-NEXT: add a1, sp, a1
; RV64-NEXT: addi a1, a1, 32
; RV64-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; RV64-NEXT: vsub.vv v8, v16, v8
; RV64-NEXT: csrr a1, vlenb
-; RV64-NEXT: slli a1, a1, 3
+; RV64-NEXT: slli a1, a1, 4
; RV64-NEXT: mv a2, a1
; RV64-NEXT: slli a1, a1, 2
; RV64-NEXT: add a1, a1, a2
; RV64-NEXT: add a1, sp, a1
; RV64-NEXT: addi a1, a1, 32
-; RV64-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; RV64-NEXT: vs8r.v v16, (s5)
-; RV64-NEXT: add s4, s0, s4
-; RV64-NEXT: csrr a1, vlenb
-; RV64-NEXT: slli a1, a1, 5
-; RV64-NEXT: add a1, sp, a1
-; RV64-NEXT: addi a1, a1, 32
-; RV64-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; RV64-NEXT: csrr a1, vlenb
-; RV64-NEXT: slli a1, a1, 3
-; RV64-NEXT: add a1, sp, a1
-; RV64-NEXT: addi a1, a1, 32
-; RV64-NEXT: vl8r.v v0, (a1) # vscale x 64-byte Folded Reload
-; RV64-NEXT: vsub.vv v16, v0, v16
-; RV64-NEXT: vs8r.v v8, (s4)
-; RV64-NEXT: add s9, s0, s9
+; RV64-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; RV64-NEXT: vsub.vv v16, v16, v24
+; RV64-NEXT: vs8r.v v8, (s6)
+; RV64-NEXT: add s0, s0, s5
; RV64-NEXT: csrr a1, vlenb
; RV64-NEXT: slli a1, a1, 4
; RV64-NEXT: add a1, sp, a1
; RV64-NEXT: addi a1, a1, 32
; RV64-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
-; RV64-NEXT: addi a1, sp, 32
-; RV64-NEXT: vl8r.v v0, (a1) # vscale x 64-byte Folded Reload
-; RV64-NEXT: vsub.vv v8, v0, v8
-; RV64-NEXT: vs8r.v v16, (s9)
-; RV64-NEXT: add a0, s0, a0
; RV64-NEXT: csrr a1, vlenb
; RV64-NEXT: slli a1, a1, 3
-; RV64-NEXT: mv a2, a1
-; RV64-NEXT: slli a1, a1, 1
-; RV64-NEXT: add a2, a2, a1
-; RV64-NEXT: slli a1, a1, 2
-; RV64-NEXT: add a1, a1, a2
; RV64-NEXT: add a1, sp, a1
; RV64-NEXT: addi a1, a1, 32
-; RV64-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
-; RV64-NEXT: vsub.vv v16, v24, v16
+; RV64-NEXT: vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; RV64-NEXT: vsub.vv v8, v24, v8
+; RV64-NEXT: vs8r.v v16, (s0)
+; RV64-NEXT: add a0, a0, s5
; RV64-NEXT: vs8r.v v8, (a0)
-; RV64-NEXT: add s8, s0, s8
-; RV64-NEXT: csrr a0, vlenb
-; RV64-NEXT: slli a0, a0, 5
-; RV64-NEXT: mv a1, a0
-; RV64-NEXT: slli a0, a0, 1
-; RV64-NEXT: add a0, a0, a1
-; RV64-NEXT: add a0, sp, a0
-; RV64-NEXT: addi a0, a0, 32
-; RV64-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
-; RV64-NEXT: csrr a0, vlenb
-; RV64-NEXT: slli a0, a0, 4
-; RV64-NEXT: mv a1, a0
-; RV64-NEXT: slli a0, a0, 2
-; RV64-NEXT: add a0, a0, a1
-; RV64-NEXT: add a0, sp, a0
-; RV64-NEXT: addi a0, a0, 32
-; RV64-NEXT: vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-; RV64-NEXT: vsub.vv v8, v8, v24
-; RV64-NEXT: vs8r.v v16, (s8)
-; RV64-NEXT: add s0, s0, s7
-; RV64-NEXT: vs8r.v v8, (s0)
; RV64-NEXT: csrr a0, vlenb
; RV64-NEXT: slli a0, a0, 3
; RV64-NEXT: mv a1, a0
@@ -1594,18 +1566,16 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV64-NEXT: slli a0, a0, 1
; RV64-NEXT: add a0, a0, a1
; RV64-NEXT: add sp, sp, a0
-; RV64-NEXT: .cfi_def_cfa sp, 128
-; RV64-NEXT: ld ra, 120(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s0, 112(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s1, 104(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s2, 96(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s3, 88(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s4, 80(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s5, 72(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s6, 64(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s7, 56(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s8, 48(sp) # 8-byte Folded Reload
-; RV64-NEXT: ld s9, 40(sp) # 8-byte Folded Reload
+; RV64-NEXT: .cfi_def_cfa sp, 112
+; RV64-NEXT: ld ra, 104(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s0, 96(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s1, 88(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s2, 80(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s3, 72(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s4, 64(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s5, 56(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s6, 48(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s7, 40(sp) # 8-byte Folded Reload
; RV64-NEXT: .cfi_restore ra
; RV64-NEXT: .cfi_restore s0
; RV64-NEXT: .cfi_restore s1
@@ -1615,9 +1585,7 @@ define <vscale x 64 x i64> @vsub_vv_nxv64i64(<vscale x 64 x i64> %va, <vscale x
; RV64-NEXT: .cfi_restore s5
; RV64-NEXT: .cfi_restore s6
; RV64-NEXT: .cfi_restore s7
-; RV64-NEXT: .cfi_restore s8
-; RV64-NEXT: .cfi_restore s9
-; RV64-NEXT: addi sp, sp, 128
+; RV64-NEXT: addi sp, sp, 112
; RV64-NEXT: .cfi_def_cfa_offset 0
; RV64-NEXT: ret
%vc = sub <vscale x 64 x i64> %va, %vb
>From 2f8bbc838c0f9421fff33afb7db0e29cd070f568 Mon Sep 17 00:00:00 2001
From: FoolgryGamer <1353470175 at qq.com>
Date: Sat, 26 Sep 2026 12:28:52 +0800
Subject: [PATCH 3/3] [RISCV] Fix redundant add instructions generated during
the fold of vscale add The previous changes in DAGCombiner.cpp would cause a
performance regression on the AArch64 architecture. In this submission, the
fold of vscale add has been lowered to the RISC-V architecture to ensure no
performance regression occurs on the AArch64 architecture.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 5 +
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 4 +-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 33 ++
llvm/lib/Target/RISCV/RISCVISelLowering.h | 2 +
...plex-deinterleaving-reductions-scalable.ll | 25 +-
llvm/test/CodeGen/AArch64/misched-cutoff.mir | 2 +-
.../AArch64/named-vector-shuffles-sve.ll | 66 +--
llvm/test/CodeGen/AArch64/nontemporal-load.ll | 20 +-
.../scalable_masked_deinterleaved_loads.ll | 4 +-
.../scalable_masked_interleaved_stores.ll | 6 +-
llvm/test/CodeGen/AArch64/sve-aliasing.ll | 30 +-
.../AArch64/sve-calling-convention-mixed.ll | 84 +--
.../sve-extract-fixed-from-scalable-vector.ll | 53 +-
.../test/CodeGen/AArch64/sve-fptrunc-store.ll | 18 +-
.../AArch64/sve-gather-scatter-dag-combine.ll | 28 +-
.../test/CodeGen/AArch64/sve-insert-vector.ll | 35 +-
.../AArch64/sve-intrinsics-ldst-ext.ll | 50 +-
llvm/test/CodeGen/AArch64/sve-ldst-sext.ll | 41 +-
llvm/test/CodeGen/AArch64/sve-ldst-zext.ll | 41 +-
llvm/test/CodeGen/AArch64/sve-llrint.ll | 331 ++++++------
.../AArch64/sve-load-store-legalisation.ll | 110 ++--
llvm/test/CodeGen/AArch64/sve-lrint.ll | 331 ++++++------
.../CodeGen/AArch64/sve-masked-ldst-sext.ll | 19 +-
.../CodeGen/AArch64/sve-masked-ldst-zext.ll | 19 +-
.../AArch64/sve-masked-scatter-legalize.ll | 19 +-
.../AArch64/sve-multivector-load-stores.ll | 238 ++++-----
.../CodeGen/AArch64/sve-split-extract-elt.ll | 30 +-
.../CodeGen/AArch64/sve-split-insert-elt.ll | 29 +-
llvm/test/CodeGen/AArch64/sve-split-load.ll | 29 +-
llvm/test/CodeGen/AArch64/sve-split-store.ll | 29 +-
.../AArch64/sve-vector-interleave-widen.ll | 479 ++++++++----------
llvm/test/CodeGen/AArch64/zext-to-tbl.ll | 40 +-
32 files changed, 945 insertions(+), 1305 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index c08d0e53ec34b..9044051f154f1 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4771,6 +4771,11 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
return true;
}
+ /// Always returns true on the AArch64 architecture.
+ /// Always return false on the RISC-V architecture.
+ /// Lower the fold of Vscale Add to RISCV.
+ virtual bool isDesirableToFoldVScaleAdd() const { return true; }
+
/// GlobalISel - return true if it is profitable to move this shift by a
/// constant amount through its operand, adjusting any immediate operands as
/// necessary to preserve semantics. This transformation may not be desirable
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 35fe060d53c8d..2494438dd24a3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -3327,9 +3327,9 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
}
// fold a+vscale(c1)+vscale(c2) -> a+vscale(c1+c2)
- if (N0.getOpcode() == ISD::ADD &&
+ if (TLI.isDesirableToFoldVScaleAdd() && N0.getOpcode() == ISD::ADD &&
N0.getOperand(1).getOpcode() == ISD::VSCALE &&
- N1.getOpcode() == ISD::VSCALE && N0.hasOneUse()) {
+ N1.getOpcode() == ISD::VSCALE) {
const APInt &VS0 = N0.getOperand(1)->getConstantOperandAPInt(0);
const APInt &VS1 = N1->getConstantOperandAPInt(0);
SDValue VS = DAG.getVScale(DL, VT, VS0 + VS1);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index f17d9e90bfb73..cba61698a9943 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -2153,6 +2153,9 @@ MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
return Subtarget.getXLenVT();
}
+// Return false to lower the fold of vscale add to RISCV.
+bool RISCVTargetLowering::isDesirableToFoldVScaleAdd() const { return false; }
+
// Return false if we can lower get_vector_length to a vsetvli intrinsic.
bool RISCVTargetLowering::shouldExpandGetVectorLength(EVT TripCountVT,
unsigned VF,
@@ -18613,6 +18616,33 @@ static SDValue combineAddMulh(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(RISCVISD::MULHSU, DL, VT, X, Mulh.getOperand(1));
}
+// Fold (add (add A, (vscale C1)), (vscale C2))
+// -> (add A, (vscale (C1 + C2)))
+// Optimize only when (add A, (vscale C1)) is not reused.
+static SDValue combineVscaleAdd(SDNode *N, SelectionDAG &DAG,
+ const RISCVSubtarget &Subtarget) {
+ EVT VT = N->getOperand(0).getValueType();
+
+ using namespace SDPatternMatch;
+ SDValue A, InnerAdd, VScale0, VScale1;
+
+ if (!sd_match(N, m_Add(m_OneUse(m_Value(InnerAdd,
+ m_Add(m_Value(A), m_Value(VScale0)))),
+ m_Value(VScale1))))
+ return SDValue();
+
+ SDLoc DL(N);
+ if (VScale0.getOpcode() != ISD::VSCALE || VScale1.getOpcode() != ISD::VSCALE)
+ return SDValue();
+
+ const APInt &VS1 = VScale0->getConstantOperandAPInt(0);
+ const APInt &VS2 = VScale1->getConstantOperandAPInt(0);
+
+ SDValue VS = DAG.getVScale(DL, VT, VS1 + VS2);
+
+ return DAG.getNode(ISD::ADD, DL, VT, A, VS);
+}
+
// Fold an add of a multiply-parts product into the accumulating form.
static SDValue combineAddMulParts(SDNode *N, SelectionDAG &DAG,
const RISCVSubtarget &Subtarget) {
@@ -18806,6 +18836,8 @@ static SDValue performADDCombine(SDNode *N,
return V;
if (SDValue V = combineAddMulh(N, DAG, Subtarget))
return V;
+ if (SDValue V = combineVscaleAdd(N, DAG, Subtarget))
+ return V;
// fold (add (select lhs, rhs, cc, 0, y), x) ->
// (select lhs, rhs, cc, x, (add x, y))
@@ -24449,6 +24481,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
break;
}
case ISD::ADD: {
+ LLVM_DEBUG(dbgs() << "ISD::ADD " << "\n");
if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
return V;
if (SDValue V = combineToVWMACC(N, DAG, Subtarget))
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index e700a0963e254..ca43c754e7757 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -613,6 +613,8 @@ class RISCVTargetLowering : public TargetLowering {
bool shouldExpandGetVectorLength(EVT TripCountVT, unsigned VF,
bool IsScalable) const override;
+ bool isDesirableToFoldVScaleAdd() const override;
+
/// RVV code generation for fixed length vectors does not lower all
/// BUILD_VECTORs. This makes BUILD_VECTOR legalisation a source of stores to
/// merge. However, merging them creates a BUILD_VECTOR that is just as
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
index 3b8ec4ab4b370..480f5cca7ad7b 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
@@ -186,32 +186,29 @@ define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
; CHECK-NEXT: neg x9, x8
; CHECK-NEXT: mov w10, #1000 // =0x3e8
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: rdvl x11, #4
; CHECK-NEXT: and x9, x9, x10
-; CHECK-NEXT: rdvl x10, #2
+; CHECK-NEXT: rdvl x10, #4
; CHECK-NEXT: .LBB2_1: // %vector.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add x12, x0, x10
-; CHECK-NEXT: add x13, x1, x10
; CHECK-NEXT: ldr z4, [x0, #1, mul vl]
; CHECK-NEXT: ldr z5, [x0]
-; CHECK-NEXT: ldr z6, [x1, #1, mul vl]
-; CHECK-NEXT: ldr z7, [x12, #1, mul vl]
+; CHECK-NEXT: subs x9, x9, x8
+; CHECK-NEXT: ldr z6, [x0, #3, mul vl]
+; CHECK-NEXT: ldr z7, [x1, #1, mul vl]
; CHECK-NEXT: ldr z16, [x1]
; CHECK-NEXT: ldr z17, [x0, #2, mul vl]
-; CHECK-NEXT: subs x9, x9, x8
-; CHECK-NEXT: ldr z18, [x13, #1, mul vl]
+; CHECK-NEXT: add x0, x0, x10
+; CHECK-NEXT: ldr z18, [x1, #3, mul vl]
; CHECK-NEXT: ldr z19, [x1, #2, mul vl]
-; CHECK-NEXT: add x1, x1, x11
-; CHECK-NEXT: fcmla z0.d, p0/m, z6.d, z4.d, #0
-; CHECK-NEXT: add x0, x0, x11
+; CHECK-NEXT: add x1, x1, x10
; CHECK-NEXT: fcmla z1.d, p0/m, z16.d, z5.d, #0
+; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #0
+; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z6.d, #0
; CHECK-NEXT: fcmla z2.d, p0/m, z19.d, z17.d, #0
-; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z7.d, #0
-; CHECK-NEXT: fcmla z0.d, p0/m, z6.d, z4.d, #90
; CHECK-NEXT: fcmla z1.d, p0/m, z16.d, z5.d, #90
+; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #90
+; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z6.d, #90
; CHECK-NEXT: fcmla z2.d, p0/m, z19.d, z17.d, #90
-; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z7.d, #90
; CHECK-NEXT: b.ne .LBB2_1
; CHECK-NEXT: // %bb.2: // %exit.block
; CHECK-NEXT: uzp1 z4.d, z2.d, z3.d
diff --git a/llvm/test/CodeGen/AArch64/misched-cutoff.mir b/llvm/test/CodeGen/AArch64/misched-cutoff.mir
index a61eb64318a39..94c94f51d3f4d 100644
--- a/llvm/test/CodeGen/AArch64/misched-cutoff.mir
+++ b/llvm/test/CodeGen/AArch64/misched-cutoff.mir
@@ -42,8 +42,8 @@ body: |
; CHECK-CUTOFF: liveins: $w1, $x0
; CHECK-CUTOFF-NEXT: {{ $}}
; CHECK-CUTOFF-NEXT: $w8 = LDRWui $x0, 1, implicit-def $x8 :: (load (s32) from %ir.0)
- ; CHECK-CUTOFF-NEXT: STRWui $w1, $x0, 2 :: (store (s32) into %ir.arrayidx1)
; CHECK-CUTOFF-NEXT: $w9 = LDRWui $x0, 0, implicit-def $x9 :: (load (s32) from %ir.arrayidx19, align 8)
+ ; CHECK-CUTOFF-NEXT: STRWui $w1, $x0, 2 :: (store (s32) into %ir.arrayidx1)
; CHECK-CUTOFF-NEXT: $x0 = ADDXrr killed $x9, killed $x8
; CHECK-CUTOFF-NEXT: RET_ReallyLR implicit $x0
$w8 = LDRWui $x0, 1, implicit-def $x8 :: (load (s32) from %ir.0)
diff --git a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
index 75d125d8beda2..747c134913931 100644
--- a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
+++ b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
@@ -464,13 +464,11 @@ define <vscale x 8 x i32> @splice_nxv8i32_idx(<vscale x 8 x i32> %a, <vscale x 8
; CHECK: // %bb.0:
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-4
-; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: mov x8, sp
; CHECK-NEXT: str z1, [sp, #1, mul vl]
-; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
-; CHECK-NEXT: orr x8, x9, #0x8
+; CHECK-NEXT: orr x8, x8, #0x8
+; CHECK-NEXT: str z3, [sp, #3, mul vl]
; CHECK-NEXT: str z2, [sp, #2, mul vl]
; CHECK-NEXT: ldr z0, [x8]
; CHECK-NEXT: ldr z1, [x8, #1, mul vl]
@@ -487,28 +485,22 @@ define <vscale x 16 x float> @splice_nxv16f32_16(<vscale x 16 x float> %a, <vsca
; CHECK: // %bb.0:
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-8
-; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: add x10, x9, x8
-; CHECK-NEXT: str z3, [x10, #1, mul vl]
-; CHECK-NEXT: rdvl x10, #4
-; CHECK-NEXT: add x10, x9, x10
+; CHECK-NEXT: str z3, [sp, #3, mul vl]
+; CHECK-NEXT: mov x8, #16 // =0x10
; CHECK-NEXT: str z2, [sp, #2, mul vl]
-; CHECK-NEXT: add x11, x10, x8
+; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: str z7, [x11, #1, mul vl]
-; CHECK-NEXT: add x11, x9, #64
-; CHECK-NEXT: str z5, [x10, #1, mul vl]
-; CHECK-NEXT: add x8, x11, x8
-; CHECK-NEXT: str z6, [x10, #2, mul vl]
-; CHECK-NEXT: mov x10, #16 // =0x10
+; CHECK-NEXT: str z7, [sp, #7, mul vl]
; CHECK-NEXT: str z4, [sp, #4, mul vl]
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x9, x10, lsl #2]
-; CHECK-NEXT: ldr z3, [x8, #1, mul vl]
-; CHECK-NEXT: ldr z1, [x11, #1, mul vl]
-; CHECK-NEXT: ldr z2, [x11, #2, mul vl]
+; CHECK-NEXT: str z5, [sp, #5, mul vl]
+; CHECK-NEXT: str z6, [sp, #6, mul vl]
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x9, x8, lsl #2]
+; CHECK-NEXT: add x8, x9, #64
+; CHECK-NEXT: ldr z1, [x8, #1, mul vl]
+; CHECK-NEXT: ldr z2, [x8, #2, mul vl]
+; CHECK-NEXT: ldr z3, [x8, #3, mul vl]
; CHECK-NEXT: addvl sp, sp, #8
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
@@ -1074,7 +1066,7 @@ define <vscale x 8 x i32> @splice_nxv8i32(<vscale x 8 x i32> %a, <vscale x 8 x i
; CHECK-NEXT: mov x9, #-8 // =0xfffffffffffffff8
; CHECK-NEXT: str z0, [sp]
; CHECK-NEXT: sub x10, x8, #32
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
+; CHECK-NEXT: str z3, [sp, #3, mul vl]
; CHECK-NEXT: str z2, [sp, #2, mul vl]
; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8, x9, lsl #2]
; CHECK-NEXT: ldr z1, [x10, #1, mul vl]
@@ -1091,28 +1083,24 @@ define <vscale x 16 x float> @splice_nxv16f32_neg17(<vscale x 16 x float> %a, <v
; CHECK: // %bb.0:
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-8
-; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: rdvl x8, #4
; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: add x10, x9, x8
-; CHECK-NEXT: str z3, [x10, #1, mul vl]
-; CHECK-NEXT: rdvl x10, #4
-; CHECK-NEXT: add x9, x9, x10
+; CHECK-NEXT: str z3, [sp, #3, mul vl]
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: mov x9, #-17 // =0xffffffffffffffef
; CHECK-NEXT: str z2, [sp, #2, mul vl]
-; CHECK-NEXT: add x10, x9, x8
-; CHECK-NEXT: sub x11, x9, #68
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: add x8, x11, x8
-; CHECK-NEXT: str z7, [x10, #1, mul vl]
-; CHECK-NEXT: mov x10, #-17 // =0xffffffffffffffef
-; CHECK-NEXT: str z5, [x9, #1, mul vl]
-; CHECK-NEXT: str z6, [x9, #2, mul vl]
+; CHECK-NEXT: str z7, [sp, #7, mul vl]
; CHECK-NEXT: str z4, [sp, #4, mul vl]
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x9, x10, lsl #2]
-; CHECK-NEXT: ldr z1, [x11, #1, mul vl]
-; CHECK-NEXT: ldr z2, [x11, #2, mul vl]
-; CHECK-NEXT: ldr z3, [x8, #1, mul vl]
+; CHECK-NEXT: str z5, [sp, #5, mul vl]
+; CHECK-NEXT: str z6, [sp, #6, mul vl]
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8, x9, lsl #2]
+; CHECK-NEXT: sub x8, x8, #68
+; CHECK-NEXT: ldr z1, [x8, #1, mul vl]
+; CHECK-NEXT: ldr z2, [x8, #2, mul vl]
+; CHECK-NEXT: ldr z3, [x8, #3, mul vl]
; CHECK-NEXT: addvl sp, sp, #8
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/nontemporal-load.ll b/llvm/test/CodeGen/AArch64/nontemporal-load.ll
index ec33dbefb9309..662a95e4477b7 100644
--- a/llvm/test/CodeGen/AArch64/nontemporal-load.ll
+++ b/llvm/test/CodeGen/AArch64/nontemporal-load.ll
@@ -609,29 +609,21 @@ define <vscale x 20 x float> @test_ldnp_v20f32_vscale(ptr %A) {
; CHECK-LE-LABEL: test_ldnp_v20f32_vscale:
; CHECK-LE: ; %bb.0:
; CHECK-LE-NEXT: ptrue p0.s
-; CHECK-LE-NEXT: rdvl x8, #1
-; CHECK-LE-NEXT: add x9, x0, x8
-; CHECK-LE-NEXT: ldnt1w { z2.s }, p0/z, [x9, #1, mul vl]
-; CHECK-LE-NEXT: add x9, x9, x8
-; CHECK-LE-NEXT: add x8, x9, x8
; CHECK-LE-NEXT: ldnt1w { z0.s }, p0/z, [x0]
; CHECK-LE-NEXT: ldnt1w { z1.s }, p0/z, [x0, #1, mul vl]
-; CHECK-LE-NEXT: ldnt1w { z3.s }, p0/z, [x9, #1, mul vl]
-; CHECK-LE-NEXT: ldnt1w { z4.s }, p0/z, [x8, #1, mul vl]
+; CHECK-LE-NEXT: ldnt1w { z2.s }, p0/z, [x0, #2, mul vl]
+; CHECK-LE-NEXT: ldnt1w { z3.s }, p0/z, [x0, #3, mul vl]
+; CHECK-LE-NEXT: ldnt1w { z4.s }, p0/z, [x0, #4, mul vl]
; CHECK-LE-NEXT: ret
;
; CHECK-BE-LABEL: test_ldnp_v20f32_vscale:
; CHECK-BE: // %bb.0:
; CHECK-BE-NEXT: ptrue p0.s
-; CHECK-BE-NEXT: rdvl x8, #1
-; CHECK-BE-NEXT: add x9, x0, x8
-; CHECK-BE-NEXT: ldnt1w { z2.s }, p0/z, [x9, #1, mul vl]
-; CHECK-BE-NEXT: add x9, x9, x8
-; CHECK-BE-NEXT: add x8, x9, x8
; CHECK-BE-NEXT: ldnt1w { z0.s }, p0/z, [x0]
; CHECK-BE-NEXT: ldnt1w { z1.s }, p0/z, [x0, #1, mul vl]
-; CHECK-BE-NEXT: ldnt1w { z3.s }, p0/z, [x9, #1, mul vl]
-; CHECK-BE-NEXT: ldnt1w { z4.s }, p0/z, [x8, #1, mul vl]
+; CHECK-BE-NEXT: ldnt1w { z2.s }, p0/z, [x0, #2, mul vl]
+; CHECK-BE-NEXT: ldnt1w { z3.s }, p0/z, [x0, #3, mul vl]
+; CHECK-BE-NEXT: ldnt1w { z4.s }, p0/z, [x0, #4, mul vl]
; CHECK-BE-NEXT: ret
%lv = load<vscale x 20 x float>, ptr %A, align 8, !nontemporal !0
ret <vscale x 20 x float> %lv
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
index 868e03bde6e52..f5129c7c57826 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
@@ -223,13 +223,11 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
; CHECK-LABEL: foo_ld4_nxv16i8_bad_mask2:
; CHECK: // %bb.0:
; CHECK-NEXT: zip1 p2.b, p1.b, p1.b
-; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: zip2 p1.b, p1.b, p1.b
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: zip2 p3.b, p0.b, p0.b
; CHECK-NEXT: ld1b { z3.b }, p2/z, [x0, #2, mul vl]
; CHECK-NEXT: zip1 p0.b, p0.b, p0.b
-; CHECK-NEXT: ld1b { z2.b }, p1/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1b { z2.b }, p1/z, [x0, #3, mul vl]
; CHECK-NEXT: ld1b { z0.b }, p3/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0]
; CHECK-NEXT: uzp1 z4.b, z3.b, z2.b
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index ccdbb21a6e92b..766777925e909 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -271,19 +271,17 @@ define void @foo_st4_nxv4i32_bad_mask2(<vscale x 8 x i1> %mask, <vscale x 4 x i3
; CHECK: // %bb.0:
; CHECK-NEXT: zip2 z4.s, z1.s, z3.s
; CHECK-NEXT: zip2 z5.s, z0.s, z2.s
-; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: zip2 p1.h, p0.h, p0.h
; CHECK-NEXT: zip1 z1.s, z1.s, z3.s
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: zip1 z0.s, z0.s, z2.s
; CHECK-NEXT: zip1 p0.h, p0.h, p0.h
-; CHECK-NEXT: zip2 z2.s, z5.s, z4.s
; CHECK-NEXT: punpkhi p2.h, p1.b
+; CHECK-NEXT: zip2 z2.s, z5.s, z4.s
; CHECK-NEXT: zip1 z3.s, z5.s, z4.s
; CHECK-NEXT: punpklo p1.h, p1.b
; CHECK-NEXT: zip2 z4.s, z0.s, z1.s
; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
-; CHECK-NEXT: st1w { z2.s }, p2, [x8, #1, mul vl]
+; CHECK-NEXT: st1w { z2.s }, p2, [x0, #3, mul vl]
; CHECK-NEXT: punpkhi p2.h, p0.b
; CHECK-NEXT: punpklo p0.h, p0.b
; CHECK-NEXT: st1w { z3.s }, p1, [x0, #2, mul vl]
diff --git a/llvm/test/CodeGen/AArch64/sve-aliasing.ll b/llvm/test/CodeGen/AArch64/sve-aliasing.ll
index 0c91472d2bc98..a27429a256250 100644
--- a/llvm/test/CodeGen/AArch64/sve-aliasing.ll
+++ b/llvm/test/CodeGen/AArch64/sve-aliasing.ll
@@ -458,21 +458,19 @@ define void @triple_v16i8(ptr noalias nocapture noundef %l0) {
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: add x8, x0, x8
-; CHECK-NEXT: movprfx z2, z0
-; CHECK-NEXT: mul z2.b, p0/m, z2.b, z0.b
-; CHECK-NEXT: movprfx z3, z1
-; CHECK-NEXT: mul z3.b, p0/m, z3.b, z1.b
-; CHECK-NEXT: eor z0.d, z2.d, z0.d
-; CHECK-NEXT: eor z1.d, z3.d, z1.d
+; CHECK-NEXT: ldr z2, [x0, #2, mul vl]
+; CHECK-NEXT: movprfx z3, z0
+; CHECK-NEXT: mul z3.b, p0/m, z3.b, z0.b
+; CHECK-NEXT: movprfx z4, z1
+; CHECK-NEXT: mul z4.b, p0/m, z4.b, z1.b
+; CHECK-NEXT: movprfx z5, z2
+; CHECK-NEXT: mul z5.b, p0/m, z5.b, z2.b
+; CHECK-NEXT: eor z0.d, z3.d, z0.d
+; CHECK-NEXT: eor z1.d, z4.d, z1.d
+; CHECK-NEXT: eor z2.d, z5.d, z2.d
; CHECK-NEXT: str z0, [x0]
; CHECK-NEXT: str z1, [x0, #1, mul vl]
-; CHECK-NEXT: ldr z0, [x8, #1, mul vl]
-; CHECK-NEXT: movprfx z1, z0
-; CHECK-NEXT: mul z1.b, p0/m, z1.b, z0.b
-; CHECK-NEXT: eor z0.d, z1.d, z0.d
-; CHECK-NEXT: str z0, [x8, #1, mul vl]
+; CHECK-NEXT: str z2, [x0, #2, mul vl]
; CHECK-NEXT: ret
%l3 = load <vscale x 16 x i8>, ptr %l0, align 16
%l5 = mul <vscale x 16 x i8> %l3, %l3
@@ -499,7 +497,6 @@ define void @negative_tripletooshort_v16i8(ptr noalias nocapture noundef %l0) {
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ptrue p0.b
; CHECK-NEXT: cntw x8
-; CHECK-NEXT: add x9, x0, x8
; CHECK-NEXT: movprfx z1, z0
; CHECK-NEXT: mul z1.b, p0/m, z1.b, z0.b
; CHECK-NEXT: eor z0.d, z1.d, z0.d
@@ -509,11 +506,12 @@ define void @negative_tripletooshort_v16i8(ptr noalias nocapture noundef %l0) {
; CHECK-NEXT: mul z1.b, p0/m, z1.b, z0.b
; CHECK-NEXT: eor z0.d, z1.d, z0.d
; CHECK-NEXT: st1b { z0.b }, p0, [x0, x8]
-; CHECK-NEXT: ld1b { z0.b }, p0/z, [x9, x8]
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x8]
; CHECK-NEXT: movprfx z1, z0
; CHECK-NEXT: mul z1.b, p0/m, z1.b, z0.b
; CHECK-NEXT: eor z0.d, z1.d, z0.d
-; CHECK-NEXT: st1b { z0.b }, p0, [x9, x8]
+; CHECK-NEXT: st1b { z0.b }, p0, [x0, x8]
; CHECK-NEXT: ret
%l3 = load <vscale x 16 x i8>, ptr %l0, align 16
%l5 = mul <vscale x 16 x i8> %l3, %l3
diff --git a/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll b/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll
index c8cb64ffe1091..3c5e788dabd56 100644
--- a/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll
+++ b/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll
@@ -12,17 +12,13 @@ define float @foo1(ptr %x0, ptr %x1, ptr %x2) nounwind {
; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: fmov s0, #1.00000000
-; CHECK-NEXT: add x9, x9, x8
; CHECK-NEXT: ld4d { z1.d - z4.d }, p0/z, [x0]
-; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: mov x0, sp
; CHECK-NEXT: ld4d { z16.d - z19.d }, p0/z, [x1]
; CHECK-NEXT: ld1d { z5.d }, p0/z, [x2]
-; CHECK-NEXT: str z19, [x8, #1, mul vl]
-; CHECK-NEXT: str z18, [x9, #1, mul vl]
+; CHECK-NEXT: str z19, [sp, #3, mul vl]
+; CHECK-NEXT: str z18, [sp, #2, mul vl]
; CHECK-NEXT: str z17, [sp, #1, mul vl]
; CHECK-NEXT: str z16, [sp]
; CHECK-NEXT: bl callee1
@@ -61,25 +57,21 @@ define float @foo2(ptr %x0, ptr %x1) nounwind {
; CHECK-NEXT: sub sp, sp, #16
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: add x9, sp, #16
-; CHECK-NEXT: add x10, x9, x8
; CHECK-NEXT: fmov s0, #1.00000000
+; CHECK-NEXT: add x8, sp, #16
; CHECK-NEXT: mov w2, #2 // =0x2
-; CHECK-NEXT: ld4d { z1.d - z4.d }, p0/z, [x0]
-; CHECK-NEXT: add x8, x10, x8
-; CHECK-NEXT: mov w0, wzr
; CHECK-NEXT: mov w3, #3 // =0x3
; CHECK-NEXT: mov w4, #4 // =0x4
+; CHECK-NEXT: ld4d { z1.d - z4.d }, p0/z, [x0]
+; CHECK-NEXT: mov w0, wzr
; CHECK-NEXT: mov w5, #5 // =0x5
; CHECK-NEXT: mov w6, #6 // =0x6
; CHECK-NEXT: mov w7, #7 // =0x7
; CHECK-NEXT: ld4d { z16.d - z19.d }, p0/z, [x1]
; CHECK-NEXT: mov w1, #1 // =0x1
-; CHECK-NEXT: str z19, [x8, #1, mul vl]
-; CHECK-NEXT: add x8, sp, #16
-; CHECK-NEXT: str z18, [x10, #1, mul vl]
-; CHECK-NEXT: str x9, [sp]
+; CHECK-NEXT: str x8, [sp]
+; CHECK-NEXT: str z19, [x8, #3, mul vl]
+; CHECK-NEXT: str z18, [x8, #2, mul vl]
; CHECK-NEXT: str z17, [x8, #1, mul vl]
; CHECK-NEXT: str z16, [x8]
; CHECK-NEXT: bl callee2
@@ -119,14 +111,11 @@ define float @foo3(ptr %x0, ptr %x1, ptr %x2) nounwind {
; CHECK-NEXT: ptrue p0.b
; CHECK-NEXT: fmov s0, #1.00000000
; CHECK-NEXT: fmov s1, #2.00000000
-; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: ld4d { z2.d - z5.d }, p0/z, [x0]
-; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: mov x0, sp
; CHECK-NEXT: ld3d { z16.d - z18.d }, p0/z, [x1]
; CHECK-NEXT: ld1d { z6.d }, p0/z, [x2]
-; CHECK-NEXT: str z18, [x8, #1, mul vl]
+; CHECK-NEXT: str z18, [sp, #2, mul vl]
; CHECK-NEXT: str z17, [sp, #1, mul vl]
; CHECK-NEXT: str z16, [sp]
; CHECK-NEXT: bl callee3
@@ -161,24 +150,18 @@ entry:
define double @foo4(double %x0, ptr %ptr1, ptr %ptr2, ptr %ptr3, <vscale x 8 x double> %x1, <vscale x 8 x double> %x2, <vscale x 2 x double> %x3) nounwind {
; CHECK-LABEL: foo4:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: ldr z24, [x3, #1, mul vl]
-; CHECK-NEXT: ldr z25, [x3]
-; CHECK-NEXT: add x9, x3, x8
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: ldr z7, [x9, #1, mul vl]
-; CHECK-NEXT: ldr z6, [x8, #1, mul vl]
-; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: ldr z6, [x3, #1, mul vl]
+; CHECK-NEXT: ldr z7, [x3]
+; CHECK-NEXT: ldr z24, [x3, #3, mul vl]
+; CHECK-NEXT: ldr z25, [x3, #2, mul vl]
+; CHECK-NEXT: str z4, [x0, #3, mul vl]
; CHECK-NEXT: str z3, [x0, #2, mul vl]
-; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: add x8, x1, x8
; CHECK-NEXT: str z2, [x0, #1, mul vl]
; CHECK-NEXT: str z1, [x0]
-; CHECK-NEXT: str z4, [x9, #1, mul vl]
-; CHECK-NEXT: str z7, [x1, #2, mul vl]
-; CHECK-NEXT: str z25, [x1]
-; CHECK-NEXT: str z24, [x1, #1, mul vl]
-; CHECK-NEXT: str z6, [x8, #1, mul vl]
+; CHECK-NEXT: str z25, [x1, #2, mul vl]
+; CHECK-NEXT: str z24, [x1, #3, mul vl]
+; CHECK-NEXT: str z7, [x1]
+; CHECK-NEXT: str z6, [x1, #1, mul vl]
; CHECK-NEXT: str z5, [x2]
; CHECK-NEXT: ret
entry:
@@ -192,24 +175,18 @@ define double @foo5(i32 %i0, i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, ptr %p
; CHECK-LABEL: foo5:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr x8, [sp]
-; CHECK-NEXT: rdvl x9, #1
-; CHECK-NEXT: add x10, x8, x9
; CHECK-NEXT: ldr z5, [x8, #1, mul vl]
; CHECK-NEXT: ldr z6, [x8]
-; CHECK-NEXT: add x9, x10, x9
-; CHECK-NEXT: ldr z7, [x10, #1, mul vl]
-; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: ldr z24, [x9, #1, mul vl]
-; CHECK-NEXT: add x9, x6, x8
-; CHECK-NEXT: add x8, x7, x8
+; CHECK-NEXT: ldr z7, [x8, #3, mul vl]
+; CHECK-NEXT: ldr z24, [x8, #2, mul vl]
+; CHECK-NEXT: str z4, [x6, #3, mul vl]
; CHECK-NEXT: str z3, [x6, #2, mul vl]
; CHECK-NEXT: str z2, [x6, #1, mul vl]
; CHECK-NEXT: str z1, [x6]
-; CHECK-NEXT: str z4, [x9, #1, mul vl]
-; CHECK-NEXT: str z7, [x7, #2, mul vl]
+; CHECK-NEXT: str z24, [x7, #2, mul vl]
+; CHECK-NEXT: str z7, [x7, #3, mul vl]
; CHECK-NEXT: str z6, [x7]
; CHECK-NEXT: str z5, [x7, #1, mul vl]
-; CHECK-NEXT: str z24, [x8, #1, mul vl]
; CHECK-NEXT: ret
entry:
store volatile <vscale x 8 x double> %x1, ptr %ptr1
@@ -220,21 +197,16 @@ entry:
define double @foo6(double %x0, double %x1, ptr %ptr1, ptr %ptr2, <vscale x 8 x double> %x2, <vscale x 6 x double> %x3) nounwind {
; CHECK-LABEL: foo6:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: ldr z6, [x2]
+; CHECK-NEXT: ldr z1, [x2]
+; CHECK-NEXT: ldr z6, [x2, #2, mul vl]
; CHECK-NEXT: ldr z7, [x2, #1, mul vl]
-; CHECK-NEXT: add x9, x2, x8
-; CHECK-NEXT: add x8, x1, x8
-; CHECK-NEXT: ldr z1, [x9, #1, mul vl]
-; CHECK-NEXT: rdvl x9, #2
+; CHECK-NEXT: str z5, [x0, #3, mul vl]
; CHECK-NEXT: str z4, [x0, #2, mul vl]
-; CHECK-NEXT: add x9, x0, x9
; CHECK-NEXT: str z3, [x0, #1, mul vl]
; CHECK-NEXT: str z2, [x0]
-; CHECK-NEXT: str z5, [x9, #1, mul vl]
; CHECK-NEXT: str z7, [x1, #1, mul vl]
-; CHECK-NEXT: str z1, [x8, #1, mul vl]
-; CHECK-NEXT: str z6, [x1]
+; CHECK-NEXT: str z6, [x1, #2, mul vl]
+; CHECK-NEXT: str z1, [x1]
; CHECK-NEXT: ret
entry:
store volatile <vscale x 8 x double> %x2, ptr %ptr1
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
index 0e8cc3abe70de..3473b063e35ee 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
@@ -10,10 +10,7 @@ define <4 x i32> @extract_v4i32_nxv16i32_12(<vscale x 16 x i32> %arg) {
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
+; CHECK-NEXT: str z3, [sp, #3, mul vl]
; CHECK-NEXT: str z2, [sp, #2, mul vl]
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
@@ -49,10 +46,7 @@ define <4 x i16> @extract_v4i16_nxv32i16_8(<vscale x 32 x i16> %arg) {
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
+; CHECK-NEXT: str z3, [sp, #3, mul vl]
; CHECK-NEXT: str z2, [sp, #2, mul vl]
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
@@ -73,22 +67,19 @@ define <2 x i16> @extract_v2i16_nxv32i16_8(<vscale x 32 x i16> %arg) {
; CHECK-NEXT: addvl sp, sp, #-8
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xc0, 0x00, 0x1e, 0x22 // sp + 16 + 64 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: addvl x11, sp, #4
-; CHECK-NEXT: add x10, x9, x8
-; CHECK-NEXT: add x8, x11, x8
-; CHECK-NEXT: str z3, [x10, #1, mul vl]
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
-; CHECK-NEXT: add x8, x9, #32
+; CHECK-NEXT: mov x8, sp
+; CHECK-NEXT: str z3, [sp, #3, mul vl]
; CHECK-NEXT: str z2, [sp, #2, mul vl]
+; CHECK-NEXT: add x8, x8, #32
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
+; CHECK-NEXT: str z3, [sp, #7, mul vl]
; CHECK-NEXT: str z2, [sp, #6, mul vl]
; CHECK-NEXT: str z1, [sp, #5, mul vl]
; CHECK-NEXT: str z0, [sp, #4, mul vl]
; CHECK-NEXT: ld1 { v0.h }[0], [x8]
-; CHECK-NEXT: add x8, x11, #34
+; CHECK-NEXT: addvl x8, sp, #4
+; CHECK-NEXT: add x8, x8, #34
; CHECK-NEXT: ld1 { v0.h }[2], [x8]
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: addvl sp, sp, #8
@@ -105,19 +96,17 @@ define <2 x i64> @extract_v2i64_nxv8i64_8(<vscale x 8 x i64> %arg) {
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: cnth x10
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: sub x10, x10, #2
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
-; CHECK-NEXT: mov w8, #8 // =0x8
-; CHECK-NEXT: cmp x10, #8
-; CHECK-NEXT: csel x8, x10, x8, lo
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: mov w9, #8 // =0x8
+; CHECK-NEXT: str z3, [sp, #3, mul vl]
+; CHECK-NEXT: sub x8, x8, #2
; CHECK-NEXT: str z2, [sp, #2, mul vl]
-; CHECK-NEXT: lsl x8, x8, #3
+; CHECK-NEXT: cmp x8, #8
; CHECK-NEXT: str z1, [sp, #1, mul vl]
+; CHECK-NEXT: csel x8, x8, x9, lo
; CHECK-NEXT: str z0, [sp]
+; CHECK-NEXT: mov x9, sp
+; CHECK-NEXT: lsl x8, x8, #3
; CHECK-NEXT: ldr q0, [x9, x8]
; CHECK-NEXT: addvl sp, sp, #4
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -133,10 +122,7 @@ define <4 x float> @extract_v4f32_nxv16f32_12(<vscale x 16 x float> %arg) {
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
+; CHECK-NEXT: str z3, [sp, #3, mul vl]
; CHECK-NEXT: str z2, [sp, #2, mul vl]
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
@@ -179,16 +165,13 @@ define <4 x i1> @extract_v4i1_nxv32i1_16(<vscale x 32 x i1> %arg) {
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: punpkhi p2.h, p1.b
-; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: punpklo p1.h, p1.b
-; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: mov z0.h, p2/z, #1 // =0x1
; CHECK-NEXT: punpkhi p2.h, p0.b
; CHECK-NEXT: punpklo p0.h, p0.b
; CHECK-NEXT: mov z1.h, p1/z, #1 // =0x1
; CHECK-NEXT: mov z2.h, p2/z, #1 // =0x1
-; CHECK-NEXT: str z0, [x8, #1, mul vl]
+; CHECK-NEXT: str z0, [sp, #3, mul vl]
; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1
; CHECK-NEXT: str z1, [sp, #2, mul vl]
; CHECK-NEXT: str z2, [sp, #1, mul vl]
diff --git a/llvm/test/CodeGen/AArch64/sve-fptrunc-store.ll b/llvm/test/CodeGen/AArch64/sve-fptrunc-store.ll
index 8b4eae8789eac..573958771658c 100644
--- a/llvm/test/CodeGen/AArch64/sve-fptrunc-store.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fptrunc-store.ll
@@ -64,20 +64,18 @@ entry:
define void @fptrunc8_f64_f16(ptr %dst, ptr %src) {
; CHECK-LABEL: fptrunc8_f64_f16:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: ldr z0, [x1, #1, mul vl]
-; CHECK-NEXT: ldr z2, [x1, #2, mul vl]
-; CHECK-NEXT: add x8, x1, x8
-; CHECK-NEXT: ldr z3, [x1]
+; CHECK-NEXT: ldr z0, [x1, #3, mul vl]
+; CHECK-NEXT: ldr z1, [x1]
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ldr z1, [x8, #1, mul vl]
+; CHECK-NEXT: ldr z2, [x1, #1, mul vl]
+; CHECK-NEXT: ldr z3, [x1, #2, mul vl]
; CHECK-NEXT: fcvt z0.h, p0/m, z0.d
-; CHECK-NEXT: fcvt z2.h, p0/m, z2.d
-; CHECK-NEXT: fcvt z3.h, p0/m, z3.d
; CHECK-NEXT: fcvt z1.h, p0/m, z1.d
+; CHECK-NEXT: fcvt z3.h, p0/m, z3.d
+; CHECK-NEXT: fcvt z2.h, p0/m, z2.d
; CHECK-NEXT: uzp1 z0.s, z3.s, z0.s
-; CHECK-NEXT: uzp1 z1.s, z2.s, z1.s
-; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z1.s, z1.s, z2.s
+; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
; CHECK-NEXT: str z0, [x0]
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll b/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
index 06fc5c395a8d7..66068e86ce8a7 100644
--- a/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
+++ b/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
@@ -79,20 +79,18 @@ define <vscale x 16 x i8> @narrow_i64_gather_index_i8_zext(ptr %out, ptr %in, <v
; CHECK-LABEL: narrow_i64_gather_index_i8_zext:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: cnth x8
-; CHECK-NEXT: add x9, x1, x2
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: ld1b { z0.s }, p0/z, [x1, x2]
-; CHECK-NEXT: ld1b { z1.s }, p0/z, [x9, #1, mul vl]
-; CHECK-NEXT: ld1b { z2.s }, p0/z, [x9, #2, mul vl]
+; CHECK-NEXT: add x8, x1, x2
+; CHECK-NEXT: ld1b { z0.s }, p0/z, [x8, #3, mul vl]
+; CHECK-NEXT: ld1b { z1.s }, p0/z, [x8, #2, mul vl]
+; CHECK-NEXT: ld1b { z2.s }, p0/z, [x1, x2]
; CHECK-NEXT: ld1b { z3.s }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ld1b { z0.s }, p0/z, [x1, z0.s, uxtw]
; CHECK-NEXT: ld1b { z1.s }, p0/z, [x1, z1.s, uxtw]
; CHECK-NEXT: ld1b { z2.s }, p0/z, [x1, z2.s, uxtw]
; CHECK-NEXT: ld1b { z3.s }, p0/z, [x1, z3.s, uxtw]
-; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
; CHECK-NEXT: uzp1 z1.h, z2.h, z3.h
-; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT: uzp1 z0.b, z1.b, z0.b
; CHECK-NEXT: ret
%1 = getelementptr inbounds i8, ptr %in, i64 %ptr
%2 = bitcast ptr %1 to ptr
@@ -107,20 +105,18 @@ define <vscale x 16 x i8> @narrow_i64_gather_index_i8_sext(ptr %out, ptr %in, <v
; CHECK-LABEL: narrow_i64_gather_index_i8_sext:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: cnth x8
-; CHECK-NEXT: add x9, x1, x2
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: ld1sb { z0.s }, p0/z, [x1, x2]
-; CHECK-NEXT: ld1sb { z1.s }, p0/z, [x9, #1, mul vl]
-; CHECK-NEXT: ld1sb { z2.s }, p0/z, [x9, #2, mul vl]
+; CHECK-NEXT: add x8, x1, x2
+; CHECK-NEXT: ld1sb { z0.s }, p0/z, [x8, #3, mul vl]
+; CHECK-NEXT: ld1sb { z1.s }, p0/z, [x8, #2, mul vl]
+; CHECK-NEXT: ld1sb { z2.s }, p0/z, [x1, x2]
; CHECK-NEXT: ld1sb { z3.s }, p0/z, [x8, #1, mul vl]
; CHECK-NEXT: ld1b { z0.s }, p0/z, [x1, z0.s, sxtw]
; CHECK-NEXT: ld1b { z1.s }, p0/z, [x1, z1.s, sxtw]
; CHECK-NEXT: ld1b { z2.s }, p0/z, [x1, z2.s, sxtw]
; CHECK-NEXT: ld1b { z3.s }, p0/z, [x1, z3.s, sxtw]
-; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
; CHECK-NEXT: uzp1 z1.h, z2.h, z3.h
-; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT: uzp1 z0.b, z1.b, z0.b
; CHECK-NEXT: ret
%1 = getelementptr inbounds i8, ptr %in, i64 %ptr
%2 = bitcast ptr %1 to ptr
diff --git a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll
index 09ee82b4fce2d..041a065687b05 100644
--- a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll
@@ -138,19 +138,14 @@ define <vscale x 16 x i8> @insert_v16i8_nxv16i8_idx16(<vscale x 16 x i8> %vec, <
define void @insert_nxv8i64_nxv16i64(<vscale x 8 x i64> %sv0, <vscale x 8 x i64> %sv1, ptr %out) {
; CHECK-LABEL: insert_nxv8i64_nxv16i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #4
-; CHECK-NEXT: rdvl x9, #2
+; CHECK-NEXT: str z7, [x0, #7, mul vl]
+; CHECK-NEXT: str z6, [x0, #6, mul vl]
+; CHECK-NEXT: str z5, [x0, #5, mul vl]
; CHECK-NEXT: str z4, [x0, #4, mul vl]
-; CHECK-NEXT: add x8, x0, x8
+; CHECK-NEXT: str z3, [x0, #3, mul vl]
; CHECK-NEXT: str z2, [x0, #2, mul vl]
-; CHECK-NEXT: add x10, x0, x9
; CHECK-NEXT: str z1, [x0, #1, mul vl]
; CHECK-NEXT: str z0, [x0]
-; CHECK-NEXT: str z6, [x8, #2, mul vl]
-; CHECK-NEXT: str z5, [x8, #1, mul vl]
-; CHECK-NEXT: add x8, x8, x9
-; CHECK-NEXT: str z3, [x10, #1, mul vl]
-; CHECK-NEXT: str z7, [x8, #1, mul vl]
; CHECK-NEXT: ret
%v0 = call <vscale x 16 x i64> @llvm.vector.insert.nxv8i64.nxv16i64(<vscale x 16 x i64> poison, <vscale x 8 x i64> %sv0, i64 0)
%v = call <vscale x 16 x i64> @llvm.vector.insert.nxv8i64.nxv16i64(<vscale x 16 x i64> %v0, <vscale x 8 x i64> %sv1, i64 8)
@@ -161,12 +156,10 @@ define void @insert_nxv8i64_nxv16i64(<vscale x 8 x i64> %sv0, <vscale x 8 x i64>
define void @insert_nxv8i64_nxv16i64_lo(<vscale x 8 x i64> %sv0, ptr %out) {
; CHECK-LABEL: insert_nxv8i64_nxv16i64_lo:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: str z3, [x0, #3, mul vl]
; CHECK-NEXT: str z2, [x0, #2, mul vl]
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: str z1, [x0, #1, mul vl]
; CHECK-NEXT: str z0, [x0]
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
; CHECK-NEXT: ret
%v = call <vscale x 16 x i64> @llvm.vector.insert.nxv8i64.nxv16i64(<vscale x 16 x i64> poison, <vscale x 8 x i64> %sv0, i64 0)
store <vscale x 16 x i64> %v, ptr %out
@@ -176,14 +169,10 @@ define void @insert_nxv8i64_nxv16i64_lo(<vscale x 8 x i64> %sv0, ptr %out) {
define void @insert_nxv8i64_nxv16i64_hi(<vscale x 8 x i64> %sv0, ptr %out) {
; CHECK-LABEL: insert_nxv8i64_nxv16i64_hi:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #4
-; CHECK-NEXT: rdvl x9, #2
+; CHECK-NEXT: str z3, [x0, #7, mul vl]
+; CHECK-NEXT: str z2, [x0, #6, mul vl]
+; CHECK-NEXT: str z1, [x0, #5, mul vl]
; CHECK-NEXT: str z0, [x0, #4, mul vl]
-; CHECK-NEXT: add x8, x0, x8
-; CHECK-NEXT: str z2, [x8, #2, mul vl]
-; CHECK-NEXT: str z1, [x8, #1, mul vl]
-; CHECK-NEXT: add x8, x8, x9
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
; CHECK-NEXT: ret
%v = call <vscale x 16 x i64> @llvm.vector.insert.nxv8i64.nxv16i64(<vscale x 16 x i64> poison, <vscale x 8 x i64> %sv0, i64 8)
store <vscale x 16 x i64> %v, ptr %out
@@ -198,18 +187,14 @@ define void @insert_v2i64_nxv16i64(<2 x i64> %sv0, <2 x i64> %sv1, ptr %out) uwt
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
-; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT: str z0, [sp]
; CHECK-NEXT: str q1, [sp, #32]
-; CHECK-NEXT: add x9, x9, x8
-; CHECK-NEXT: add x8, x0, x8
-; CHECK-NEXT: ldr z0, [x9, #1, mul vl]
+; CHECK-NEXT: ldr z0, [sp, #3, mul vl]
; CHECK-NEXT: ldr z1, [sp, #2, mul vl]
; CHECK-NEXT: ldr z2, [sp, #1, mul vl]
; CHECK-NEXT: ldr z3, [sp]
-; CHECK-NEXT: str z0, [x8, #1, mul vl]
+; CHECK-NEXT: str z0, [x0, #3, mul vl]
; CHECK-NEXT: str z1, [x0, #2, mul vl]
; CHECK-NEXT: str z2, [x0, #1, mul vl]
; CHECK-NEXT: str z3, [x0]
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
index 5103bbb7258a5..9698f1a6768fd 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
@@ -9,12 +9,10 @@ define <vscale x 16 x i32> @ld1b_i8_sext_i32(ptr %base) {
; CHECK-LABEL: ld1b_i8_sext_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: cnth x8
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1sb { z0.s }, p0/z, [x0]
; CHECK-NEXT: ld1sb { z1.s }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z2.s }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1sb { z3.s }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1sb { z3.s }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ret
%wide.load = load <vscale x 16 x i8>, ptr %base
%res = sext <vscale x 16 x i8> %wide.load to <vscale x 16 x i32>
@@ -25,12 +23,10 @@ define <vscale x 16 x i32> @ld1b_i8_zext_i32(ptr %base) {
; CHECK-LABEL: ld1b_i8_zext_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: cnth x8
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0]
; CHECK-NEXT: ld1b { z1.s }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z2.s }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1b { z3.s }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1b { z3.s }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ret
%wide.load = load <vscale x 16 x i8>, ptr %base
%res = zext <vscale x 16 x i8> %wide.load to <vscale x 16 x i32>
@@ -40,20 +36,15 @@ define <vscale x 16 x i32> @ld1b_i8_zext_i32(ptr %base) {
define <vscale x 16 x i64> @ld1b_i8_sext(ptr %base) {
; CHECK-LABEL: ld1b_i8_sext:
; CHECK: // %bb.0:
-; CHECK-NEXT: cnth x9
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cntw x8
-; CHECK-NEXT: add x9, x0, x9
-; CHECK-NEXT: add x10, x0, x8
-; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1sb { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z2.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ld1sb { z4.d }, p0/z, [x0, #4, mul vl]
-; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x10, #1, mul vl]
-; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x9, #1, mul vl]
-; CHECK-NEXT: ld1sb { z6.d }, p0/z, [x9, #2, mul vl]
-; CHECK-NEXT: ld1sb { z7.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x0, #5, mul vl]
+; CHECK-NEXT: ld1sb { z6.d }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT: ld1sb { z7.d }, p0/z, [x0, #7, mul vl]
; CHECK-NEXT: ret
%wide.load = load <vscale x 16 x i8>, ptr %base
%res = sext <vscale x 16 x i8> %wide.load to <vscale x 16 x i64>
@@ -63,20 +54,15 @@ define <vscale x 16 x i64> @ld1b_i8_sext(ptr %base) {
define <vscale x 16 x i64> @ld1b_i8_zext(ptr %base) {
; CHECK-LABEL: ld1b_i8_zext:
; CHECK: // %bb.0:
-; CHECK-NEXT: cnth x9
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cntw x8
-; CHECK-NEXT: add x9, x0, x9
-; CHECK-NEXT: add x10, x0, x8
-; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1b { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z2.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: ld1b { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ld1b { z4.d }, p0/z, [x0, #4, mul vl]
-; CHECK-NEXT: ld1b { z3.d }, p0/z, [x10, #1, mul vl]
-; CHECK-NEXT: ld1b { z5.d }, p0/z, [x9, #1, mul vl]
-; CHECK-NEXT: ld1b { z6.d }, p0/z, [x9, #2, mul vl]
-; CHECK-NEXT: ld1b { z7.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1b { z5.d }, p0/z, [x0, #5, mul vl]
+; CHECK-NEXT: ld1b { z6.d }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT: ld1b { z7.d }, p0/z, [x0, #7, mul vl]
; CHECK-NEXT: ret
%wide.load = load <vscale x 16 x i8>, ptr %base
%res = zext <vscale x 16 x i8> %wide.load to <vscale x 16 x i64>
@@ -91,12 +77,10 @@ define <vscale x 8 x i64> @ld1h_i16_sext(ptr %base) {
; CHECK-LABEL: ld1h_i16_sext:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cnth x8
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1sh { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1sh { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sh { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1sh { z3.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1sh { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ret
%wide.load = load <vscale x 8 x i16>, ptr %base
%res = sext <vscale x 8 x i16> %wide.load to <vscale x 8 x i64>
@@ -107,12 +91,10 @@ define <vscale x 8 x i64> @ld1h_i16_zext(ptr %base) {
; CHECK-LABEL: ld1h_i16_zext:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cnth x8
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1h { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1h { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1h { z3.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1h { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ret
%wide.load = load <vscale x 8 x i16>, ptr %base
%res = zext <vscale x 8 x i16> %wide.load to <vscale x 8 x i64>
@@ -190,12 +172,10 @@ define <vscale x 8 x i64> @zload_8i8_8i64(ptr %a) {
; CHECK-LABEL: zload_8i8_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cntw x8
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1b { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1b { z3.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1b { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ret
%aval = load <vscale x 8 x i8>, ptr %a
%aext = zext <vscale x 8 x i8> %aval to <vscale x 8 x i64>
@@ -242,12 +222,10 @@ define <vscale x 8 x i64> @sload_8i8_8i64(ptr %a) {
; CHECK-LABEL: sload_8i8_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cntw x8
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1sb { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ret
%aval = load <vscale x 8 x i8>, ptr %a
%aext = sext <vscale x 8 x i8> %aval to <vscale x 8 x i64>
diff --git a/llvm/test/CodeGen/AArch64/sve-ldst-sext.ll b/llvm/test/CodeGen/AArch64/sve-ldst-sext.ll
index e05d37dcec07c..78abf3239343c 100644
--- a/llvm/test/CodeGen/AArch64/sve-ldst-sext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-ldst-sext.ll
@@ -72,12 +72,10 @@ define <vscale x 8 x i64> @sload_nxv8i16(ptr %a) {
; CHECK-LABEL: sload_nxv8i16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cnth x8
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1sh { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1sh { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sh { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1sh { z3.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1sh { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ret
%load = load <vscale x 8 x i16>, ptr %a, align 2
%ext = sext <vscale x 8 x i16> %load to <vscale x 8 x i64>
@@ -152,12 +150,10 @@ define <vscale x 8 x i64> @sload_8i8_8i64(ptr %a) {
; CHECK-LABEL: sload_8i8_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cntw x8
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1sb { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ret
%aval = load <vscale x 8 x i8>, ptr %a, align 1
%aext = sext <vscale x 8 x i8> %aval to <vscale x 8 x i64>
@@ -224,16 +220,13 @@ define <vscale x 8 x i32> @sload_x2_8i8_8i32(ptr %a, ptr %b) {
define <vscale x 8 x i64> @sload_x2_8i8_8i64(ptr %a, ptr %b) {
; CHECK-LABEL: sload_x2_8i8_8i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: cntw x8
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: add x8, x1, x8
-; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x9, #1, mul vl]
+; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ld1sb { z2.d }, p0/z, [x0, #2, mul vl]
; CHECK-NEXT: ld1sb { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1sb { z4.d }, p0/z, [x1]
-; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x1, #3, mul vl]
; CHECK-NEXT: ld1sb { z6.d }, p0/z, [x1, #2, mul vl]
; CHECK-NEXT: ld1sb { z7.d }, p0/z, [x1, #1, mul vl]
; CHECK-NEXT: add z0.d, z0.d, z4.d
@@ -384,31 +377,21 @@ define <vscale x 16 x i64> @load_frozen_before_zext_multiuse5_dst_illegal(ptr %s
; CHECK-LABEL: load_frozen_before_zext_multiuse5_dst_illegal:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cntw x8
; CHECK-NEXT: mov z24.d, #3 // =0x3
-; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: rdvl x10, #4
-; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x9, #1, mul vl]
-; CHECK-NEXT: cnth x9
; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]
-; CHECK-NEXT: add x9, x0, x9
; CHECK-NEXT: ld1sb { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ld1sb { z4.d }, p0/z, [x0, #4, mul vl]
-; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x9, #1, mul vl]
-; CHECK-NEXT: ld1sb { z6.d }, p0/z, [x9, #2, mul vl]
-; CHECK-NEXT: add x9, x1, x10
-; CHECK-NEXT: rdvl x10, #2
-; CHECK-NEXT: ld1sb { z7.d }, p0/z, [x8, #1, mul vl]
-; CHECK-NEXT: add x8, x9, x10
-; CHECK-NEXT: str z24, [x9, #2, mul vl]
-; CHECK-NEXT: str z24, [x8, #1, mul vl]
-; CHECK-NEXT: add x8, x1, x10
+; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x0, #5, mul vl]
+; CHECK-NEXT: ld1sb { z6.d }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT: ld1sb { z7.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: str z24, [x1, #6, mul vl]
+; CHECK-NEXT: str z24, [x1, #7, mul vl]
; CHECK-NEXT: str z24, [x1, #4, mul vl]
-; CHECK-NEXT: str z24, [x9, #1, mul vl]
+; CHECK-NEXT: str z24, [x1, #5, mul vl]
; CHECK-NEXT: str z24, [x1, #2, mul vl]
-; CHECK-NEXT: str z24, [x8, #1, mul vl]
+; CHECK-NEXT: str z24, [x1, #3, mul vl]
; CHECK-NEXT: str z24, [x1]
; CHECK-NEXT: str z24, [x1, #1, mul vl]
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-ldst-zext.ll b/llvm/test/CodeGen/AArch64/sve-ldst-zext.ll
index f4d3677effc88..0a1cf0cacbf2f 100644
--- a/llvm/test/CodeGen/AArch64/sve-ldst-zext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-ldst-zext.ll
@@ -72,12 +72,10 @@ define <vscale x 8 x i64> @zload_nxv8i16(ptr %a) {
; CHECK-LABEL: zload_nxv8i16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cnth x8
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1h { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1h { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1h { z3.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1h { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ret
%load = load <vscale x 8 x i16>, ptr %a, align 2
%ext = zext <vscale x 8 x i16> %load to <vscale x 8 x i64>
@@ -152,12 +150,10 @@ define <vscale x 8 x i64> @zload_8i8_8i64(ptr %a) {
; CHECK-LABEL: zload_8i8_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cntw x8
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1b { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1b { z3.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1b { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ret
%aval = load <vscale x 8 x i8>, ptr %a, align 1
%aext = zext <vscale x 8 x i8> %aval to <vscale x 8 x i64>
@@ -224,16 +220,13 @@ define <vscale x 8 x i32> @zload_x2_8i8_8i32(ptr %a, ptr %b) {
define <vscale x 8 x i64> @zload_x2_8i8_8i64(ptr %a, ptr %b) {
; CHECK-LABEL: zload_x2_8i8_8i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: cntw x8
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: add x8, x1, x8
-; CHECK-NEXT: ld1b { z3.d }, p0/z, [x9, #1, mul vl]
+; CHECK-NEXT: ld1b { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ld1b { z2.d }, p0/z, [x0, #2, mul vl]
; CHECK-NEXT: ld1b { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1b { z4.d }, p0/z, [x1]
-; CHECK-NEXT: ld1b { z5.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1b { z5.d }, p0/z, [x1, #3, mul vl]
; CHECK-NEXT: ld1b { z6.d }, p0/z, [x1, #2, mul vl]
; CHECK-NEXT: ld1b { z7.d }, p0/z, [x1, #1, mul vl]
; CHECK-NEXT: add z0.d, z0.d, z4.d
@@ -375,31 +368,21 @@ define <vscale x 16 x i64> @load_frozen_before_zext_multiuse5_dst_illegal(ptr %s
; CHECK-LABEL: load_frozen_before_zext_multiuse5_dst_illegal:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: cntw x8
; CHECK-NEXT: mov z24.d, #3 // =0x3
-; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: rdvl x10, #4
-; CHECK-NEXT: ld1b { z3.d }, p0/z, [x9, #1, mul vl]
-; CHECK-NEXT: cnth x9
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
-; CHECK-NEXT: add x9, x0, x9
; CHECK-NEXT: ld1b { z1.d }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z2.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: ld1b { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ld1b { z4.d }, p0/z, [x0, #4, mul vl]
-; CHECK-NEXT: ld1b { z5.d }, p0/z, [x9, #1, mul vl]
-; CHECK-NEXT: ld1b { z6.d }, p0/z, [x9, #2, mul vl]
-; CHECK-NEXT: add x9, x1, x10
-; CHECK-NEXT: rdvl x10, #2
-; CHECK-NEXT: ld1b { z7.d }, p0/z, [x8, #1, mul vl]
-; CHECK-NEXT: add x8, x9, x10
-; CHECK-NEXT: str z24, [x9, #2, mul vl]
-; CHECK-NEXT: str z24, [x8, #1, mul vl]
-; CHECK-NEXT: add x8, x1, x10
+; CHECK-NEXT: ld1b { z5.d }, p0/z, [x0, #5, mul vl]
+; CHECK-NEXT: ld1b { z6.d }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT: ld1b { z7.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT: str z24, [x1, #6, mul vl]
+; CHECK-NEXT: str z24, [x1, #7, mul vl]
; CHECK-NEXT: str z24, [x1, #4, mul vl]
-; CHECK-NEXT: str z24, [x9, #1, mul vl]
+; CHECK-NEXT: str z24, [x1, #5, mul vl]
; CHECK-NEXT: str z24, [x1, #2, mul vl]
-; CHECK-NEXT: str z24, [x8, #1, mul vl]
+; CHECK-NEXT: str z24, [x1, #3, mul vl]
; CHECK-NEXT: str z24, [x1]
; CHECK-NEXT: str z24, [x1, #1, mul vl]
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-llrint.ll b/llvm/test/CodeGen/AArch64/sve-llrint.ll
index 9e9ef518f5029..8ecf378d5623d 100644
--- a/llvm/test/CodeGen/AArch64/sve-llrint.ll
+++ b/llvm/test/CodeGen/AArch64/sve-llrint.ll
@@ -126,87 +126,77 @@ define <vscale x 32 x i64> @llrint_v32i64_v32f16(<vscale x 32 x half> %x) {
; CHECK: // %bb.0:
; CHECK-NEXT: uunpkhi z4.s, z3.h
; CHECK-NEXT: uunpklo z3.s, z3.h
-; CHECK-NEXT: rdvl x10, #8
-; CHECK-NEXT: uunpkhi z5.s, z2.h
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: rdvl x11, #4
+; CHECK-NEXT: uunpkhi z7.s, z2.h
; CHECK-NEXT: uunpklo z2.s, z2.h
-; CHECK-NEXT: uunpklo z27.s, z1.h
-; CHECK-NEXT: rdvl x9, #2
-; CHECK-NEXT: uunpkhi z1.s, z1.h
-; CHECK-NEXT: uunpklo z25.s, z0.h
-; CHECK-NEXT: add x10, x8, x10
-; CHECK-NEXT: uunpklo z6.d, z4.s
-; CHECK-NEXT: uunpkhi z7.d, z3.s
-; CHECK-NEXT: add x12, x10, x11
+; CHECK-NEXT: uunpkhi z24.s, z0.h
+; CHECK-NEXT: uunpkhi z25.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpkhi z5.d, z4.s
+; CHECK-NEXT: uunpklo z4.d, z4.s
+; CHECK-NEXT: uunpkhi z6.d, z3.s
; CHECK-NEXT: uunpklo z3.d, z3.s
-; CHECK-NEXT: uunpkhi z24.d, z5.s
-; CHECK-NEXT: add x13, x10, x9
-; CHECK-NEXT: uunpklo z5.d, z5.s
-; CHECK-NEXT: uunpkhi z0.s, z0.h
-; CHECK-NEXT: uunpkhi z26.d, z2.s
+; CHECK-NEXT: uunpkhi z26.d, z7.s
+; CHECK-NEXT: uunpklo z7.d, z7.s
+; CHECK-NEXT: uunpkhi z27.d, z2.s
+; CHECK-NEXT: uunpkhi z28.d, z24.s
; CHECK-NEXT: uunpklo z2.d, z2.s
-; CHECK-NEXT: uunpkhi z29.d, z1.s
-; CHECK-NEXT: uunpklo z1.d, z1.s
+; CHECK-NEXT: uunpkhi z29.d, z25.s
+; CHECK-NEXT: uunpklo z25.d, z25.s
+; CHECK-NEXT: frintx z5.h, p0/m, z5.h
+; CHECK-NEXT: frintx z4.h, p0/m, z4.h
; CHECK-NEXT: frintx z6.h, p0/m, z6.h
-; CHECK-NEXT: frintx z7.h, p0/m, z7.h
-; CHECK-NEXT: uunpkhi z30.d, z27.s
; CHECK-NEXT: frintx z3.h, p0/m, z3.h
-; CHECK-NEXT: frintx z24.h, p0/m, z24.h
-; CHECK-NEXT: uunpkhi z4.d, z4.s
-; CHECK-NEXT: frintx z5.h, p0/m, z5.h
-; CHECK-NEXT: uunpklo z28.d, z0.s
-; CHECK-NEXT: uunpkhi z0.d, z0.s
; CHECK-NEXT: frintx z26.h, p0/m, z26.h
+; CHECK-NEXT: frintx z7.h, p0/m, z7.h
+; CHECK-NEXT: frintx z27.h, p0/m, z27.h
; CHECK-NEXT: frintx z2.h, p0/m, z2.h
+; CHECK-NEXT: frintx z28.h, p0/m, z28.h
; CHECK-NEXT: frintx z29.h, p0/m, z29.h
+; CHECK-NEXT: frintx z25.h, p0/m, z25.h
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
+; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.h
; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.h
-; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.h
-; CHECK-NEXT: frintx z1.h, p0/m, z1.h
; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.h
-; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.h
-; CHECK-NEXT: frintx z30.h, p0/m, z30.h
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
-; CHECK-NEXT: frintx z0.h, p0/m, z0.h
-; CHECK-NEXT: frintx z28.h, p0/m, z28.h
; CHECK-NEXT: fcvtzs z26.d, p0/m, z26.h
-; CHECK-NEXT: frintx z4.h, p0/m, z4.h
+; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.h
+; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.h
; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.h
-; CHECK-NEXT: str z6, [x12, #2, mul vl]
-; CHECK-NEXT: uunpklo z6.d, z27.s
+; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.h
; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.h
-; CHECK-NEXT: str z7, [x12, #1, mul vl]
+; CHECK-NEXT: fcvtzs z25.d, p0/m, z25.h
+; CHECK-NEXT: str z5, [x8, #15, mul vl]
+; CHECK-NEXT: uunpkhi z5.d, z1.s
+; CHECK-NEXT: uunpklo z1.d, z1.s
+; CHECK-NEXT: str z4, [x8, #14, mul vl]
+; CHECK-NEXT: uunpkhi z4.d, z0.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: str z3, [x8, #12, mul vl]
+; CHECK-NEXT: uunpklo z3.d, z24.s
+; CHECK-NEXT: str z6, [x8, #13, mul vl]
+; CHECK-NEXT: str z26, [x8, #11, mul vl]
+; CHECK-NEXT: frintx z5.h, p0/m, z5.h
+; CHECK-NEXT: frintx z1.h, p0/m, z1.h
+; CHECK-NEXT: str z7, [x8, #10, mul vl]
+; CHECK-NEXT: frintx z4.h, p0/m, z4.h
+; CHECK-NEXT: frintx z0.h, p0/m, z0.h
+; CHECK-NEXT: str z27, [x8, #9, mul vl]
+; CHECK-NEXT: frintx z3.h, p0/m, z3.h
+; CHECK-NEXT: str z2, [x8, #8, mul vl]
+; CHECK-NEXT: str z29, [x8, #7, mul vl]
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.h
-; CHECK-NEXT: fcvtzs z30.d, p0/m, z30.h
-; CHECK-NEXT: str z3, [x10, #4, mul vl]
-; CHECK-NEXT: uunpkhi z3.d, z25.s
-; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.h
-; CHECK-NEXT: str z24, [x13, #1, mul vl]
-; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.h
+; CHECK-NEXT: str z25, [x8, #6, mul vl]
; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.h
-; CHECK-NEXT: str z5, [x10, #2, mul vl]
-; CHECK-NEXT: uunpklo z5.d, z25.s
-; CHECK-NEXT: frintx z6.h, p0/m, z6.h
-; CHECK-NEXT: str z26, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x8, x11
-; CHECK-NEXT: add x11, x10, x9
-; CHECK-NEXT: str z2, [x8, #8, mul vl]
-; CHECK-NEXT: frintx z3.h, p0/m, z3.h
-; CHECK-NEXT: str z29, [x11, #1, mul vl]
-; CHECK-NEXT: str z1, [x10, #2, mul vl]
-; CHECK-NEXT: frintx z5.h, p0/m, z5.h
-; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.h
-; CHECK-NEXT: str z30, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x8, x9
+; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.h
+; CHECK-NEXT: str z28, [x8, #3, mul vl]
; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.h
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
-; CHECK-NEXT: str z6, [x8, #4, mul vl]
-; CHECK-NEXT: str z0, [x10, #1, mul vl]
-; CHECK-NEXT: str z28, [x8, #2, mul vl]
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
-; CHECK-NEXT: str z5, [x8]
-; CHECK-NEXT: add x8, x12, x9
+; CHECK-NEXT: str z5, [x8, #5, mul vl]
+; CHECK-NEXT: str z1, [x8, #4, mul vl]
+; CHECK-NEXT: str z3, [x8, #2, mul vl]
; CHECK-NEXT: str z4, [x8, #1, mul vl]
+; CHECK-NEXT: str z0, [x8]
; CHECK-NEXT: ret
%a = call <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv32f16(<vscale x 32 x half> %x)
ret <vscale x 32 x i64> %a
@@ -333,81 +323,71 @@ declare <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f32(<vscale x 16 x float>
define <vscale x 32 x i64> @llrint_v32i64_v32f32(<vscale x 32 x float> %x) {
; CHECK-LABEL: llrint_v32i64_v32f32:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpklo z24.d, z6.s
-; CHECK-NEXT: uunpklo z25.d, z5.s
-; CHECK-NEXT: rdvl x9, #8
-; CHECK-NEXT: uunpkhi z26.d, z4.s
-; CHECK-NEXT: uunpklo z4.d, z4.s
-; CHECK-NEXT: add x9, x8, x9
+; CHECK-NEXT: uunpkhi z24.d, z7.s
+; CHECK-NEXT: uunpkhi z25.d, z6.s
+; CHECK-NEXT: uunpklo z6.d, z6.s
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: uunpklo z28.d, z3.s
-; CHECK-NEXT: rdvl x10, #4
-; CHECK-NEXT: uunpkhi z29.d, z2.s
-; CHECK-NEXT: uunpklo z30.d, z1.s
-; CHECK-NEXT: rdvl x12, #2
+; CHECK-NEXT: uunpklo z7.d, z7.s
+; CHECK-NEXT: uunpkhi z26.d, z5.s
+; CHECK-NEXT: uunpklo z5.d, z5.s
+; CHECK-NEXT: uunpkhi z27.d, z4.s
+; CHECK-NEXT: uunpkhi z28.d, z1.s
+; CHECK-NEXT: uunpklo z4.d, z4.s
+; CHECK-NEXT: uunpkhi z29.d, z3.s
+; CHECK-NEXT: uunpklo z3.d, z3.s
; CHECK-NEXT: frintx z24.s, p0/m, z24.s
+; CHECK-NEXT: frintx z6.s, p0/m, z6.s
+; CHECK-NEXT: uunpklo z1.d, z1.s
+; CHECK-NEXT: frintx z7.s, p0/m, z7.s
; CHECK-NEXT: frintx z25.s, p0/m, z25.s
-; CHECK-NEXT: add x11, x8, x10
; CHECK-NEXT: frintx z26.s, p0/m, z26.s
+; CHECK-NEXT: frintx z5.s, p0/m, z5.s
+; CHECK-NEXT: frintx z27.s, p0/m, z27.s
+; CHECK-NEXT: frintx z28.s, p0/m, z28.s
; CHECK-NEXT: frintx z4.s, p0/m, z4.s
-; CHECK-NEXT: add x13, x8, x12
-; CHECK-NEXT: uunpklo z2.d, z2.s
-; CHECK-NEXT: uunpkhi z1.d, z1.s
-; CHECK-NEXT: uunpklo z31.d, z0.s
-; CHECK-NEXT: uunpkhi z0.d, z0.s
-; CHECK-NEXT: uunpkhi z27.d, z7.s
-; CHECK-NEXT: uunpkhi z6.d, z6.s
+; CHECK-NEXT: frintx z29.s, p0/m, z29.s
+; CHECK-NEXT: frintx z3.s, p0/m, z3.s
; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.s
+; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.s
+; CHECK-NEXT: frintx z1.s, p0/m, z1.s
+; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.s
; CHECK-NEXT: fcvtzs z25.d, p0/m, z25.s
-; CHECK-NEXT: uunpklo z7.d, z7.s
; CHECK-NEXT: fcvtzs z26.d, p0/m, z26.s
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.s
+; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.s
+; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.s
; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.s
-; CHECK-NEXT: uunpkhi z3.d, z3.s
-; CHECK-NEXT: frintx z28.s, p0/m, z28.s
-; CHECK-NEXT: frintx z29.s, p0/m, z29.s
+; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.s
+; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.s
+; CHECK-NEXT: str z24, [x8, #15, mul vl]
+; CHECK-NEXT: uunpkhi z24.d, z2.s
+; CHECK-NEXT: uunpklo z2.d, z2.s
+; CHECK-NEXT: str z6, [x8, #12, mul vl]
+; CHECK-NEXT: uunpkhi z6.d, z0.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: str z7, [x8, #14, mul vl]
+; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.s
+; CHECK-NEXT: str z25, [x8, #13, mul vl]
+; CHECK-NEXT: str z26, [x8, #11, mul vl]
+; CHECK-NEXT: frintx z24.s, p0/m, z24.s
; CHECK-NEXT: frintx z2.s, p0/m, z2.s
-; CHECK-NEXT: frintx z1.s, p0/m, z1.s
-; CHECK-NEXT: frintx z30.s, p0/m, z30.s
-; CHECK-NEXT: frintx z0.s, p0/m, z0.s
-; CHECK-NEXT: str z24, [x9, #4, mul vl]
-; CHECK-NEXT: frintx z31.s, p0/m, z31.s
-; CHECK-NEXT: frintx z7.s, p0/m, z7.s
-; CHECK-NEXT: str z25, [x9, #2, mul vl]
+; CHECK-NEXT: str z5, [x8, #10, mul vl]
; CHECK-NEXT: frintx z6.s, p0/m, z6.s
-; CHECK-NEXT: frintx z3.s, p0/m, z3.s
-; CHECK-NEXT: str z26, [x9, #1, mul vl]
-; CHECK-NEXT: frintx z27.s, p0/m, z27.s
-; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.s
+; CHECK-NEXT: frintx z0.s, p0/m, z0.s
+; CHECK-NEXT: str z27, [x8, #9, mul vl]
; CHECK-NEXT: str z4, [x8, #8, mul vl]
-; CHECK-NEXT: uunpkhi z4.d, z5.s
-; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.s
+; CHECK-NEXT: str z29, [x8, #7, mul vl]
+; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.s
; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.s
-; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.s
-; CHECK-NEXT: fcvtzs z30.d, p0/m, z30.s
-; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.s
-; CHECK-NEXT: fcvtzs z31.d, p0/m, z31.s
-; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.s
-; CHECK-NEXT: str z28, [x11, #2, mul vl]
+; CHECK-NEXT: str z3, [x8, #6, mul vl]
; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.s
-; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.s
-; CHECK-NEXT: str z29, [x11, #1, mul vl]
-; CHECK-NEXT: frintx z4.s, p0/m, z4.s
-; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.s
+; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.s
+; CHECK-NEXT: str z28, [x8, #3, mul vl]
+; CHECK-NEXT: str z1, [x8, #2, mul vl]
+; CHECK-NEXT: str z24, [x8, #5, mul vl]
; CHECK-NEXT: str z2, [x8, #4, mul vl]
-; CHECK-NEXT: str z1, [x13, #1, mul vl]
-; CHECK-NEXT: str z30, [x8, #2, mul vl]
-; CHECK-NEXT: str z0, [x8, #1, mul vl]
-; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.s
-; CHECK-NEXT: str z31, [x8]
-; CHECK-NEXT: add x8, x9, x10
-; CHECK-NEXT: add x9, x9, x12
-; CHECK-NEXT: str z7, [x8, #2, mul vl]
; CHECK-NEXT: str z6, [x8, #1, mul vl]
-; CHECK-NEXT: add x8, x8, x12
-; CHECK-NEXT: str z4, [x9, #1, mul vl]
-; CHECK-NEXT: add x9, x11, x12
-; CHECK-NEXT: str z3, [x9, #1, mul vl]
-; CHECK-NEXT: str z27, [x8, #1, mul vl]
+; CHECK-NEXT: str z0, [x8]
; CHECK-NEXT: ret
%a = call <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv32f32(<vscale x 32 x float> %x)
ret <vscale x 32 x i64> %a
@@ -499,96 +479,71 @@ declare <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f64(<vscale x 16 x double
define <vscale x 32 x i64> @llrint_v32f64(<vscale x 32 x double> %x) {
; CHECK-LABEL: llrint_v32f64:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x9, #1
-; CHECK-NEXT: ldr z0, [x0, #1, mul vl]
-; CHECK-NEXT: ldr z1, [x0]
-; CHECK-NEXT: add x10, x0, x9
+; CHECK-NEXT: ldr z0, [x0, #15, mul vl]
+; CHECK-NEXT: ldr z1, [x0, #14, mul vl]
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: add x11, x10, x9
-; CHECK-NEXT: ldr z2, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x11, x9
-; CHECK-NEXT: ldr z3, [x11, #1, mul vl]
-; CHECK-NEXT: frintx z1.d, p0/m, z1.d
-; CHECK-NEXT: ldr z4, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: ldr z2, [x0, #13, mul vl]
+; CHECK-NEXT: ldr z3, [x0, #12, mul vl]
+; CHECK-NEXT: ldr z4, [x0, #11, mul vl]
+; CHECK-NEXT: ldr z5, [x0, #10, mul vl]
+; CHECK-NEXT: ldr z6, [x0, #9, mul vl]
+; CHECK-NEXT: ldr z7, [x0, #8, mul vl]
; CHECK-NEXT: frintx z0.d, p0/m, z0.d
-; CHECK-NEXT: add x11, x10, x9
-; CHECK-NEXT: ldr z5, [x10, #1, mul vl]
+; CHECK-NEXT: frintx z1.d, p0/m, z1.d
+; CHECK-NEXT: ldr z24, [x0, #7, mul vl]
+; CHECK-NEXT: ldr z25, [x0, #6, mul vl]
; CHECK-NEXT: frintx z2.d, p0/m, z2.d
-; CHECK-NEXT: add x10, x11, x9
-; CHECK-NEXT: ldr z6, [x11, #1, mul vl]
; CHECK-NEXT: frintx z3.d, p0/m, z3.d
-; CHECK-NEXT: ldr z7, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: ldr z26, [x0, #5, mul vl]
+; CHECK-NEXT: ldr z27, [x0, #4, mul vl]
; CHECK-NEXT: frintx z4.d, p0/m, z4.d
-; CHECK-NEXT: ldr z24, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: ldr z28, [x0, #3, mul vl]
+; CHECK-NEXT: ldr z29, [x0, #2, mul vl]
; CHECK-NEXT: frintx z5.d, p0/m, z5.d
-; CHECK-NEXT: add x11, x10, x9
-; CHECK-NEXT: ldr z25, [x10, #1, mul vl]
+; CHECK-NEXT: ldr z30, [x0, #1, mul vl]
+; CHECK-NEXT: ldr z31, [x0]
; CHECK-NEXT: frintx z6.d, p0/m, z6.d
-; CHECK-NEXT: add x10, x11, x9
-; CHECK-NEXT: ldr z26, [x11, #1, mul vl]
; CHECK-NEXT: frintx z7.d, p0/m, z7.d
-; CHECK-NEXT: ldr z27, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x10, x9
-; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d
-; CHECK-NEXT: ldr z28, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x10, x9
-; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d
-; CHECK-NEXT: add x11, x10, x9
; CHECK-NEXT: frintx z24.d, p0/m, z24.d
-; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.d
-; CHECK-NEXT: add x9, x11, x9
-; CHECK-NEXT: ldr z29, [x11, #1, mul vl]
-; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d
-; CHECK-NEXT: ldr z31, [x10, #1, mul vl]
-; CHECK-NEXT: ldr z30, [x9, #1, mul vl]
; CHECK-NEXT: frintx z25.d, p0/m, z25.d
-; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.d
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.d
; CHECK-NEXT: frintx z26.d, p0/m, z26.d
; CHECK-NEXT: frintx z27.d, p0/m, z27.d
; CHECK-NEXT: frintx z28.d, p0/m, z28.d
-; CHECK-NEXT: rdvl x9, #2
-; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d
; CHECK-NEXT: frintx z29.d, p0/m, z29.d
-; CHECK-NEXT: add x10, x8, x9
-; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.d
-; CHECK-NEXT: frintx z31.d, p0/m, z31.d
-; CHECK-NEXT: str z0, [x8, #1, mul vl]
-; CHECK-NEXT: str z1, [x8]
; CHECK-NEXT: frintx z30.d, p0/m, z30.d
+; CHECK-NEXT: frintx z31.d, p0/m, z31.d
+; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d
+; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d
+; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d
+; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.d
+; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.d
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.d
+; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d
+; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.d
; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.d
-; CHECK-NEXT: str z2, [x8, #2, mul vl]
; CHECK-NEXT: fcvtzs z25.d, p0/m, z25.d
; CHECK-NEXT: fcvtzs z26.d, p0/m, z26.d
-; CHECK-NEXT: str z3, [x10, #1, mul vl]
-; CHECK-NEXT: rdvl x10, #4
; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.d
-; CHECK-NEXT: add x11, x8, x10
-; CHECK-NEXT: str z4, [x8, #4, mul vl]
; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.d
-; CHECK-NEXT: str z5, [x11, #1, mul vl]
-; CHECK-NEXT: fcvtzs z31.d, p0/m, z31.d
; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.d
-; CHECK-NEXT: str z6, [x11, #2, mul vl]
-; CHECK-NEXT: add x11, x11, x9
; CHECK-NEXT: fcvtzs z30.d, p0/m, z30.d
-; CHECK-NEXT: str z7, [x11, #1, mul vl]
-; CHECK-NEXT: rdvl x11, #8
-; CHECK-NEXT: str z24, [x8, #8, mul vl]
-; CHECK-NEXT: add x8, x8, x11
-; CHECK-NEXT: add x11, x8, x9
-; CHECK-NEXT: str z25, [x8, #1, mul vl]
-; CHECK-NEXT: str z26, [x8, #2, mul vl]
-; CHECK-NEXT: str z27, [x11, #1, mul vl]
-; CHECK-NEXT: str z28, [x8, #4, mul vl]
-; CHECK-NEXT: add x8, x8, x10
-; CHECK-NEXT: str z31, [x8, #1, mul vl]
+; CHECK-NEXT: fcvtzs z31.d, p0/m, z31.d
+; CHECK-NEXT: str z0, [x8, #15, mul vl]
+; CHECK-NEXT: str z1, [x8, #14, mul vl]
+; CHECK-NEXT: str z2, [x8, #13, mul vl]
+; CHECK-NEXT: str z3, [x8, #12, mul vl]
+; CHECK-NEXT: str z4, [x8, #11, mul vl]
+; CHECK-NEXT: str z5, [x8, #10, mul vl]
+; CHECK-NEXT: str z6, [x8, #9, mul vl]
+; CHECK-NEXT: str z7, [x8, #8, mul vl]
+; CHECK-NEXT: str z24, [x8, #7, mul vl]
+; CHECK-NEXT: str z25, [x8, #6, mul vl]
+; CHECK-NEXT: str z26, [x8, #5, mul vl]
+; CHECK-NEXT: str z27, [x8, #4, mul vl]
+; CHECK-NEXT: str z28, [x8, #3, mul vl]
; CHECK-NEXT: str z29, [x8, #2, mul vl]
-; CHECK-NEXT: add x8, x8, x9
; CHECK-NEXT: str z30, [x8, #1, mul vl]
+; CHECK-NEXT: str z31, [x8]
; CHECK-NEXT: ret
%a = call <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv16f64(<vscale x 32 x double> %x)
ret <vscale x 32 x i64> %a
diff --git a/llvm/test/CodeGen/AArch64/sve-load-store-legalisation.ll b/llvm/test/CodeGen/AArch64/sve-load-store-legalisation.ll
index 4c3a15b453840..4f9dc9beb9e08 100644
--- a/llvm/test/CodeGen/AArch64/sve-load-store-legalisation.ll
+++ b/llvm/test/CodeGen/AArch64/sve-load-store-legalisation.ll
@@ -213,12 +213,9 @@ define void @sve_load_store_nxv13i8(ptr %a, ptr %b) {
define void @sve_load_store_nxv14i8(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv14i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: cnth x8
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: add x9, x0, x8
; CHECK-NEXT: ptrue p1.s
-; CHECK-NEXT: add x8, x1, x8
-; CHECK-NEXT: ld1b { z0.d }, p0/z, [x9, #2, mul vl]
+; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, #6, mul vl]
; CHECK-NEXT: ptrue p2.h
; CHECK-NEXT: ld1b { z1.s }, p1/z, [x0, #2, mul vl]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
@@ -232,7 +229,7 @@ define void @sve_load_store_nxv14i8(ptr %a, ptr %b) {
; CHECK-NEXT: st1b { z0.h }, p2, [x1]
; CHECK-NEXT: uunpklo z2.d, z2.s
; CHECK-NEXT: st1b { z1.s }, p1, [x1, #2, mul vl]
-; CHECK-NEXT: st1b { z2.d }, p0, [x8, #2, mul vl]
+; CHECK-NEXT: st1b { z2.d }, p0, [x1, #6, mul vl]
; CHECK-NEXT: ret
%c = load <vscale x 14 x i8>, ptr %a
store <vscale x 14 x i8> %c, ptr %b
@@ -366,13 +363,11 @@ define void @sve_load_store_nxv22i8(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv22i8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: cntw x8, all, mul #5
; CHECK-NEXT: ldr z2, [x0]
; CHECK-NEXT: ptrue p1.d
-; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: add x8, x1, x8
; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, #4, mul vl]
-; CHECK-NEXT: ld1b { z1.d }, p1/z, [x9, #2, mul vl]
+; CHECK-NEXT: ld1b { z1.d }, p1/z, [x0, x8]
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b
; CHECK-NEXT: uunpklo z0.h, z0.b
@@ -384,8 +379,8 @@ define void @sve_load_store_nxv22i8(ptr %a, ptr %b) {
; CHECK-NEXT: uunpkhi z1.s, z0.h
; CHECK-NEXT: uunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z1.d, z1.s
+; CHECK-NEXT: st1b { z1.d }, p1, [x1, x8]
; CHECK-NEXT: st1b { z0.s }, p0, [x1, #4, mul vl]
-; CHECK-NEXT: st1b { z1.d }, p1, [x8, #2, mul vl]
; CHECK-NEXT: str z2, [x1]
; CHECK-NEXT: ret
%c = load <vscale x 22 x i8>, ptr %a
@@ -450,13 +445,11 @@ define void @sve_load_store_nxv25i8(ptr %a, ptr %b) {
define void @sve_load_store_nxv26i8(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv26i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: cnth x8, all, mul #3
; CHECK-NEXT: ldr z2, [x0]
-; CHECK-NEXT: add x9, x0, x8
; CHECK-NEXT: ptrue p1.h
-; CHECK-NEXT: add x8, x1, x8
-; CHECK-NEXT: ld1b { z0.d }, p0/z, [x9, #4, mul vl]
+; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, x8]
; CHECK-NEXT: ld1b { z1.h }, p1/z, [x0, #2, mul vl]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
@@ -464,9 +457,9 @@ define void @sve_load_store_nxv26i8(ptr %a, ptr %b) {
; CHECK-NEXT: uunpkhi z1.h, z0.b
; CHECK-NEXT: uunpklo z0.h, z0.b
; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: st1b { z0.h }, p1, [x1, #2, mul vl]
; CHECK-NEXT: uunpklo z1.d, z1.s
-; CHECK-NEXT: st1b { z1.d }, p0, [x8, #4, mul vl]
+; CHECK-NEXT: st1b { z1.d }, p0, [x1, x8]
+; CHECK-NEXT: st1b { z0.h }, p1, [x1, #2, mul vl]
; CHECK-NEXT: str z2, [x1]
; CHECK-NEXT: ret
%c = load <vscale x 26 x i8>, ptr %a
@@ -496,22 +489,19 @@ define void @sve_load_store_nxv27i8(ptr %a, ptr %b) {
define void @sve_load_store_nxv28i8(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv28i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: ptrue p0.s
; CHECK-NEXT: ldr z2, [x0]
-; CHECK-NEXT: add x9, x0, x8
; CHECK-NEXT: ptrue p1.h
-; CHECK-NEXT: add x8, x1, x8
-; CHECK-NEXT: ld1b { z0.s }, p0/z, [x9, #2, mul vl]
+; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, #6, mul vl]
; CHECK-NEXT: ld1b { z1.h }, p1/z, [x0, #2, mul vl]
+; CHECK-NEXT: str z2, [x1]
; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
; CHECK-NEXT: uzp1 z0.b, z1.b, z0.b
; CHECK-NEXT: uunpkhi z1.h, z0.b
; CHECK-NEXT: uunpklo z0.h, z0.b
; CHECK-NEXT: uunpklo z1.s, z1.h
; CHECK-NEXT: st1b { z0.h }, p1, [x1, #2, mul vl]
-; CHECK-NEXT: st1b { z1.s }, p0, [x8, #2, mul vl]
-; CHECK-NEXT: str z2, [x1]
+; CHECK-NEXT: st1b { z1.s }, p0, [x1, #6, mul vl]
; CHECK-NEXT: ret
%c = load <vscale x 28 x i8>, ptr %a
store <vscale x 28 x i8> %c, ptr %b
@@ -540,18 +530,13 @@ define void @sve_load_store_nxv29i8(ptr %a, ptr %b) {
define void @sve_load_store_nxv30i8(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv30i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: cnth x9
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: add x10, x0, x8
-; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: cntw x8, all, mul #7
; CHECK-NEXT: ldr z3, [x0]
-; CHECK-NEXT: add x11, x10, x9
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, x8]
; CHECK-NEXT: ptrue p2.h
-; CHECK-NEXT: add x8, x1, x8
-; CHECK-NEXT: ld1b { z0.d }, p0/z, [x11, #2, mul vl]
-; CHECK-NEXT: ld1b { z1.s }, p1/z, [x10, #2, mul vl]
-; CHECK-NEXT: add x9, x8, x9
+; CHECK-NEXT: ld1b { z1.s }, p1/z, [x0, #6, mul vl]
; CHECK-NEXT: ld1b { z2.h }, p2/z, [x0, #2, mul vl]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
@@ -560,10 +545,10 @@ define void @sve_load_store_nxv30i8(ptr %a, ptr %b) {
; CHECK-NEXT: uunpklo z0.h, z0.b
; CHECK-NEXT: uunpkhi z2.s, z1.h
; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: st1b { z0.h }, p2, [x1, #2, mul vl]
; CHECK-NEXT: uunpklo z2.d, z2.s
-; CHECK-NEXT: st1b { z1.s }, p1, [x8, #2, mul vl]
-; CHECK-NEXT: st1b { z2.d }, p0, [x9, #2, mul vl]
+; CHECK-NEXT: st1b { z2.d }, p0, [x1, x8]
+; CHECK-NEXT: st1b { z0.h }, p2, [x1, #2, mul vl]
+; CHECK-NEXT: st1b { z1.s }, p1, [x1, #6, mul vl]
; CHECK-NEXT: str z3, [x1]
; CHECK-NEXT: ret
%c = load <vscale x 30 x i8>, ptr %a
@@ -817,22 +802,19 @@ define void @sve_load_store_nxv13i16(ptr %a, ptr %b) {
define void @sve_load_store_nxv14i16(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv14i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: ldr z2, [x0]
-; CHECK-NEXT: add x9, x0, x8
; CHECK-NEXT: ptrue p1.s
-; CHECK-NEXT: add x8, x1, x8
-; CHECK-NEXT: ld1h { z0.d }, p0/z, [x9, #2, mul vl]
+; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, #6, mul vl]
; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0, #2, mul vl]
+; CHECK-NEXT: str z2, [x1]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
; CHECK-NEXT: uunpkhi z1.s, z0.h
; CHECK-NEXT: uunpklo z0.s, z0.h
; CHECK-NEXT: uunpklo z1.d, z1.s
; CHECK-NEXT: st1h { z0.s }, p1, [x1, #2, mul vl]
-; CHECK-NEXT: st1h { z1.d }, p0, [x8, #2, mul vl]
-; CHECK-NEXT: str z2, [x1]
+; CHECK-NEXT: st1h { z1.d }, p0, [x1, #6, mul vl]
; CHECK-NEXT: ret
%c = load <vscale x 14 x i16>, ptr %a
store <vscale x 14 x i16> %c, ptr %b
@@ -1255,17 +1237,14 @@ define void @sve_load_store_nxv13f16(ptr %a, ptr %b) {
define void @sve_load_store_nxv14f16(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv14f16:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: ldr z2, [x0]
; CHECK-NEXT: ptrue p1.s
-; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: add x8, x1, x8
-; CHECK-NEXT: ld1h { z0.d }, p0/z, [x9, #2, mul vl]
+; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, #6, mul vl]
; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0, #2, mul vl]
-; CHECK-NEXT: st1h { z0.d }, p0, [x8, #2, mul vl]
-; CHECK-NEXT: st1h { z1.s }, p1, [x1, #2, mul vl]
; CHECK-NEXT: str z2, [x1]
+; CHECK-NEXT: st1h { z0.d }, p0, [x1, #6, mul vl]
+; CHECK-NEXT: st1h { z1.s }, p1, [x1, #2, mul vl]
; CHECK-NEXT: ret
%c = load <vscale x 14 x half>, ptr %a
store <vscale x 14 x half> %c, ptr %b
@@ -1686,17 +1665,14 @@ define void @sve_load_store_nxv13bf16(ptr %a, ptr %b) {
define void @sve_load_store_nxv14bf16(ptr %a, ptr %b) {
; CHECK-LABEL: sve_load_store_nxv14bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: ldr z2, [x0]
; CHECK-NEXT: ptrue p1.s
-; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: add x8, x1, x8
-; CHECK-NEXT: ld1h { z0.d }, p0/z, [x9, #2, mul vl]
+; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, #6, mul vl]
; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0, #2, mul vl]
-; CHECK-NEXT: st1h { z0.d }, p0, [x8, #2, mul vl]
-; CHECK-NEXT: st1h { z1.s }, p1, [x1, #2, mul vl]
; CHECK-NEXT: str z2, [x1]
+; CHECK-NEXT: st1h { z0.d }, p0, [x1, #6, mul vl]
+; CHECK-NEXT: st1h { z1.s }, p1, [x1, #2, mul vl]
; CHECK-NEXT: ret
%c = load <vscale x 14 x bfloat>, ptr %a
store <vscale x 14 x bfloat> %c, ptr %b
@@ -1987,23 +1963,20 @@ define <vscale x 14 x i16> @sve_sextload_nxv14i8(ptr %a, ptr %b) {
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: cntd x8, all, mul #7
; CHECK-NEXT: cnth x9
-; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: ptrue p1.s
; CHECK-NEXT: whilelo p0.h, x9, x8
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: whilelo p0.h, xzr, x8
-; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: ld1sb { z2.h }, p0/z, [x0]
-; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: uunpkhi z1.s, z0.h
; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: uunpklo z1.d, z1.s
-; CHECK-NEXT: st1h { z1.d }, p1, [x8, #2, mul vl]
-; CHECK-NEXT: st1h { z0.s }, p0, [sp, #2, mul vl]
; CHECK-NEXT: str z2, [sp]
-; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-NEXT: uunpklo z1.d, z1.s
+; CHECK-NEXT: st1h { z0.s }, p1, [sp, #2, mul vl]
; CHECK-NEXT: ldr z0, [sp]
+; CHECK-NEXT: st1h { z1.d }, p0, [sp, #6, mul vl]
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
; CHECK-NEXT: addvl sp, sp, #2
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
@@ -2512,23 +2485,20 @@ define <vscale x 14 x i16> @sve_zextload_nxv14i8(ptr %a, ptr %b) {
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: cntd x8, all, mul #7
; CHECK-NEXT: cnth x9
-; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: ptrue p1.s
; CHECK-NEXT: whilelo p0.h, x9, x8
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: whilelo p0.h, xzr, x8
-; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: ld1sb { z2.h }, p0/z, [x0]
-; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: uunpkhi z1.s, z0.h
; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: uunpklo z1.d, z1.s
-; CHECK-NEXT: st1h { z1.d }, p1, [x8, #2, mul vl]
-; CHECK-NEXT: st1h { z0.s }, p0, [sp, #2, mul vl]
; CHECK-NEXT: str z2, [sp]
-; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-NEXT: uunpklo z1.d, z1.s
+; CHECK-NEXT: st1h { z0.s }, p1, [sp, #2, mul vl]
; CHECK-NEXT: ldr z0, [sp]
+; CHECK-NEXT: st1h { z1.d }, p0, [sp, #6, mul vl]
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
; CHECK-NEXT: addvl sp, sp, #2
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-lrint.ll b/llvm/test/CodeGen/AArch64/sve-lrint.ll
index 7efff72a15662..889d4a1c8f73a 100644
--- a/llvm/test/CodeGen/AArch64/sve-lrint.ll
+++ b/llvm/test/CodeGen/AArch64/sve-lrint.ll
@@ -121,87 +121,77 @@ define <vscale x 32 x i64> @lrint_v32f16(<vscale x 32 x half> %x) {
; CHECK: // %bb.0:
; CHECK-NEXT: uunpkhi z4.s, z3.h
; CHECK-NEXT: uunpklo z3.s, z3.h
-; CHECK-NEXT: rdvl x10, #8
-; CHECK-NEXT: uunpkhi z5.s, z2.h
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: rdvl x11, #4
+; CHECK-NEXT: uunpkhi z7.s, z2.h
; CHECK-NEXT: uunpklo z2.s, z2.h
-; CHECK-NEXT: uunpklo z27.s, z1.h
-; CHECK-NEXT: rdvl x9, #2
-; CHECK-NEXT: uunpkhi z1.s, z1.h
-; CHECK-NEXT: uunpklo z25.s, z0.h
-; CHECK-NEXT: add x10, x8, x10
-; CHECK-NEXT: uunpklo z6.d, z4.s
-; CHECK-NEXT: uunpkhi z7.d, z3.s
-; CHECK-NEXT: add x12, x10, x11
+; CHECK-NEXT: uunpkhi z24.s, z0.h
+; CHECK-NEXT: uunpkhi z25.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpkhi z5.d, z4.s
+; CHECK-NEXT: uunpklo z4.d, z4.s
+; CHECK-NEXT: uunpkhi z6.d, z3.s
; CHECK-NEXT: uunpklo z3.d, z3.s
-; CHECK-NEXT: uunpkhi z24.d, z5.s
-; CHECK-NEXT: add x13, x10, x9
-; CHECK-NEXT: uunpklo z5.d, z5.s
-; CHECK-NEXT: uunpkhi z0.s, z0.h
-; CHECK-NEXT: uunpkhi z26.d, z2.s
+; CHECK-NEXT: uunpkhi z26.d, z7.s
+; CHECK-NEXT: uunpklo z7.d, z7.s
+; CHECK-NEXT: uunpkhi z27.d, z2.s
+; CHECK-NEXT: uunpkhi z28.d, z24.s
; CHECK-NEXT: uunpklo z2.d, z2.s
-; CHECK-NEXT: uunpkhi z29.d, z1.s
-; CHECK-NEXT: uunpklo z1.d, z1.s
+; CHECK-NEXT: uunpkhi z29.d, z25.s
+; CHECK-NEXT: uunpklo z25.d, z25.s
+; CHECK-NEXT: frintx z5.h, p0/m, z5.h
+; CHECK-NEXT: frintx z4.h, p0/m, z4.h
; CHECK-NEXT: frintx z6.h, p0/m, z6.h
-; CHECK-NEXT: frintx z7.h, p0/m, z7.h
-; CHECK-NEXT: uunpkhi z30.d, z27.s
; CHECK-NEXT: frintx z3.h, p0/m, z3.h
-; CHECK-NEXT: frintx z24.h, p0/m, z24.h
-; CHECK-NEXT: uunpkhi z4.d, z4.s
-; CHECK-NEXT: frintx z5.h, p0/m, z5.h
-; CHECK-NEXT: uunpklo z28.d, z0.s
-; CHECK-NEXT: uunpkhi z0.d, z0.s
; CHECK-NEXT: frintx z26.h, p0/m, z26.h
+; CHECK-NEXT: frintx z7.h, p0/m, z7.h
+; CHECK-NEXT: frintx z27.h, p0/m, z27.h
; CHECK-NEXT: frintx z2.h, p0/m, z2.h
+; CHECK-NEXT: frintx z28.h, p0/m, z28.h
; CHECK-NEXT: frintx z29.h, p0/m, z29.h
+; CHECK-NEXT: frintx z25.h, p0/m, z25.h
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
+; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.h
; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.h
-; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.h
-; CHECK-NEXT: frintx z1.h, p0/m, z1.h
; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.h
-; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.h
-; CHECK-NEXT: frintx z30.h, p0/m, z30.h
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
-; CHECK-NEXT: frintx z0.h, p0/m, z0.h
-; CHECK-NEXT: frintx z28.h, p0/m, z28.h
; CHECK-NEXT: fcvtzs z26.d, p0/m, z26.h
-; CHECK-NEXT: frintx z4.h, p0/m, z4.h
+; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.h
+; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.h
; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.h
-; CHECK-NEXT: str z6, [x12, #2, mul vl]
-; CHECK-NEXT: uunpklo z6.d, z27.s
+; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.h
; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.h
-; CHECK-NEXT: str z7, [x12, #1, mul vl]
+; CHECK-NEXT: fcvtzs z25.d, p0/m, z25.h
+; CHECK-NEXT: str z5, [x8, #15, mul vl]
+; CHECK-NEXT: uunpkhi z5.d, z1.s
+; CHECK-NEXT: uunpklo z1.d, z1.s
+; CHECK-NEXT: str z4, [x8, #14, mul vl]
+; CHECK-NEXT: uunpkhi z4.d, z0.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: str z3, [x8, #12, mul vl]
+; CHECK-NEXT: uunpklo z3.d, z24.s
+; CHECK-NEXT: str z6, [x8, #13, mul vl]
+; CHECK-NEXT: str z26, [x8, #11, mul vl]
+; CHECK-NEXT: frintx z5.h, p0/m, z5.h
+; CHECK-NEXT: frintx z1.h, p0/m, z1.h
+; CHECK-NEXT: str z7, [x8, #10, mul vl]
+; CHECK-NEXT: frintx z4.h, p0/m, z4.h
+; CHECK-NEXT: frintx z0.h, p0/m, z0.h
+; CHECK-NEXT: str z27, [x8, #9, mul vl]
+; CHECK-NEXT: frintx z3.h, p0/m, z3.h
+; CHECK-NEXT: str z2, [x8, #8, mul vl]
+; CHECK-NEXT: str z29, [x8, #7, mul vl]
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.h
-; CHECK-NEXT: fcvtzs z30.d, p0/m, z30.h
-; CHECK-NEXT: str z3, [x10, #4, mul vl]
-; CHECK-NEXT: uunpkhi z3.d, z25.s
-; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.h
-; CHECK-NEXT: str z24, [x13, #1, mul vl]
-; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.h
+; CHECK-NEXT: str z25, [x8, #6, mul vl]
; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.h
-; CHECK-NEXT: str z5, [x10, #2, mul vl]
-; CHECK-NEXT: uunpklo z5.d, z25.s
-; CHECK-NEXT: frintx z6.h, p0/m, z6.h
-; CHECK-NEXT: str z26, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x8, x11
-; CHECK-NEXT: add x11, x10, x9
-; CHECK-NEXT: str z2, [x8, #8, mul vl]
-; CHECK-NEXT: frintx z3.h, p0/m, z3.h
-; CHECK-NEXT: str z29, [x11, #1, mul vl]
-; CHECK-NEXT: str z1, [x10, #2, mul vl]
-; CHECK-NEXT: frintx z5.h, p0/m, z5.h
-; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.h
-; CHECK-NEXT: str z30, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x8, x9
+; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.h
+; CHECK-NEXT: str z28, [x8, #3, mul vl]
; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.h
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.h
-; CHECK-NEXT: str z6, [x8, #4, mul vl]
-; CHECK-NEXT: str z0, [x10, #1, mul vl]
-; CHECK-NEXT: str z28, [x8, #2, mul vl]
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
-; CHECK-NEXT: str z5, [x8]
-; CHECK-NEXT: add x8, x12, x9
+; CHECK-NEXT: str z5, [x8, #5, mul vl]
+; CHECK-NEXT: str z1, [x8, #4, mul vl]
+; CHECK-NEXT: str z3, [x8, #2, mul vl]
; CHECK-NEXT: str z4, [x8, #1, mul vl]
+; CHECK-NEXT: str z0, [x8]
; CHECK-NEXT: ret
%a = call <vscale x 32 x i64> @llvm.lrint.nxv32i64.nxv32f16(<vscale x 32 x half> %x)
ret <vscale x 32 x i64> %a
@@ -322,81 +312,71 @@ define <vscale x 16 x i64> @lrint_v16f32(<vscale x 16 x float> %x) {
define <vscale x 32 x i64> @lrint_v32f32(<vscale x 32 x float> %x) {
; CHECK-LABEL: lrint_v32f32:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpklo z24.d, z6.s
-; CHECK-NEXT: uunpklo z25.d, z5.s
-; CHECK-NEXT: rdvl x9, #8
-; CHECK-NEXT: uunpkhi z26.d, z4.s
-; CHECK-NEXT: uunpklo z4.d, z4.s
-; CHECK-NEXT: add x9, x8, x9
+; CHECK-NEXT: uunpkhi z24.d, z7.s
+; CHECK-NEXT: uunpkhi z25.d, z6.s
+; CHECK-NEXT: uunpklo z6.d, z6.s
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: uunpklo z28.d, z3.s
-; CHECK-NEXT: rdvl x10, #4
-; CHECK-NEXT: uunpkhi z29.d, z2.s
-; CHECK-NEXT: uunpklo z30.d, z1.s
-; CHECK-NEXT: rdvl x12, #2
+; CHECK-NEXT: uunpklo z7.d, z7.s
+; CHECK-NEXT: uunpkhi z26.d, z5.s
+; CHECK-NEXT: uunpklo z5.d, z5.s
+; CHECK-NEXT: uunpkhi z27.d, z4.s
+; CHECK-NEXT: uunpkhi z28.d, z1.s
+; CHECK-NEXT: uunpklo z4.d, z4.s
+; CHECK-NEXT: uunpkhi z29.d, z3.s
+; CHECK-NEXT: uunpklo z3.d, z3.s
; CHECK-NEXT: frintx z24.s, p0/m, z24.s
+; CHECK-NEXT: frintx z6.s, p0/m, z6.s
+; CHECK-NEXT: uunpklo z1.d, z1.s
+; CHECK-NEXT: frintx z7.s, p0/m, z7.s
; CHECK-NEXT: frintx z25.s, p0/m, z25.s
-; CHECK-NEXT: add x11, x8, x10
; CHECK-NEXT: frintx z26.s, p0/m, z26.s
+; CHECK-NEXT: frintx z5.s, p0/m, z5.s
+; CHECK-NEXT: frintx z27.s, p0/m, z27.s
+; CHECK-NEXT: frintx z28.s, p0/m, z28.s
; CHECK-NEXT: frintx z4.s, p0/m, z4.s
-; CHECK-NEXT: add x13, x8, x12
-; CHECK-NEXT: uunpklo z2.d, z2.s
-; CHECK-NEXT: uunpkhi z1.d, z1.s
-; CHECK-NEXT: uunpklo z31.d, z0.s
-; CHECK-NEXT: uunpkhi z0.d, z0.s
-; CHECK-NEXT: uunpkhi z27.d, z7.s
-; CHECK-NEXT: uunpkhi z6.d, z6.s
+; CHECK-NEXT: frintx z29.s, p0/m, z29.s
+; CHECK-NEXT: frintx z3.s, p0/m, z3.s
; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.s
+; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.s
+; CHECK-NEXT: frintx z1.s, p0/m, z1.s
+; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.s
; CHECK-NEXT: fcvtzs z25.d, p0/m, z25.s
-; CHECK-NEXT: uunpklo z7.d, z7.s
; CHECK-NEXT: fcvtzs z26.d, p0/m, z26.s
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.s
+; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.s
+; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.s
; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.s
-; CHECK-NEXT: uunpkhi z3.d, z3.s
-; CHECK-NEXT: frintx z28.s, p0/m, z28.s
-; CHECK-NEXT: frintx z29.s, p0/m, z29.s
+; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.s
+; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.s
+; CHECK-NEXT: str z24, [x8, #15, mul vl]
+; CHECK-NEXT: uunpkhi z24.d, z2.s
+; CHECK-NEXT: uunpklo z2.d, z2.s
+; CHECK-NEXT: str z6, [x8, #12, mul vl]
+; CHECK-NEXT: uunpkhi z6.d, z0.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: str z7, [x8, #14, mul vl]
+; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.s
+; CHECK-NEXT: str z25, [x8, #13, mul vl]
+; CHECK-NEXT: str z26, [x8, #11, mul vl]
+; CHECK-NEXT: frintx z24.s, p0/m, z24.s
; CHECK-NEXT: frintx z2.s, p0/m, z2.s
-; CHECK-NEXT: frintx z1.s, p0/m, z1.s
-; CHECK-NEXT: frintx z30.s, p0/m, z30.s
-; CHECK-NEXT: frintx z0.s, p0/m, z0.s
-; CHECK-NEXT: str z24, [x9, #4, mul vl]
-; CHECK-NEXT: frintx z31.s, p0/m, z31.s
-; CHECK-NEXT: frintx z7.s, p0/m, z7.s
-; CHECK-NEXT: str z25, [x9, #2, mul vl]
+; CHECK-NEXT: str z5, [x8, #10, mul vl]
; CHECK-NEXT: frintx z6.s, p0/m, z6.s
-; CHECK-NEXT: frintx z3.s, p0/m, z3.s
-; CHECK-NEXT: str z26, [x9, #1, mul vl]
-; CHECK-NEXT: frintx z27.s, p0/m, z27.s
-; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.s
+; CHECK-NEXT: frintx z0.s, p0/m, z0.s
+; CHECK-NEXT: str z27, [x8, #9, mul vl]
; CHECK-NEXT: str z4, [x8, #8, mul vl]
-; CHECK-NEXT: uunpkhi z4.d, z5.s
-; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.s
+; CHECK-NEXT: str z29, [x8, #7, mul vl]
+; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.s
; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.s
-; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.s
-; CHECK-NEXT: fcvtzs z30.d, p0/m, z30.s
-; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.s
-; CHECK-NEXT: fcvtzs z31.d, p0/m, z31.s
-; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.s
-; CHECK-NEXT: str z28, [x11, #2, mul vl]
+; CHECK-NEXT: str z3, [x8, #6, mul vl]
; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.s
-; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.s
-; CHECK-NEXT: str z29, [x11, #1, mul vl]
-; CHECK-NEXT: frintx z4.s, p0/m, z4.s
-; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.s
+; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.s
+; CHECK-NEXT: str z28, [x8, #3, mul vl]
+; CHECK-NEXT: str z1, [x8, #2, mul vl]
+; CHECK-NEXT: str z24, [x8, #5, mul vl]
; CHECK-NEXT: str z2, [x8, #4, mul vl]
-; CHECK-NEXT: str z1, [x13, #1, mul vl]
-; CHECK-NEXT: str z30, [x8, #2, mul vl]
-; CHECK-NEXT: str z0, [x8, #1, mul vl]
-; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.s
-; CHECK-NEXT: str z31, [x8]
-; CHECK-NEXT: add x8, x9, x10
-; CHECK-NEXT: add x9, x9, x12
-; CHECK-NEXT: str z7, [x8, #2, mul vl]
; CHECK-NEXT: str z6, [x8, #1, mul vl]
-; CHECK-NEXT: add x8, x8, x12
-; CHECK-NEXT: str z4, [x9, #1, mul vl]
-; CHECK-NEXT: add x9, x11, x12
-; CHECK-NEXT: str z3, [x9, #1, mul vl]
-; CHECK-NEXT: str z27, [x8, #1, mul vl]
+; CHECK-NEXT: str z0, [x8]
; CHECK-NEXT: ret
%a = call <vscale x 32 x i64> @llvm.lrint.nxv32i64.nxv32f32(<vscale x 32 x float> %x)
ret <vscale x 32 x i64> %a
@@ -482,96 +462,71 @@ define <vscale x 16 x i64> @lrint_v16f64(<vscale x 16 x double> %x) {
define <vscale x 32 x i64> @lrint_v32f64(<vscale x 32 x double> %x) {
; CHECK-LABEL: lrint_v32f64:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x9, #1
-; CHECK-NEXT: ldr z0, [x0, #1, mul vl]
-; CHECK-NEXT: ldr z1, [x0]
-; CHECK-NEXT: add x10, x0, x9
+; CHECK-NEXT: ldr z0, [x0, #15, mul vl]
+; CHECK-NEXT: ldr z1, [x0, #14, mul vl]
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: add x11, x10, x9
-; CHECK-NEXT: ldr z2, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x11, x9
-; CHECK-NEXT: ldr z3, [x11, #1, mul vl]
-; CHECK-NEXT: frintx z1.d, p0/m, z1.d
-; CHECK-NEXT: ldr z4, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: ldr z2, [x0, #13, mul vl]
+; CHECK-NEXT: ldr z3, [x0, #12, mul vl]
+; CHECK-NEXT: ldr z4, [x0, #11, mul vl]
+; CHECK-NEXT: ldr z5, [x0, #10, mul vl]
+; CHECK-NEXT: ldr z6, [x0, #9, mul vl]
+; CHECK-NEXT: ldr z7, [x0, #8, mul vl]
; CHECK-NEXT: frintx z0.d, p0/m, z0.d
-; CHECK-NEXT: add x11, x10, x9
-; CHECK-NEXT: ldr z5, [x10, #1, mul vl]
+; CHECK-NEXT: frintx z1.d, p0/m, z1.d
+; CHECK-NEXT: ldr z24, [x0, #7, mul vl]
+; CHECK-NEXT: ldr z25, [x0, #6, mul vl]
; CHECK-NEXT: frintx z2.d, p0/m, z2.d
-; CHECK-NEXT: add x10, x11, x9
-; CHECK-NEXT: ldr z6, [x11, #1, mul vl]
; CHECK-NEXT: frintx z3.d, p0/m, z3.d
-; CHECK-NEXT: ldr z7, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: ldr z26, [x0, #5, mul vl]
+; CHECK-NEXT: ldr z27, [x0, #4, mul vl]
; CHECK-NEXT: frintx z4.d, p0/m, z4.d
-; CHECK-NEXT: ldr z24, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: ldr z28, [x0, #3, mul vl]
+; CHECK-NEXT: ldr z29, [x0, #2, mul vl]
; CHECK-NEXT: frintx z5.d, p0/m, z5.d
-; CHECK-NEXT: add x11, x10, x9
-; CHECK-NEXT: ldr z25, [x10, #1, mul vl]
+; CHECK-NEXT: ldr z30, [x0, #1, mul vl]
+; CHECK-NEXT: ldr z31, [x0]
; CHECK-NEXT: frintx z6.d, p0/m, z6.d
-; CHECK-NEXT: add x10, x11, x9
-; CHECK-NEXT: ldr z26, [x11, #1, mul vl]
; CHECK-NEXT: frintx z7.d, p0/m, z7.d
-; CHECK-NEXT: ldr z27, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x10, x9
-; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d
-; CHECK-NEXT: ldr z28, [x10, #1, mul vl]
-; CHECK-NEXT: add x10, x10, x9
-; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d
-; CHECK-NEXT: add x11, x10, x9
; CHECK-NEXT: frintx z24.d, p0/m, z24.d
-; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.d
-; CHECK-NEXT: add x9, x11, x9
-; CHECK-NEXT: ldr z29, [x11, #1, mul vl]
-; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d
-; CHECK-NEXT: ldr z31, [x10, #1, mul vl]
-; CHECK-NEXT: ldr z30, [x9, #1, mul vl]
; CHECK-NEXT: frintx z25.d, p0/m, z25.d
-; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.d
-; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.d
; CHECK-NEXT: frintx z26.d, p0/m, z26.d
; CHECK-NEXT: frintx z27.d, p0/m, z27.d
; CHECK-NEXT: frintx z28.d, p0/m, z28.d
-; CHECK-NEXT: rdvl x9, #2
-; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d
; CHECK-NEXT: frintx z29.d, p0/m, z29.d
-; CHECK-NEXT: add x10, x8, x9
-; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.d
-; CHECK-NEXT: frintx z31.d, p0/m, z31.d
-; CHECK-NEXT: str z0, [x8, #1, mul vl]
-; CHECK-NEXT: str z1, [x8]
; CHECK-NEXT: frintx z30.d, p0/m, z30.d
+; CHECK-NEXT: frintx z31.d, p0/m, z31.d
+; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d
+; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d
+; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d
+; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.d
+; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.d
+; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.d
+; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d
+; CHECK-NEXT: fcvtzs z7.d, p0/m, z7.d
; CHECK-NEXT: fcvtzs z24.d, p0/m, z24.d
-; CHECK-NEXT: str z2, [x8, #2, mul vl]
; CHECK-NEXT: fcvtzs z25.d, p0/m, z25.d
; CHECK-NEXT: fcvtzs z26.d, p0/m, z26.d
-; CHECK-NEXT: str z3, [x10, #1, mul vl]
-; CHECK-NEXT: rdvl x10, #4
; CHECK-NEXT: fcvtzs z27.d, p0/m, z27.d
-; CHECK-NEXT: add x11, x8, x10
-; CHECK-NEXT: str z4, [x8, #4, mul vl]
; CHECK-NEXT: fcvtzs z28.d, p0/m, z28.d
-; CHECK-NEXT: str z5, [x11, #1, mul vl]
-; CHECK-NEXT: fcvtzs z31.d, p0/m, z31.d
; CHECK-NEXT: fcvtzs z29.d, p0/m, z29.d
-; CHECK-NEXT: str z6, [x11, #2, mul vl]
-; CHECK-NEXT: add x11, x11, x9
; CHECK-NEXT: fcvtzs z30.d, p0/m, z30.d
-; CHECK-NEXT: str z7, [x11, #1, mul vl]
-; CHECK-NEXT: rdvl x11, #8
-; CHECK-NEXT: str z24, [x8, #8, mul vl]
-; CHECK-NEXT: add x8, x8, x11
-; CHECK-NEXT: add x11, x8, x9
-; CHECK-NEXT: str z25, [x8, #1, mul vl]
-; CHECK-NEXT: str z26, [x8, #2, mul vl]
-; CHECK-NEXT: str z27, [x11, #1, mul vl]
-; CHECK-NEXT: str z28, [x8, #4, mul vl]
-; CHECK-NEXT: add x8, x8, x10
-; CHECK-NEXT: str z31, [x8, #1, mul vl]
+; CHECK-NEXT: fcvtzs z31.d, p0/m, z31.d
+; CHECK-NEXT: str z0, [x8, #15, mul vl]
+; CHECK-NEXT: str z1, [x8, #14, mul vl]
+; CHECK-NEXT: str z2, [x8, #13, mul vl]
+; CHECK-NEXT: str z3, [x8, #12, mul vl]
+; CHECK-NEXT: str z4, [x8, #11, mul vl]
+; CHECK-NEXT: str z5, [x8, #10, mul vl]
+; CHECK-NEXT: str z6, [x8, #9, mul vl]
+; CHECK-NEXT: str z7, [x8, #8, mul vl]
+; CHECK-NEXT: str z24, [x8, #7, mul vl]
+; CHECK-NEXT: str z25, [x8, #6, mul vl]
+; CHECK-NEXT: str z26, [x8, #5, mul vl]
+; CHECK-NEXT: str z27, [x8, #4, mul vl]
+; CHECK-NEXT: str z28, [x8, #3, mul vl]
; CHECK-NEXT: str z29, [x8, #2, mul vl]
-; CHECK-NEXT: add x8, x8, x9
; CHECK-NEXT: str z30, [x8, #1, mul vl]
+; CHECK-NEXT: str z31, [x8]
; CHECK-NEXT: ret
%a = call <vscale x 32 x i64> @llvm.lrint.nxv32i64.nxv16f64(<vscale x 32 x double> %x)
ret <vscale x 32 x i64> %a
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll
index 82407d56a9671..5c506ab20e6f4 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll
@@ -231,22 +231,19 @@ define <vscale x 8 x i64> @masked_sload_x2_8i8_8i64(ptr %a, ptr %b, <vscale x 8
; CHECK-LABEL: masked_sload_x2_8i8_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: punpkhi p1.h, p0.b
-; CHECK-NEXT: cntw x8
-; CHECK-NEXT: punpklo p2.h, p0.b
-; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: add x8, x1, x8
-; CHECK-NEXT: punpkhi p0.h, p1.b
+; CHECK-NEXT: punpklo p0.h, p0.b
+; CHECK-NEXT: punpkhi p2.h, p1.b
; CHECK-NEXT: punpklo p1.h, p1.b
-; CHECK-NEXT: punpkhi p3.h, p2.b
-; CHECK-NEXT: ld1sb { z3.d }, p0/z, [x9, #1, mul vl]
-; CHECK-NEXT: ld1sb { z5.d }, p0/z, [x8, #1, mul vl]
-; CHECK-NEXT: punpklo p2.h, p2.b
+; CHECK-NEXT: punpkhi p3.h, p0.b
+; CHECK-NEXT: ld1sb { z3.d }, p2/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1sb { z5.d }, p2/z, [x1, #3, mul vl]
+; CHECK-NEXT: punpklo p0.h, p0.b
; CHECK-NEXT: ld1sb { z2.d }, p1/z, [x0, #2, mul vl]
; CHECK-NEXT: ld1sb { z6.d }, p1/z, [x1, #2, mul vl]
; CHECK-NEXT: ld1sb { z1.d }, p3/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1sb { z7.d }, p3/z, [x1, #1, mul vl]
-; CHECK-NEXT: ld1sb { z0.d }, p2/z, [x0]
-; CHECK-NEXT: ld1sb { z4.d }, p2/z, [x1]
+; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1sb { z4.d }, p0/z, [x1]
; CHECK-NEXT: add z3.d, z3.d, z5.d
; CHECK-NEXT: add z2.d, z2.d, z6.d
; CHECK-NEXT: add z1.d, z1.d, z7.d
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-zext.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-zext.ll
index 535fb5f4f9660..09e276f1cc433 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-ldst-zext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-zext.ll
@@ -226,22 +226,19 @@ define <vscale x 8 x i64> @masked_zload_x2_8i8_8i64(ptr %a, ptr %b, <vscale x 8
; CHECK-LABEL: masked_zload_x2_8i8_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: punpkhi p1.h, p0.b
-; CHECK-NEXT: cntw x8
-; CHECK-NEXT: punpklo p2.h, p0.b
-; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: add x8, x1, x8
-; CHECK-NEXT: punpkhi p0.h, p1.b
+; CHECK-NEXT: punpklo p0.h, p0.b
+; CHECK-NEXT: punpkhi p2.h, p1.b
; CHECK-NEXT: punpklo p1.h, p1.b
-; CHECK-NEXT: punpkhi p3.h, p2.b
-; CHECK-NEXT: ld1b { z3.d }, p0/z, [x9, #1, mul vl]
-; CHECK-NEXT: ld1b { z5.d }, p0/z, [x8, #1, mul vl]
-; CHECK-NEXT: punpklo p2.h, p2.b
+; CHECK-NEXT: punpkhi p3.h, p0.b
+; CHECK-NEXT: ld1b { z3.d }, p2/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1b { z5.d }, p2/z, [x1, #3, mul vl]
+; CHECK-NEXT: punpklo p0.h, p0.b
; CHECK-NEXT: ld1b { z2.d }, p1/z, [x0, #2, mul vl]
; CHECK-NEXT: ld1b { z6.d }, p1/z, [x1, #2, mul vl]
; CHECK-NEXT: ld1b { z1.d }, p3/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1b { z7.d }, p3/z, [x1, #1, mul vl]
-; CHECK-NEXT: ld1b { z0.d }, p2/z, [x0]
-; CHECK-NEXT: ld1b { z4.d }, p2/z, [x1]
+; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
+; CHECK-NEXT: ld1b { z4.d }, p0/z, [x1]
; CHECK-NEXT: add z3.d, z3.d, z5.d
; CHECK-NEXT: add z2.d, z2.d, z6.d
; CHECK-NEXT: add z1.d, z1.d, z7.d
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter-legalize.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter-legalize.ll
index 302e22db91734..622040eeb2f33 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter-legalize.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter-legalize.ll
@@ -89,25 +89,18 @@ define void @masked_scatter_nxv8f32(<vscale x 8 x float> %data, ptr %base, <vsca
define void @masked_scatter_nxv32i32(<vscale x 32 x i32> %data, ptr %base, <vscale x 32 x i32> %offsets, <vscale x 32 x i1> %mask) #0 {
; CHECK-LABEL: masked_scatter_nxv32i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: punpklo p2.h, p0.b
; CHECK-NEXT: ldr z30, [x1, #1, mul vl]
-; CHECK-NEXT: add x9, x1, x8
; CHECK-NEXT: ldr z31, [x1]
; CHECK-NEXT: punpkhi p0.h, p0.b
-; CHECK-NEXT: add x10, x9, x8
+; CHECK-NEXT: ldr z28, [x1, #3, mul vl]
+; CHECK-NEXT: ldr z29, [x1, #2, mul vl]
; CHECK-NEXT: punpklo p3.h, p2.b
-; CHECK-NEXT: ldr z29, [x9, #1, mul vl]
-; CHECK-NEXT: add x11, x10, x8
+; CHECK-NEXT: ldr z24, [x1, #7, mul vl]
+; CHECK-NEXT: ldr z25, [x1, #6, mul vl]
; CHECK-NEXT: punpkhi p2.h, p2.b
-; CHECK-NEXT: ldr z28, [x10, #1, mul vl]
-; CHECK-NEXT: add x12, x11, x8
-; CHECK-NEXT: ldr z27, [x11, #1, mul vl]
-; CHECK-NEXT: add x13, x12, x8
-; CHECK-NEXT: ldr z26, [x12, #1, mul vl]
-; CHECK-NEXT: add x8, x13, x8
-; CHECK-NEXT: ldr z25, [x13, #1, mul vl]
-; CHECK-NEXT: ldr z24, [x8, #1, mul vl]
+; CHECK-NEXT: ldr z26, [x1, #5, mul vl]
+; CHECK-NEXT: ldr z27, [x1, #4, mul vl]
; CHECK-NEXT: st1w { z0.s }, p3, [x0, z31.s, sxtw #2]
; CHECK-NEXT: st1w { z1.s }, p2, [x0, z30.s, sxtw #2]
; CHECK-NEXT: punpklo p2.h, p0.b
diff --git a/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll b/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
index cda0c7aef56d1..714b6a0874cd7 100644
--- a/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
@@ -615,20 +615,18 @@ define void @load_store_2x_vectors_bf16_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_i8_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_i8_r:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0]
-; SVE2p1-NEXT: incb x8, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: ldr z2, [x0]
+; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
; SVE2p1-NEXT: add z0.b, z0.b, #5 // =0x5
; SVE2p1-NEXT: add z1.b, z1.b, #5 // =0x5
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: add z2.b, z2.b, #5 // =0x5
-; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
; SVE2p1-NEXT: add z3.b, z3.b, #5 // =0x5
-; SVE2p1-NEXT: str z1, [x0]
-; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
-; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
+; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: str z2, [x0]
+; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i8_r:
@@ -661,21 +659,19 @@ define void @load_store_4x_vectors_i8_r(ptr %addr) {
define void @load_store_4x_vectors_i8_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_i8_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: add x8, x0, x1
; SVE2p1-NEXT: ptrue p0.b
-; SVE2p1-NEXT: add x9, x0, x1
-; SVE2p1-NEXT: incb x8, all, mul #2
-; SVE2p1-NEXT: ldr z1, [x9, #2, mul vl]
-; SVE2p1-NEXT: ldr z2, [x9, #1, mul vl]
-; SVE2p1-NEXT: ld1b { z0.b }, p0/z, [x0, x1]
+; SVE2p1-NEXT: add x8, x0, x1
+; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
+; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: ld1b { z0.b }, p0/z, [x0, x1]
; SVE2p1-NEXT: add z1.b, z1.b, #5 // =0x5
; SVE2p1-NEXT: add z2.b, z2.b, #5 // =0x5
-; SVE2p1-NEXT: add z0.b, z0.b, #5 // =0x5
; SVE2p1-NEXT: add z3.b, z3.b, #5 // =0x5
+; SVE2p1-NEXT: add z0.b, z0.b, #5 // =0x5
; SVE2p1-NEXT: st1b { z0.b }, p0, [x0, x1]
-; SVE2p1-NEXT: str z1, [x9, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x9, #1, mul vl]
+; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
+; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
@@ -712,20 +708,18 @@ define void @load_store_4x_vectors_i8_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_i16_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_i16_r:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0]
-; SVE2p1-NEXT: incb x8, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: ldr z2, [x0]
+; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
; SVE2p1-NEXT: add z0.h, z0.h, #5 // =0x5
; SVE2p1-NEXT: add z1.h, z1.h, #5 // =0x5
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: add z2.h, z2.h, #5 // =0x5
-; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
; SVE2p1-NEXT: add z3.h, z3.h, #5 // =0x5
-; SVE2p1-NEXT: str z1, [x0]
-; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
-; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
+; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: str z2, [x0]
+; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i16_r:
@@ -758,22 +752,20 @@ define void @load_store_4x_vectors_i16_r(ptr %addr) {
define void @load_store_4x_vectors_i16_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_i16_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: add x8, x0, x1, lsl #1
; SVE2p1-NEXT: ptrue p0.h
-; SVE2p1-NEXT: mov x9, x8
+; SVE2p1-NEXT: add x8, x0, x1, lsl #1
; SVE2p1-NEXT: ld1h { z0.h }, p0/z, [x0, x1, lsl #1]
; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: incb x9, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x8, #1, mul vl]
+; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: add z0.h, z0.h, #5 // =0x5
; SVE2p1-NEXT: add z1.h, z1.h, #5 // =0x5
-; SVE2p1-NEXT: ldr z3, [x9, #1, mul vl]
; SVE2p1-NEXT: add z2.h, z2.h, #5 // =0x5
-; SVE2p1-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1]
; SVE2p1-NEXT: add z3.h, z3.h, #5 // =0x5
+; SVE2p1-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1]
; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x8, #1, mul vl]
-; SVE2p1-NEXT: str z3, [x9, #1, mul vl]
+; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
+; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i16_rr:
@@ -809,20 +801,18 @@ define void @load_store_4x_vectors_i16_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_i32_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_i32_r:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0]
-; SVE2p1-NEXT: incb x8, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: ldr z2, [x0]
+; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
; SVE2p1-NEXT: add z0.s, z0.s, #5 // =0x5
; SVE2p1-NEXT: add z1.s, z1.s, #5 // =0x5
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: add z2.s, z2.s, #5 // =0x5
-; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
; SVE2p1-NEXT: add z3.s, z3.s, #5 // =0x5
-; SVE2p1-NEXT: str z1, [x0]
-; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
-; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
+; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: str z2, [x0]
+; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i32_r:
@@ -855,22 +845,20 @@ define void @load_store_4x_vectors_i32_r(ptr %addr) {
define void @load_store_4x_vectors_i32_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_i32_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: add x8, x0, x1, lsl #2
; SVE2p1-NEXT: ptrue p0.s
-; SVE2p1-NEXT: mov x9, x8
+; SVE2p1-NEXT: add x8, x0, x1, lsl #2
; SVE2p1-NEXT: ld1w { z0.s }, p0/z, [x0, x1, lsl #2]
; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: incb x9, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x8, #1, mul vl]
+; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: add z0.s, z0.s, #5 // =0x5
; SVE2p1-NEXT: add z1.s, z1.s, #5 // =0x5
-; SVE2p1-NEXT: ldr z3, [x9, #1, mul vl]
; SVE2p1-NEXT: add z2.s, z2.s, #5 // =0x5
-; SVE2p1-NEXT: st1w { z0.s }, p0, [x0, x1, lsl #2]
; SVE2p1-NEXT: add z3.s, z3.s, #5 // =0x5
+; SVE2p1-NEXT: st1w { z0.s }, p0, [x0, x1, lsl #2]
; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x8, #1, mul vl]
-; SVE2p1-NEXT: str z3, [x9, #1, mul vl]
+; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
+; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i32_rr:
@@ -906,20 +894,18 @@ define void @load_store_4x_vectors_i32_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_i64_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_i64_r:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0]
-; SVE2p1-NEXT: incb x8, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: ldr z2, [x0]
+; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
; SVE2p1-NEXT: add z0.d, z0.d, #5 // =0x5
; SVE2p1-NEXT: add z1.d, z1.d, #5 // =0x5
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: add z2.d, z2.d, #5 // =0x5
-; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
; SVE2p1-NEXT: add z3.d, z3.d, #5 // =0x5
-; SVE2p1-NEXT: str z1, [x0]
-; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
-; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
+; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: str z2, [x0]
+; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i64_r:
@@ -952,22 +938,20 @@ define void @load_store_4x_vectors_i64_r(ptr %addr) {
define void @load_store_4x_vectors_i64_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_i64_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: add x8, x0, x1, lsl #3
; SVE2p1-NEXT: ptrue p0.d
-; SVE2p1-NEXT: mov x9, x8
+; SVE2p1-NEXT: add x8, x0, x1, lsl #3
; SVE2p1-NEXT: ld1d { z0.d }, p0/z, [x0, x1, lsl #3]
; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: incb x9, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x8, #1, mul vl]
+; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: add z0.d, z0.d, #5 // =0x5
; SVE2p1-NEXT: add z1.d, z1.d, #5 // =0x5
-; SVE2p1-NEXT: ldr z3, [x9, #1, mul vl]
; SVE2p1-NEXT: add z2.d, z2.d, #5 // =0x5
-; SVE2p1-NEXT: st1d { z0.d }, p0, [x0, x1, lsl #3]
; SVE2p1-NEXT: add z3.d, z3.d, #5 // =0x5
+; SVE2p1-NEXT: st1d { z0.d }, p0, [x0, x1, lsl #3]
; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x8, #1, mul vl]
-; SVE2p1-NEXT: str z3, [x9, #1, mul vl]
+; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
+; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_i64_rr:
@@ -1003,21 +987,19 @@ define void @load_store_4x_vectors_i64_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_f16_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_f16_r:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0]
-; SVE2p1-NEXT: incb x8, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
; SVE2p1-NEXT: ptrue p0.h
+; SVE2p1-NEXT: ldr z2, [x0]
+; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
; SVE2p1-NEXT: fadd z0.h, p0/m, z0.h, #1.0
; SVE2p1-NEXT: fadd z1.h, p0/m, z1.h, #1.0
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: fadd z2.h, p0/m, z2.h, #1.0
; SVE2p1-NEXT: fadd z3.h, p0/m, z3.h, #1.0
; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: str z1, [x0]
-; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
-; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: str z2, [x0]
+; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_f16_r:
@@ -1052,22 +1034,20 @@ define void @load_store_4x_vectors_f16_r(ptr %addr) {
define void @load_store_4x_vectors_f16_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_f16_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: add x8, x0, x1, lsl #1
; SVE2p1-NEXT: ptrue p0.h
-; SVE2p1-NEXT: mov x9, x8
+; SVE2p1-NEXT: add x8, x0, x1, lsl #1
; SVE2p1-NEXT: ld1h { z0.h }, p0/z, [x0, x1, lsl #1]
; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: incb x9, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x8, #1, mul vl]
+; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: fadd z0.h, p0/m, z0.h, #1.0
; SVE2p1-NEXT: fadd z1.h, p0/m, z1.h, #1.0
-; SVE2p1-NEXT: ldr z3, [x9, #1, mul vl]
; SVE2p1-NEXT: fadd z2.h, p0/m, z2.h, #1.0
; SVE2p1-NEXT: fadd z3.h, p0/m, z3.h, #1.0
; SVE2p1-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1]
; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x8, #1, mul vl]
-; SVE2p1-NEXT: str z3, [x9, #1, mul vl]
+; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
+; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_f16_rr:
@@ -1105,21 +1085,19 @@ define void @load_store_4x_vectors_f16_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_f32_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_f32_r:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0]
-; SVE2p1-NEXT: incb x8, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
; SVE2p1-NEXT: ptrue p0.s
+; SVE2p1-NEXT: ldr z2, [x0]
+; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
; SVE2p1-NEXT: fadd z0.s, p0/m, z0.s, #1.0
; SVE2p1-NEXT: fadd z1.s, p0/m, z1.s, #1.0
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: fadd z2.s, p0/m, z2.s, #1.0
; SVE2p1-NEXT: fadd z3.s, p0/m, z3.s, #1.0
; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: str z1, [x0]
-; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
-; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: str z2, [x0]
+; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_f32_r:
@@ -1154,22 +1132,20 @@ define void @load_store_4x_vectors_f32_r(ptr %addr) {
define void @load_store_4x_vectors_f32_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_f32_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: add x8, x0, x1, lsl #2
; SVE2p1-NEXT: ptrue p0.s
-; SVE2p1-NEXT: mov x9, x8
+; SVE2p1-NEXT: add x8, x0, x1, lsl #2
; SVE2p1-NEXT: ld1w { z0.s }, p0/z, [x0, x1, lsl #2]
; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: incb x9, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x8, #1, mul vl]
+; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: fadd z0.s, p0/m, z0.s, #1.0
; SVE2p1-NEXT: fadd z1.s, p0/m, z1.s, #1.0
-; SVE2p1-NEXT: ldr z3, [x9, #1, mul vl]
; SVE2p1-NEXT: fadd z2.s, p0/m, z2.s, #1.0
; SVE2p1-NEXT: fadd z3.s, p0/m, z3.s, #1.0
; SVE2p1-NEXT: st1w { z0.s }, p0, [x0, x1, lsl #2]
; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x8, #1, mul vl]
-; SVE2p1-NEXT: str z3, [x9, #1, mul vl]
+; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
+; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_f32_rr:
@@ -1207,21 +1183,19 @@ define void @load_store_4x_vectors_f32_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_f64_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_f64_r:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: ldr z1, [x0]
-; SVE2p1-NEXT: incb x8, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x0, #1, mul vl]
+; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
; SVE2p1-NEXT: ptrue p0.d
+; SVE2p1-NEXT: ldr z2, [x0]
+; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
; SVE2p1-NEXT: fadd z0.d, p0/m, z0.d, #1.0
; SVE2p1-NEXT: fadd z1.d, p0/m, z1.d, #1.0
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: fadd z2.d, p0/m, z2.d, #1.0
; SVE2p1-NEXT: fadd z3.d, p0/m, z3.d, #1.0
; SVE2p1-NEXT: str z0, [x0, #2, mul vl]
-; SVE2p1-NEXT: str z1, [x0]
-; SVE2p1-NEXT: str z2, [x0, #1, mul vl]
-; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z1, [x0, #3, mul vl]
+; SVE2p1-NEXT: str z2, [x0]
+; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_f64_r:
@@ -1256,22 +1230,20 @@ define void @load_store_4x_vectors_f64_r(ptr %addr) {
define void @load_store_4x_vectors_f64_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_f64_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: add x8, x0, x1, lsl #3
; SVE2p1-NEXT: ptrue p0.d
-; SVE2p1-NEXT: mov x9, x8
+; SVE2p1-NEXT: add x8, x0, x1, lsl #3
; SVE2p1-NEXT: ld1d { z0.d }, p0/z, [x0, x1, lsl #3]
; SVE2p1-NEXT: ldr z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: incb x9, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x8, #1, mul vl]
+; SVE2p1-NEXT: ldr z2, [x8, #3, mul vl]
+; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
; SVE2p1-NEXT: fadd z0.d, p0/m, z0.d, #1.0
; SVE2p1-NEXT: fadd z1.d, p0/m, z1.d, #1.0
-; SVE2p1-NEXT: ldr z3, [x9, #1, mul vl]
; SVE2p1-NEXT: fadd z2.d, p0/m, z2.d, #1.0
; SVE2p1-NEXT: fadd z3.d, p0/m, z3.d, #1.0
; SVE2p1-NEXT: st1d { z0.d }, p0, [x0, x1, lsl #3]
; SVE2p1-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-NEXT: str z2, [x8, #1, mul vl]
-; SVE2p1-NEXT: str z3, [x9, #1, mul vl]
+; SVE2p1-NEXT: str z2, [x8, #3, mul vl]
+; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_f64_rr:
@@ -1309,21 +1281,19 @@ define void @load_store_4x_vectors_f64_rr(ptr %base, i64 %idx) {
define void @load_store_4x_vectors_bf16_r(ptr %addr) {
; SVE2p1-LABEL: load_store_4x_vectors_bf16_r:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: mov x8, x0
; SVE2p1-NEXT: fmov z0.h, #1.87500000
; SVE2p1-NEXT: ldr z1, [x0, #2, mul vl]
-; SVE2p1-NEXT: incb x8, all, mul #2
-; SVE2p1-NEXT: ldr z2, [x0]
-; SVE2p1-NEXT: ldr z3, [x0, #1, mul vl]
+; SVE2p1-NEXT: ldr z2, [x0, #3, mul vl]
+; SVE2p1-NEXT: ldr z3, [x0]
+; SVE2p1-NEXT: ldr z4, [x0, #1, mul vl]
; SVE2p1-NEXT: bfadd z1.h, z1.h, z0.h
-; SVE2p1-NEXT: ldr z4, [x8, #1, mul vl]
; SVE2p1-NEXT: bfadd z2.h, z2.h, z0.h
; SVE2p1-NEXT: str z1, [x0, #2, mul vl]
; SVE2p1-NEXT: bfadd z1.h, z3.h, z0.h
-; SVE2p1-NEXT: str z2, [x0]
-; SVE2p1-NEXT: str z1, [x0, #1, mul vl]
; SVE2p1-NEXT: bfadd z0.h, z4.h, z0.h
-; SVE2p1-NEXT: str z0, [x8, #1, mul vl]
+; SVE2p1-NEXT: str z2, [x0, #3, mul vl]
+; SVE2p1-NEXT: str z1, [x0]
+; SVE2p1-NEXT: str z0, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_bf16_r:
@@ -1358,23 +1328,21 @@ define void @load_store_4x_vectors_bf16_r(ptr %addr) {
define void @load_store_4x_vectors_bf16_rr(ptr %base, i64 %idx) {
; SVE2p1-LABEL: load_store_4x_vectors_bf16_rr:
; SVE2p1: // %bb.0:
-; SVE2p1-NEXT: add x8, x0, x1, lsl #1
; SVE2p1-NEXT: ptrue p0.h
-; SVE2p1-NEXT: fmov z1.h, #1.87500000
-; SVE2p1-NEXT: mov x9, x8
-; SVE2p1-NEXT: ld1h { z0.h }, p0/z, [x0, x1, lsl #1]
+; SVE2p1-NEXT: add x8, x0, x1, lsl #1
+; SVE2p1-NEXT: fmov z0.h, #1.87500000
+; SVE2p1-NEXT: ld1h { z1.h }, p0/z, [x0, x1, lsl #1]
; SVE2p1-NEXT: ldr z2, [x8, #2, mul vl]
-; SVE2p1-NEXT: incb x9, all, mul #2
-; SVE2p1-NEXT: ldr z3, [x8, #1, mul vl]
-; SVE2p1-NEXT: bfadd z0.h, z0.h, z1.h
-; SVE2p1-NEXT: bfadd z2.h, z2.h, z1.h
-; SVE2p1-NEXT: ldr z4, [x9, #1, mul vl]
-; SVE2p1-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-NEXT: bfadd z0.h, z3.h, z1.h
+; SVE2p1-NEXT: ldr z3, [x8, #3, mul vl]
+; SVE2p1-NEXT: ldr z4, [x8, #1, mul vl]
+; SVE2p1-NEXT: bfadd z1.h, z1.h, z0.h
+; SVE2p1-NEXT: bfadd z2.h, z2.h, z0.h
+; SVE2p1-NEXT: st1h { z1.h }, p0, [x0, x1, lsl #1]
+; SVE2p1-NEXT: bfadd z1.h, z3.h, z0.h
+; SVE2p1-NEXT: bfadd z0.h, z4.h, z0.h
; SVE2p1-NEXT: str z2, [x8, #2, mul vl]
+; SVE2p1-NEXT: str z1, [x8, #3, mul vl]
; SVE2p1-NEXT: str z0, [x8, #1, mul vl]
-; SVE2p1-NEXT: bfadd z1.h, z4.h, z1.h
-; SVE2p1-NEXT: str z1, [x9, #1, mul vl]
; SVE2p1-NEXT: ret
;
; SVE2p1-SL-LABEL: load_store_4x_vectors_bf16_rr:
diff --git a/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll b/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll
index d7ef9b520373d..0bc8cb8bc5003 100644
--- a/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll
+++ b/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll
@@ -91,18 +91,16 @@ define i64 @split_extract_8i64_idx(<vscale x 8 x i64> %a, i32 %idx) {
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: mov w10, w0
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
; CHECK-NEXT: cnth x8
+; CHECK-NEXT: mov w9, w0
+; CHECK-NEXT: str z3, [sp, #3, mul vl]
; CHECK-NEXT: sub x8, x8, #1
; CHECK-NEXT: str z2, [sp, #2, mul vl]
-; CHECK-NEXT: cmp x10, x8
+; CHECK-NEXT: cmp x9, x8
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: csel x8, x10, x8, lo
+; CHECK-NEXT: csel x8, x9, x8, lo
+; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: ldr x0, [x9, x8, lsl #3]
; CHECK-NEXT: addvl sp, sp, #4
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -161,19 +159,17 @@ define i32 @split_extract_16i32(<vscale x 16 x i32> %a) {
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: rdvl x10, #1
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: sub x10, x10, #1
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
-; CHECK-NEXT: mov w8, #34464 // =0x86a0
-; CHECK-NEXT: movk w8, #1, lsl #16
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: mov w9, #34464 // =0x86a0
+; CHECK-NEXT: str z3, [sp, #3, mul vl]
+; CHECK-NEXT: movk w9, #1, lsl #16
+; CHECK-NEXT: sub x8, x8, #1
; CHECK-NEXT: str z2, [sp, #2, mul vl]
-; CHECK-NEXT: cmp x10, x8
+; CHECK-NEXT: cmp x8, x9
; CHECK-NEXT: str z1, [sp, #1, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: csel x8, x10, x8, lo
+; CHECK-NEXT: csel x8, x8, x9, lo
+; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: ldr w0, [x9, x8, lsl #2]
; CHECK-NEXT: addvl sp, sp, #4
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll b/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll
index 3fccd94bdb508..6116473d9588e 100644
--- a/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll
+++ b/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll
@@ -71,22 +71,20 @@ define <vscale x 8 x i64> @split_insert_8i64_idx(<vscale x 8 x i64> %a, i64 %elt
; CHECK-NEXT: addvl sp, sp, #-4
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: cnth x10
-; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: cnth x8
; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: sub x10, x10, #1
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: cmp x1, x10
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
-; CHECK-NEXT: csel x10, x1, x10, lo
+; CHECK-NEXT: str z3, [sp, #3, mul vl]
+; CHECK-NEXT: sub x8, x8, #1
; CHECK-NEXT: str z2, [sp, #2, mul vl]
+; CHECK-NEXT: cmp x1, x8
; CHECK-NEXT: str z1, [sp, #1, mul vl]
+; CHECK-NEXT: csel x8, x1, x8, lo
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: str x0, [x9, x10, lsl #3]
+; CHECK-NEXT: str x0, [x9, x8, lsl #3]
; CHECK-NEXT: ldr z0, [sp]
-; CHECK-NEXT: ldr z3, [x8, #1, mul vl]
; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
+; CHECK-NEXT: ldr z3, [sp, #3, mul vl]
; CHECK-NEXT: addvl sp, sp, #4
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
@@ -135,21 +133,20 @@ define <vscale x 32 x i16> @split_insert_32i16(<vscale x 32 x i16> %a, i16 %elt)
; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: rdvl x8, #2
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: mov w11, #128 // =0x80
-; CHECK-NEXT: add x10, x9, x8
+; CHECK-NEXT: mov w9, #128 // =0x80
+; CHECK-NEXT: str z3, [sp, #3, mul vl]
; CHECK-NEXT: sub x8, x8, #1
-; CHECK-NEXT: cmp x8, #128
-; CHECK-NEXT: str z3, [x10, #1, mul vl]
-; CHECK-NEXT: csel x8, x8, x11, lo
; CHECK-NEXT: str z2, [sp, #2, mul vl]
+; CHECK-NEXT: cmp x8, #128
; CHECK-NEXT: str z1, [sp, #1, mul vl]
+; CHECK-NEXT: csel x8, x8, x9, lo
+; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: str z0, [sp]
; CHECK-NEXT: strh w0, [x9, x8, lsl #1]
; CHECK-NEXT: ldr z0, [sp]
-; CHECK-NEXT: ldr z3, [x10, #1, mul vl]
; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
+; CHECK-NEXT: ldr z3, [sp, #3, mul vl]
; CHECK-NEXT: addvl sp, sp, #4
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-split-load.ll b/llvm/test/CodeGen/AArch64/sve-split-load.ll
index c14d6d10ad49c..e1dd66c9d249a 100644
--- a/llvm/test/CodeGen/AArch64/sve-split-load.ll
+++ b/llvm/test/CodeGen/AArch64/sve-split-load.ll
@@ -26,11 +26,9 @@ define <vscale x 16 x i16> @load_split_i16(ptr %a) {
define <vscale x 24 x i16> @load_split_24i16(ptr %a) {
; CHECK-LABEL: load_split_24i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
-; CHECK-NEXT: add x8, x0, x8
-; CHECK-NEXT: ldr z2, [x8, #1, mul vl]
+; CHECK-NEXT: ldr z2, [x0, #2, mul vl]
; CHECK-NEXT: ret
%load = load <vscale x 24 x i16>, ptr %a
ret <vscale x 24 x i16> %load
@@ -39,12 +37,10 @@ define <vscale x 24 x i16> @load_split_24i16(ptr %a) {
define <vscale x 32 x i16> @load_split_32i16(ptr %a) {
; CHECK-LABEL: load_split_32i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: ldr z2, [x0, #2, mul vl]
-; CHECK-NEXT: ldr z3, [x8, #1, mul vl]
+; CHECK-NEXT: ldr z3, [x0, #3, mul vl]
; CHECK-NEXT: ret
%load = load <vscale x 32 x i16>, ptr %a
ret <vscale x 32 x i16> %load
@@ -53,19 +49,14 @@ define <vscale x 32 x i16> @load_split_32i16(ptr %a) {
define <vscale x 16 x i64> @load_split_16i64(ptr %a) {
; CHECK-LABEL: load_split_16i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
-; CHECK-NEXT: add x9, x0, x8
; CHECK-NEXT: ldr z2, [x0, #2, mul vl]
+; CHECK-NEXT: ldr z3, [x0, #3, mul vl]
; CHECK-NEXT: ldr z4, [x0, #4, mul vl]
-; CHECK-NEXT: ldr z3, [x9, #1, mul vl]
-; CHECK-NEXT: rdvl x9, #4
-; CHECK-NEXT: add x9, x0, x9
-; CHECK-NEXT: add x8, x9, x8
-; CHECK-NEXT: ldr z5, [x9, #1, mul vl]
-; CHECK-NEXT: ldr z6, [x9, #2, mul vl]
-; CHECK-NEXT: ldr z7, [x8, #1, mul vl]
+; CHECK-NEXT: ldr z5, [x0, #5, mul vl]
+; CHECK-NEXT: ldr z6, [x0, #6, mul vl]
+; CHECK-NEXT: ldr z7, [x0, #7, mul vl]
; CHECK-NEXT: ret
%load = load <vscale x 16 x i64>, ptr %a
ret <vscale x 16 x i64> %load
@@ -96,15 +87,13 @@ define <vscale x 32 x i16> @masked_load_split_32i16(ptr %a, <vscale x 32 x i1> %
; CHECK-LABEL: masked_load_split_32i16:
; CHECK: // %bb.0:
; CHECK-NEXT: punpklo p2.h, p0.b
-; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: punpkhi p0.h, p0.b
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: punpklo p3.h, p1.b
; CHECK-NEXT: ld1h { z0.h }, p2/z, [x0]
; CHECK-NEXT: punpkhi p1.h, p1.b
; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1h { z2.h }, p3/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1h { z3.h }, p1/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1h { z3.h }, p1/z, [x0, #3, mul vl]
; CHECK-NEXT: ret
%load = call <vscale x 32 x i16> @llvm.masked.load.nxv32i16(ptr %a, i32 1, <vscale x 32 x i1> %pg, <vscale x 32 x i16> poison)
ret <vscale x 32 x i16> %load
@@ -126,9 +115,7 @@ define <vscale x 8 x i64> @masked_load_split_8i64(ptr %a, <vscale x 8 x i1> %pg)
; CHECK-LABEL: masked_load_split_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: punpklo p1.h, p0.b
-; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: punpkhi p0.h, p0.b
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: punpklo p2.h, p1.b
; CHECK-NEXT: punpkhi p1.h, p1.b
; CHECK-NEXT: punpklo p3.h, p0.b
@@ -136,7 +123,7 @@ define <vscale x 8 x i64> @masked_load_split_8i64(ptr %a, <vscale x 8 x i1> %pg)
; CHECK-NEXT: punpkhi p0.h, p0.b
; CHECK-NEXT: ld1d { z1.d }, p1/z, [x0, #1, mul vl]
; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #2, mul vl]
-; CHECK-NEXT: ld1d { z3.d }, p0/z, [x8, #1, mul vl]
+; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0, #3, mul vl]
; CHECK-NEXT: ret
%load = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64(ptr %a, i32 1, <vscale x 8 x i1> %pg, <vscale x 8 x i64> poison)
ret <vscale x 8 x i64> %load
diff --git a/llvm/test/CodeGen/AArch64/sve-split-store.ll b/llvm/test/CodeGen/AArch64/sve-split-store.ll
index 774c2058253c8..b1419b3f679cf 100644
--- a/llvm/test/CodeGen/AArch64/sve-split-store.ll
+++ b/llvm/test/CodeGen/AArch64/sve-split-store.ll
@@ -26,12 +26,10 @@ define void @store_split_i16(<vscale x 16 x i16> %data, ptr %a) {
define void @store_split_16i32(<vscale x 16 x i32> %data, ptr %a) {
; CHECK-LABEL: store_split_16i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #2
+; CHECK-NEXT: str z3, [x0, #3, mul vl]
; CHECK-NEXT: str z2, [x0, #2, mul vl]
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: str z1, [x0, #1, mul vl]
; CHECK-NEXT: str z0, [x0]
-; CHECK-NEXT: str z3, [x8, #1, mul vl]
; CHECK-NEXT: ret
store <vscale x 16 x i32> %data, ptr %a
ret void
@@ -40,19 +38,14 @@ define void @store_split_16i32(<vscale x 16 x i32> %data, ptr %a) {
define void @store_split_16i64(<vscale x 16 x i64> %data, ptr %a) {
; CHECK-LABEL: store_split_16i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #4
-; CHECK-NEXT: rdvl x9, #2
+; CHECK-NEXT: str z7, [x0, #7, mul vl]
+; CHECK-NEXT: str z6, [x0, #6, mul vl]
+; CHECK-NEXT: str z5, [x0, #5, mul vl]
; CHECK-NEXT: str z4, [x0, #4, mul vl]
-; CHECK-NEXT: add x8, x0, x8
+; CHECK-NEXT: str z3, [x0, #3, mul vl]
; CHECK-NEXT: str z2, [x0, #2, mul vl]
-; CHECK-NEXT: add x10, x0, x9
; CHECK-NEXT: str z1, [x0, #1, mul vl]
; CHECK-NEXT: str z0, [x0]
-; CHECK-NEXT: str z6, [x8, #2, mul vl]
-; CHECK-NEXT: str z5, [x8, #1, mul vl]
-; CHECK-NEXT: add x8, x8, x9
-; CHECK-NEXT: str z3, [x10, #1, mul vl]
-; CHECK-NEXT: str z7, [x8, #1, mul vl]
; CHECK-NEXT: ret
store <vscale x 16 x i64> %data, ptr %a
ret void
@@ -82,15 +75,13 @@ define void @masked_store_split_32i8(<vscale x 32 x i8> %data, ptr %a, <vscale x
define void @masked_store_split_32i16(<vscale x 32 x i16> %data, ptr %a, <vscale x 32 x i1> %pg) {
; CHECK-LABEL: masked_store_split_32i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: punpkhi p2.h, p1.b
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: punpklo p1.h, p1.b
-; CHECK-NEXT: st1h { z3.h }, p2, [x8, #1, mul vl]
-; CHECK-NEXT: punpkhi p2.h, p0.b
+; CHECK-NEXT: punpkhi p3.h, p0.b
+; CHECK-NEXT: st1h { z3.h }, p2, [x0, #3, mul vl]
; CHECK-NEXT: punpklo p0.h, p0.b
; CHECK-NEXT: st1h { z2.h }, p1, [x0, #2, mul vl]
-; CHECK-NEXT: st1h { z1.h }, p2, [x0, #1, mul vl]
+; CHECK-NEXT: st1h { z1.h }, p3, [x0, #1, mul vl]
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
call void @llvm.masked.store.nxv32i16(<vscale x 32 x i16> %data, ptr %a, i32 1, <vscale x 32 x i1> %pg)
@@ -113,12 +104,10 @@ define void @masked_store_split_8i64(<vscale x 8 x i64> %data, ptr %a, <vscale x
; CHECK-LABEL: masked_store_split_8i64:
; CHECK: // %bb.0:
; CHECK-NEXT: punpkhi p1.h, p0.b
-; CHECK-NEXT: rdvl x8, #2
; CHECK-NEXT: punpklo p0.h, p0.b
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: punpkhi p2.h, p1.b
; CHECK-NEXT: punpklo p1.h, p1.b
-; CHECK-NEXT: st1d { z3.d }, p2, [x8, #1, mul vl]
+; CHECK-NEXT: st1d { z3.d }, p2, [x0, #3, mul vl]
; CHECK-NEXT: punpkhi p2.h, p0.b
; CHECK-NEXT: punpklo p0.h, p0.b
; CHECK-NEXT: st1d { z2.d }, p1, [x0, #2, mul vl]
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave-widen.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave-widen.ll
index a198abc0093c5..d67488981bbcd 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-interleave-widen.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave-widen.ll
@@ -116,70 +116,65 @@ define void @interleave4_v3f32(ptr %dst, <vscale x 3 x float> %a, <vscale x 3
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x98, 0x01, 0x1e, 0x22 // sp + 16 + 152 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: mov w9, #3 // =0x3
; CHECK-NEXT: zip2 z4.s, z1.s, z3.s
+; CHECK-NEXT: lsr x8, x8, #4
; CHECK-NEXT: zip2 z5.s, z0.s, z2.s
-; CHECK-NEXT: lsr x9, x8, #4
-; CHECK-NEXT: mov w11, #3 // =0x3
; CHECK-NEXT: zip1 z1.s, z1.s, z3.s
; CHECK-NEXT: zip1 z0.s, z0.s, z2.s
-; CHECK-NEXT: rdvl x10, #2
-; CHECK-NEXT: addvl x12, sp, #9
-; CHECK-NEXT: mul x9, x9, x11
+; CHECK-NEXT: mul x8, x8, x9
+; CHECK-NEXT: cntw x9, all, mul #3
; CHECK-NEXT: zip2 z2.s, z5.s, z4.s
; CHECK-NEXT: zip1 z3.s, z5.s, z4.s
-; CHECK-NEXT: add x11, x12, x10
-; CHECK-NEXT: add x8, x0, x8
; CHECK-NEXT: zip2 z4.s, z0.s, z1.s
; CHECK-NEXT: zip1 z0.s, z0.s, z1.s
-; CHECK-NEXT: str z2, [x11, #1, mul vl]
-; CHECK-NEXT: cntw x11, all, mul #3
+; CHECK-NEXT: str z2, [sp, #12, mul vl]
+; CHECK-NEXT: whilelo p0.s, xzr, x8
+; CHECK-NEXT: addvl x8, sp, #9
; CHECK-NEXT: str z3, [sp, #11, mul vl]
-; CHECK-NEXT: whilelo p0.s, xzr, x9
; CHECK-NEXT: str z4, [sp, #10, mul vl]
-; CHECK-NEXT: add x9, x12, x11
+; CHECK-NEXT: add x8, x8, x9
; CHECK-NEXT: str z0, [sp, #9, mul vl]
-; CHECK-NEXT: addvl x12, sp, #5
; CHECK-NEXT: str z0, [sp, #13, mul vl]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x9]
-; CHECK-NEXT: addvl x9, sp, #13
-; CHECK-NEXT: add x9, x9, x11
-; CHECK-NEXT: add x11, x12, x10
-; CHECK-NEXT: st1w { z1.s }, p0, [x9]
-; CHECK-NEXT: cnth x9, all, mul #3
-; CHECK-NEXT: str z2, [x11, #1, mul vl]
-; CHECK-NEXT: add x11, x12, x9
-; CHECK-NEXT: addvl x12, sp, #1
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x8]
+; CHECK-NEXT: addvl x8, sp, #13
+; CHECK-NEXT: add x8, x8, x9
+; CHECK-NEXT: addvl x9, sp, #5
+; CHECK-NEXT: st1w { z1.s }, p0, [x8]
+; CHECK-NEXT: cnth x8, all, mul #3
; CHECK-NEXT: ldr z1, [sp, #14, mul vl]
+; CHECK-NEXT: str z2, [sp, #8, mul vl]
+; CHECK-NEXT: add x9, x9, x8
; CHECK-NEXT: str z3, [sp, #7, mul vl]
-; CHECK-NEXT: add x10, x12, x10
; CHECK-NEXT: str z4, [sp, #6, mul vl]
-; CHECK-NEXT: str z0, [sp, #5, mul vl]
; CHECK-NEXT: str z1, [sp, #16, mul vl]
; CHECK-NEXT: ldr z1, [sp, #13, mul vl]
+; CHECK-NEXT: str z0, [sp, #5, mul vl]
; CHECK-NEXT: str z1, [sp, #15, mul vl]
-; CHECK-NEXT: ld1w { z1.s }, p0/z, [x11]
-; CHECK-NEXT: addvl x11, sp, #15
-; CHECK-NEXT: add x9, x11, x9
-; CHECK-NEXT: st1w { z1.s }, p0, [x9]
-; CHECK-NEXT: cntw x9, all, mul #9
-; CHECK-NEXT: str z2, [x10, #1, mul vl]
-; CHECK-NEXT: add x9, x12, x9
-; CHECK-NEXT: mov x10, sp
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x9]
+; CHECK-NEXT: addvl x9, sp, #15
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: addvl x9, sp, #1
+; CHECK-NEXT: st1w { z1.s }, p0, [x8]
+; CHECK-NEXT: cntw x8, all, mul #9
; CHECK-NEXT: str z0, [sp, #1, mul vl]
; CHECK-NEXT: ldr z0, [sp, #17, mul vl]
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: str z2, [sp, #4, mul vl]
+; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: str z3, [sp, #3, mul vl]
; CHECK-NEXT: str z4, [sp, #2, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x9]
-; CHECK-NEXT: cntw x9
-; CHECK-NEXT: add x9, x10, x9
-; CHECK-NEXT: st1w { z0.s }, p0, [x9]
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8]
+; CHECK-NEXT: cntw x8
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: st1w { z0.s }, p0, [x8]
; CHECK-NEXT: ldr z0, [sp, #16, mul vl]
; CHECK-NEXT: ldr z1, [sp, #15, mul vl]
; CHECK-NEXT: ldr z2, [sp]
; CHECK-NEXT: str z0, [x0, #1, mul vl]
; CHECK-NEXT: str z1, [x0]
-; CHECK-NEXT: str z2, [x8, #1, mul vl]
+; CHECK-NEXT: str z2, [x0, #2, mul vl]
; CHECK-NEXT: addvl sp, sp, #19
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
@@ -198,122 +193,109 @@ define void @interleave6_v3fp32(ptr %dst, <vscale x 3 x float> %a, <vscale x 3
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xa8, 0x03, 0x1e, 0x22 // sp + 16 + 424 * VG
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: zip2 z26.s, z2.s, z5.s
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: rdvl x8, #4
-; CHECK-NEXT: zip2 z25.s, z1.s, z4.s
; CHECK-NEXT: zip1 z7.s, z2.s, z5.s
-; CHECK-NEXT: addvl x11, sp, #31
-; CHECK-NEXT: zip2 z24.s, z0.s, z3.s
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: zip2 z25.s, z1.s, z4.s
; CHECK-NEXT: zip1 z6.s, z1.s, z4.s
-; CHECK-NEXT: add x9, x11, x8
+; CHECK-NEXT: lsr x8, x8, #4
+; CHECK-NEXT: zip2 z24.s, z0.s, z3.s
; CHECK-NEXT: zip1 z5.s, z0.s, z3.s
-; CHECK-NEXT: rdvl x10, #2
-; CHECK-NEXT: mov w13, #3 // =0x3
+; CHECK-NEXT: mov w9, #3 // =0x3
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: mul x8, x8, x9
+; CHECK-NEXT: addvl x9, sp, #31
+; CHECK-NEXT: rdvl x10, #4
; CHECK-NEXT: ptrue p1.d
; CHECK-NEXT: st3w { z24.s - z26.s }, p0, [sp, #3, mul vl]
-; CHECK-NEXT: ldr z0, [sp, #5, mul vl]
-; CHECK-NEXT: st3w { z5.s - z7.s }, p0, [sp]
-; CHECK-NEXT: str z0, [x9, #1, mul vl]
-; CHECK-NEXT: add x9, x11, x10
-; CHECK-NEXT: ldr z1, [sp, #3, mul vl]
-; CHECK-NEXT: str z1, [x9, #1, mul vl]
-; CHECK-NEXT: rdvl x9, #1
-; CHECK-NEXT: lsr x12, x9, #4
+; CHECK-NEXT: ldr z1, [sp, #5, mul vl]
; CHECK-NEXT: ldr z2, [sp, #4, mul vl]
+; CHECK-NEXT: ldr z0, [sp, #3, mul vl]
+; CHECK-NEXT: st3w { z5.s - z7.s }, p0, [sp]
+; CHECK-NEXT: whilelo p0.s, xzr, x8
+; CHECK-NEXT: cntw x8, all, mul #3
; CHECK-NEXT: ldr z3, [sp, #2, mul vl]
; CHECK-NEXT: ldr z4, [sp, #1, mul vl]
; CHECK-NEXT: ldr z5, [sp]
-; CHECK-NEXT: mul x12, x12, x13
-; CHECK-NEXT: addvl x13, sp, #15
+; CHECK-NEXT: add x9, x9, x8
+; CHECK-NEXT: str z1, [sp, #36, mul vl]
; CHECK-NEXT: str z2, [sp, #35, mul vl]
+; CHECK-NEXT: str z0, [sp, #34, mul vl]
; CHECK-NEXT: str z3, [sp, #33, mul vl]
; CHECK-NEXT: str z4, [sp, #32, mul vl]
; CHECK-NEXT: str z5, [sp, #31, mul vl]
-; CHECK-NEXT: whilelo p0.s, xzr, x12
-; CHECK-NEXT: cntw x12, all, mul #3
; CHECK-NEXT: str z5, [sp, #39, mul vl]
-; CHECK-NEXT: add x11, x11, x12
-; CHECK-NEXT: ld1w { z6.s }, p0/z, [x11]
-; CHECK-NEXT: addvl x11, sp, #31
-; CHECK-NEXT: addvl x11, x11, #8
-; CHECK-NEXT: add x11, x11, x12
-; CHECK-NEXT: addvl x12, sp, #23
-; CHECK-NEXT: st1w { z6.s }, p0, [x11]
-; CHECK-NEXT: add x11, x12, x8
-; CHECK-NEXT: str z0, [x11, #1, mul vl]
-; CHECK-NEXT: add x11, x12, x10
-; CHECK-NEXT: str z1, [x11, #1, mul vl]
-; CHECK-NEXT: cnth x11, all, mul #3
+; CHECK-NEXT: ld1w { z6.s }, p0/z, [x9]
+; CHECK-NEXT: addvl x9, sp, #31
+; CHECK-NEXT: addvl x9, x9, #8
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: addvl x9, sp, #23
+; CHECK-NEXT: st1w { z6.s }, p0, [x8]
+; CHECK-NEXT: cnth x8, all, mul #3
; CHECK-NEXT: ldr z6, [sp, #40, mul vl]
+; CHECK-NEXT: str z1, [sp, #28, mul vl]
+; CHECK-NEXT: add x9, x9, x8
; CHECK-NEXT: str z2, [sp, #27, mul vl]
-; CHECK-NEXT: add x12, x12, x11
-; CHECK-NEXT: str z3, [sp, #25, mul vl]
-; CHECK-NEXT: str z4, [sp, #24, mul vl]
+; CHECK-NEXT: str z0, [sp, #26, mul vl]
; CHECK-NEXT: str z6, [sp, #42, mul vl]
; CHECK-NEXT: ldr z6, [sp, #39, mul vl]
+; CHECK-NEXT: str z3, [sp, #25, mul vl]
+; CHECK-NEXT: str z4, [sp, #24, mul vl]
; CHECK-NEXT: str z5, [sp, #23, mul vl]
; CHECK-NEXT: str z6, [sp, #41, mul vl]
-; CHECK-NEXT: ld1w { z6.s }, p0/z, [x12]
-; CHECK-NEXT: addvl x12, sp, #31
-; CHECK-NEXT: addvl x12, x12, #10
-; CHECK-NEXT: add x11, x12, x11
-; CHECK-NEXT: st1w { z6.s }, p0, [x11]
-; CHECK-NEXT: add x11, x13, x8
-; CHECK-NEXT: str z0, [x11, #1, mul vl]
-; CHECK-NEXT: add x11, x13, x10
-; CHECK-NEXT: str z1, [x11, #1, mul vl]
-; CHECK-NEXT: cntw x11, all, mul #9
+; CHECK-NEXT: ld1w { z6.s }, p0/z, [x9]
+; CHECK-NEXT: addvl x9, sp, #31
+; CHECK-NEXT: addvl x9, x9, #10
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: addvl x9, sp, #15
+; CHECK-NEXT: st1w { z6.s }, p0, [x8]
+; CHECK-NEXT: cntw x8, all, mul #9
; CHECK-NEXT: ldr z6, [sp, #43, mul vl]
+; CHECK-NEXT: str z1, [sp, #20, mul vl]
+; CHECK-NEXT: add x8, x9, x8
; CHECK-NEXT: str z2, [sp, #19, mul vl]
-; CHECK-NEXT: add x11, x13, x11
+; CHECK-NEXT: addvl x9, sp, #6
+; CHECK-NEXT: str z0, [sp, #18, mul vl]
; CHECK-NEXT: str z3, [sp, #17, mul vl]
-; CHECK-NEXT: addvl x13, sp, #6
; CHECK-NEXT: str z4, [sp, #16, mul vl]
; CHECK-NEXT: str z5, [sp, #15, mul vl]
; CHECK-NEXT: str z6, [sp, #6, mul vl]
-; CHECK-NEXT: ld1w { z6.s }, p0/z, [x11]
-; CHECK-NEXT: cntw x11
-; CHECK-NEXT: add x11, x13, x11
-; CHECK-NEXT: addvl x13, sp, #7
-; CHECK-NEXT: st1w { z6.s }, p0, [x11]
-; CHECK-NEXT: add x11, x13, x8
-; CHECK-NEXT: str z0, [x11, #1, mul vl]
-; CHECK-NEXT: add x11, x13, x10
-; CHECK-NEXT: str z1, [x11, #1, mul vl]
-; CHECK-NEXT: add x11, x12, x10
-; CHECK-NEXT: cntw x12, all, mul #15
-; CHECK-NEXT: ldr z0, [x11, #1, mul vl]
-; CHECK-NEXT: addvl x11, sp, #31
-; CHECK-NEXT: add x13, x13, x12
-; CHECK-NEXT: addvl x11, x11, #14
-; CHECK-NEXT: add x10, x11, x10
-; CHECK-NEXT: add x12, x11, x12
-; CHECK-NEXT: add x8, x11, x8
-; CHECK-NEXT: mov z0.s, p0/m, z1.s
-; CHECK-NEXT: add x11, x0, x9
-; CHECK-NEXT: str z0, [x10, #1, mul vl]
-; CHECK-NEXT: ldr z0, [sp, #42, mul vl]
+; CHECK-NEXT: ld1w { z6.s }, p0/z, [x8]
+; CHECK-NEXT: cntw x8
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: addvl x9, sp, #7
+; CHECK-NEXT: st1w { z6.s }, p0, [x8]
+; CHECK-NEXT: cntw x8, all, mul #15
+; CHECK-NEXT: str z1, [sp, #12, mul vl]
+; CHECK-NEXT: ldr z1, [sp, #42, mul vl]
+; CHECK-NEXT: add x9, x9, x8
+; CHECK-NEXT: str z0, [sp, #10, mul vl]
; CHECK-NEXT: str z2, [sp, #11, mul vl]
+; CHECK-NEXT: str z1, [sp, #46, mul vl]
+; CHECK-NEXT: ldr z1, [sp, #41, mul vl]
; CHECK-NEXT: str z3, [sp, #9, mul vl]
; CHECK-NEXT: str z4, [sp, #8, mul vl]
-; CHECK-NEXT: str z0, [sp, #46, mul vl]
-; CHECK-NEXT: ldr z0, [sp, #41, mul vl]
+; CHECK-NEXT: str z1, [sp, #45, mul vl]
+; CHECK-NEXT: ldr z1, [sp, #44, mul vl]
; CHECK-NEXT: str z5, [sp, #7, mul vl]
-; CHECK-NEXT: str z0, [sp, #45, mul vl]
+; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s
+; CHECK-NEXT: str z0, [sp, #48, mul vl]
; CHECK-NEXT: ldr z0, [sp, #6, mul vl]
; CHECK-NEXT: str z0, [sp, #47, mul vl]
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x13]
-; CHECK-NEXT: st1w { z0.s }, p0, [x12]
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x9]
+; CHECK-NEXT: addvl x9, sp, #31
+; CHECK-NEXT: addvl x9, x9, #14
+; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: st1w { z0.s }, p0, [x8]
+; CHECK-NEXT: add x8, x9, x10
; CHECK-NEXT: ld1w { z0.d }, p1/z, [x8]
-; CHECK-NEXT: ldr z1, [x10, #1, mul vl]
+; CHECK-NEXT: ldr z1, [sp, #48, mul vl]
; CHECK-NEXT: ldr z2, [sp, #47, mul vl]
; CHECK-NEXT: ldr z3, [sp, #45, mul vl]
; CHECK-NEXT: ldr z4, [sp, #46, mul vl]
-; CHECK-NEXT: add x8, x11, x9
-; CHECK-NEXT: add x9, x8, x9
-; CHECK-NEXT: st1w { z0.d }, p1, [x9, #2, mul vl]
-; CHECK-NEXT: str z1, [x8, #1, mul vl]
-; CHECK-NEXT: str z2, [x11, #1, mul vl]
+; CHECK-NEXT: add x8, x0, x10
+; CHECK-NEXT: st1w { z0.d }, p1, [x8]
+; CHECK-NEXT: str z1, [x0, #3, mul vl]
+; CHECK-NEXT: str z2, [x0, #2, mul vl]
; CHECK-NEXT: str z4, [x0, #1, mul vl]
; CHECK-NEXT: str z3, [x0]
; CHECK-NEXT: addvl sp, sp, #31
@@ -339,191 +321,164 @@ define void @interleave8_v3fp32(ptr %dst, <vscale x 3 x float> %a, <vscale x 3
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: zip2 z24.s, z3.s, z7.s
; CHECK-NEXT: zip2 z25.s, z1.s, z5.s
-; CHECK-NEXT: addvl x12, sp, #31
+; CHECK-NEXT: rdvl x8, #1
; CHECK-NEXT: zip2 z26.s, z2.s, z6.s
; CHECK-NEXT: zip2 z27.s, z0.s, z4.s
-; CHECK-NEXT: rdvl x9, #4
-; CHECK-NEXT: addvl x12, x12, #13
+; CHECK-NEXT: lsr x8, x8, #4
+; CHECK-NEXT: zip1 z3.s, z3.s, z7.s
+; CHECK-NEXT: zip1 z5.s, z1.s, z5.s
+; CHECK-NEXT: mov w9, #3 // =0x3
+; CHECK-NEXT: zip1 z6.s, z2.s, z6.s
; CHECK-NEXT: zip1 z4.s, z0.s, z4.s
-; CHECK-NEXT: rdvl x10, #2
+; CHECK-NEXT: mul x9, x8, x9
; CHECK-NEXT: zip2 z28.s, z25.s, z24.s
-; CHECK-NEXT: zip1 z24.s, z25.s, z24.s
-; CHECK-NEXT: add x13, x12, x9
; CHECK-NEXT: zip2 z29.s, z27.s, z26.s
-; CHECK-NEXT: zip1 z25.s, z27.s, z26.s
-; CHECK-NEXT: zip1 z5.s, z1.s, z5.s
-; CHECK-NEXT: zip1 z6.s, z2.s, z6.s
-; CHECK-NEXT: add x8, x13, x10
-; CHECK-NEXT: zip1 z7.s, z3.s, z7.s
-; CHECK-NEXT: add x14, x12, x10
+; CHECK-NEXT: addvl x10, sp, #31
+; CHECK-NEXT: zip1 z7.s, z25.s, z24.s
+; CHECK-NEXT: zip1 z24.s, z27.s, z26.s
+; CHECK-NEXT: addvl x10, x10, #13
+; CHECK-NEXT: zip2 z25.s, z5.s, z3.s
+; CHECK-NEXT: zip2 z26.s, z4.s, z6.s
+; CHECK-NEXT: addvl x11, sp, #20
+; CHECK-NEXT: zip1 z27.s, z5.s, z3.s
+; CHECK-NEXT: zip1 z4.s, z4.s, z6.s
; CHECK-NEXT: zip2 z0.s, z29.s, z28.s
; CHECK-NEXT: zip1 z1.s, z29.s, z28.s
-; CHECK-NEXT: zip2 z2.s, z25.s, z24.s
-; CHECK-NEXT: zip2 z26.s, z5.s, z7.s
-; CHECK-NEXT: zip2 z27.s, z4.s, z6.s
-; CHECK-NEXT: zip1 z28.s, z5.s, z7.s
-; CHECK-NEXT: zip1 z4.s, z4.s, z6.s
-; CHECK-NEXT: zip1 z6.s, z25.s, z24.s
-; CHECK-NEXT: str z0, [x8, #1, mul vl]
-; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: lsr x11, x8, #4
-; CHECK-NEXT: str z1, [x13, #2, mul vl]
-; CHECK-NEXT: zip2 z3.s, z27.s, z26.s
-; CHECK-NEXT: str z2, [x13, #1, mul vl]
-; CHECK-NEXT: mov w13, #3 // =0x3
-; CHECK-NEXT: zip1 z7.s, z27.s, z26.s
-; CHECK-NEXT: mul x13, x11, x13
-; CHECK-NEXT: zip2 z5.s, z4.s, z28.s
-; CHECK-NEXT: zip1 z4.s, z4.s, z28.s
-; CHECK-NEXT: str z3, [x14, #1, mul vl]
-; CHECK-NEXT: addvl x14, sp, #31
-; CHECK-NEXT: str z6, [sp, #48, mul vl]
-; CHECK-NEXT: addvl x14, x14, #5
+; CHECK-NEXT: zip2 z2.s, z24.s, z7.s
+; CHECK-NEXT: zip1 z3.s, z24.s, z7.s
+; CHECK-NEXT: zip2 z5.s, z26.s, z25.s
+; CHECK-NEXT: zip1 z7.s, z26.s, z25.s
+; CHECK-NEXT: whilelo p0.s, xzr, x9
+; CHECK-NEXT: cntw x9, all, mul #3
+; CHECK-NEXT: str z0, [sp, #51, mul vl]
+; CHECK-NEXT: zip2 z6.s, z4.s, z27.s
+; CHECK-NEXT: str z1, [sp, #50, mul vl]
+; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: zip1 z4.s, z4.s, z27.s
+; CHECK-NEXT: str z2, [sp, #49, mul vl]
+; CHECK-NEXT: str z3, [sp, #48, mul vl]
+; CHECK-NEXT: str z5, [sp, #47, mul vl]
; CHECK-NEXT: str z7, [sp, #46, mul vl]
-; CHECK-NEXT: str z5, [sp, #45, mul vl]
-; CHECK-NEXT: whilelo p0.s, xzr, x13
-; CHECK-NEXT: cntw x13, all, mul #3
+; CHECK-NEXT: str z6, [sp, #45, mul vl]
; CHECK-NEXT: str z4, [sp, #44, mul vl]
; CHECK-NEXT: str z4, [sp, #52, mul vl]
-; CHECK-NEXT: add x12, x12, x13
-; CHECK-NEXT: ld1w { z24.s }, p0/z, [x12]
-; CHECK-NEXT: addvl x12, sp, #31
-; CHECK-NEXT: addvl x12, x12, #21
-; CHECK-NEXT: add x12, x12, x13
-; CHECK-NEXT: add x13, x14, x9
-; CHECK-NEXT: st1w { z24.s }, p0, [x12]
-; CHECK-NEXT: add x12, x13, x10
-; CHECK-NEXT: str z0, [x12, #1, mul vl]
-; CHECK-NEXT: add x12, x14, x10
-; CHECK-NEXT: str z1, [x13, #2, mul vl]
-; CHECK-NEXT: str z2, [x13, #1, mul vl]
-; CHECK-NEXT: str z3, [x12, #1, mul vl]
-; CHECK-NEXT: cnth x12, all, mul #3
+; CHECK-NEXT: ld1w { z24.s }, p0/z, [x10]
+; CHECK-NEXT: addvl x10, sp, #31
+; CHECK-NEXT: addvl x10, x10, #21
+; CHECK-NEXT: add x9, x10, x9
+; CHECK-NEXT: addvl x10, sp, #31
+; CHECK-NEXT: st1w { z24.s }, p0, [x9]
+; CHECK-NEXT: cnth x9, all, mul #3
+; CHECK-NEXT: addvl x10, x10, #5
; CHECK-NEXT: ldr z24, [sp, #53, mul vl]
-; CHECK-NEXT: str z6, [sp, #40, mul vl]
-; CHECK-NEXT: add x13, x14, x12
-; CHECK-NEXT: str z7, [sp, #38, mul vl]
-; CHECK-NEXT: addvl x14, sp, #28
-; CHECK-NEXT: str z5, [sp, #37, mul vl]
-; CHECK-NEXT: add x15, x14, x9
+; CHECK-NEXT: str z0, [sp, #43, mul vl]
+; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: str z1, [sp, #42, mul vl]
+; CHECK-NEXT: str z2, [sp, #41, mul vl]
; CHECK-NEXT: str z24, [sp, #55, mul vl]
; CHECK-NEXT: ldr z24, [sp, #52, mul vl]
+; CHECK-NEXT: str z3, [sp, #40, mul vl]
+; CHECK-NEXT: str z5, [sp, #39, mul vl]
+; CHECK-NEXT: str z7, [sp, #38, mul vl]
+; CHECK-NEXT: str z6, [sp, #37, mul vl]
; CHECK-NEXT: str z4, [sp, #36, mul vl]
; CHECK-NEXT: str z24, [sp, #54, mul vl]
-; CHECK-NEXT: ld1w { z24.s }, p0/z, [x13]
-; CHECK-NEXT: addvl x13, sp, #31
-; CHECK-NEXT: addvl x13, x13, #23
-; CHECK-NEXT: add x12, x13, x12
-; CHECK-NEXT: add x13, x13, x10
-; CHECK-NEXT: st1w { z24.s }, p0, [x12]
-; CHECK-NEXT: add x12, x15, x10
-; CHECK-NEXT: str z0, [x12, #1, mul vl]
-; CHECK-NEXT: add x12, x14, x10
-; CHECK-NEXT: str z1, [x15, #2, mul vl]
-; CHECK-NEXT: str z2, [x15, #1, mul vl]
-; CHECK-NEXT: addvl x15, sp, #20
-; CHECK-NEXT: str z3, [x12, #1, mul vl]
-; CHECK-NEXT: cntw x12, all, mul #9
-; CHECK-NEXT: add x16, x15, x9
+; CHECK-NEXT: ld1w { z24.s }, p0/z, [x10]
+; CHECK-NEXT: addvl x10, sp, #31
+; CHECK-NEXT: addvl x10, x10, #23
+; CHECK-NEXT: add x9, x10, x9
+; CHECK-NEXT: addvl x10, sp, #28
+; CHECK-NEXT: st1w { z24.s }, p0, [x9]
+; CHECK-NEXT: cntw x9, all, mul #9
; CHECK-NEXT: ldr z24, [sp, #56, mul vl]
-; CHECK-NEXT: str z6, [sp, #32, mul vl]
-; CHECK-NEXT: add x12, x14, x12
+; CHECK-NEXT: str z0, [sp, #35, mul vl]
+; CHECK-NEXT: add x9, x10, x9
+; CHECK-NEXT: str z1, [sp, #34, mul vl]
+; CHECK-NEXT: addvl x10, sp, #3
+; CHECK-NEXT: str z2, [sp, #33, mul vl]
+; CHECK-NEXT: str z3, [sp, #32, mul vl]
+; CHECK-NEXT: str z5, [sp, #31, mul vl]
; CHECK-NEXT: str z7, [sp, #30, mul vl]
-; CHECK-NEXT: addvl x14, sp, #3
-; CHECK-NEXT: add x17, x16, x10
-; CHECK-NEXT: str z5, [sp, #29, mul vl]
+; CHECK-NEXT: str z6, [sp, #29, mul vl]
; CHECK-NEXT: str z4, [sp, #28, mul vl]
; CHECK-NEXT: str z24, [sp, #3, mul vl]
-; CHECK-NEXT: ld1w { z24.s }, p0/z, [x12]
-; CHECK-NEXT: cntw x12
-; CHECK-NEXT: add x14, x14, x12
-; CHECK-NEXT: st1w { z24.s }, p0, [x14]
-; CHECK-NEXT: add x14, x15, x10
-; CHECK-NEXT: str z0, [x17, #1, mul vl]
-; CHECK-NEXT: str z1, [x16, #2, mul vl]
-; CHECK-NEXT: str z2, [x16, #1, mul vl]
-; CHECK-NEXT: cntw x16, all, mul #15
-; CHECK-NEXT: str z3, [x14, #1, mul vl]
-; CHECK-NEXT: addvl x14, sp, #31
-; CHECK-NEXT: add x15, x15, x16
-; CHECK-NEXT: ldr z24, [x13, #1, mul vl]
-; CHECK-NEXT: addvl x14, x14, #27
-; CHECK-NEXT: add x13, x14, x10
-; CHECK-NEXT: add x16, x14, x16
-; CHECK-NEXT: add x14, x14, x9
-; CHECK-NEXT: mov z24.s, p0/m, z3.s
-; CHECK-NEXT: str z24, [x13, #1, mul vl]
+; CHECK-NEXT: ld1w { z24.s }, p0/z, [x9]
+; CHECK-NEXT: cntw x9
+; CHECK-NEXT: add x10, x10, x9
+; CHECK-NEXT: st1w { z24.s }, p0, [x10]
+; CHECK-NEXT: cntw x10, all, mul #15
; CHECK-NEXT: ldr z24, [sp, #55, mul vl]
-; CHECK-NEXT: str z6, [sp, #24, mul vl]
-; CHECK-NEXT: str z7, [sp, #22, mul vl]
-; CHECK-NEXT: str z5, [sp, #21, mul vl]
+; CHECK-NEXT: str z0, [sp, #27, mul vl]
+; CHECK-NEXT: add x11, x11, x10
+; CHECK-NEXT: str z1, [sp, #26, mul vl]
+; CHECK-NEXT: str z2, [sp, #25, mul vl]
; CHECK-NEXT: str z24, [sp, #59, mul vl]
; CHECK-NEXT: ldr z24, [sp, #54, mul vl]
-; CHECK-NEXT: str z4, [sp, #20, mul vl]
+; CHECK-NEXT: str z3, [sp, #24, mul vl]
+; CHECK-NEXT: str z5, [sp, #23, mul vl]
; CHECK-NEXT: str z24, [sp, #58, mul vl]
+; CHECK-NEXT: ldr z24, [sp, #57, mul vl]
+; CHECK-NEXT: str z7, [sp, #22, mul vl]
+; CHECK-NEXT: str z6, [sp, #21, mul vl]
+; CHECK-NEXT: mov z24.s, p0/m, z5.s
+; CHECK-NEXT: str z4, [sp, #20, mul vl]
+; CHECK-NEXT: str z24, [sp, #61, mul vl]
; CHECK-NEXT: ldr z24, [sp, #3, mul vl]
; CHECK-NEXT: str z24, [sp, #60, mul vl]
-; CHECK-NEXT: ld1w { z24.s }, p0/z, [x15]
-; CHECK-NEXT: addvl x15, sp, #12
-; CHECK-NEXT: add x17, x15, x9
-; CHECK-NEXT: st1w { z24.s }, p0, [x16]
-; CHECK-NEXT: add x16, x17, x10
-; CHECK-NEXT: str z0, [x16, #1, mul vl]
-; CHECK-NEXT: add x16, x15, x10
-; CHECK-NEXT: str z1, [x17, #2, mul vl]
-; CHECK-NEXT: str z2, [x17, #1, mul vl]
-; CHECK-NEXT: str z3, [x16, #1, mul vl]
-; CHECK-NEXT: addvl x16, sp, #4
-; CHECK-NEXT: str z6, [sp, #16, mul vl]
-; CHECK-NEXT: add x9, x16, x9
-; CHECK-NEXT: str z7, [sp, #14, mul vl]
-; CHECK-NEXT: str z5, [sp, #13, mul vl]
-; CHECK-NEXT: str z4, [sp, #12, mul vl]
-; CHECK-NEXT: ldr z24, [x14, #1, mul vl]
-; CHECK-NEXT: cnth x14, all, mul #9
-; CHECK-NEXT: add x14, x15, x14
-; CHECK-NEXT: addvl x15, sp, #1
+; CHECK-NEXT: ld1w { z24.s }, p0/z, [x11]
+; CHECK-NEXT: addvl x11, sp, #31
+; CHECK-NEXT: addvl x11, x11, #27
+; CHECK-NEXT: add x10, x11, x10
+; CHECK-NEXT: addvl x11, sp, #12
+; CHECK-NEXT: st1w { z24.s }, p0, [x10]
+; CHECK-NEXT: cnth x10, all, mul #9
+; CHECK-NEXT: ldr z24, [sp, #63, mul vl]
+; CHECK-NEXT: str z0, [sp, #19, mul vl]
+; CHECK-NEXT: add x10, x11, x10
+; CHECK-NEXT: str z1, [sp, #18, mul vl]
+; CHECK-NEXT: addvl x11, sp, #1
+; CHECK-NEXT: str z2, [sp, #17, mul vl]
; CHECK-NEXT: str z24, [sp, #2, mul vl]
; CHECK-NEXT: ldr z24, [sp, #62, mul vl]
+; CHECK-NEXT: str z3, [sp, #16, mul vl]
+; CHECK-NEXT: str z5, [sp, #15, mul vl]
+; CHECK-NEXT: str z7, [sp, #14, mul vl]
+; CHECK-NEXT: str z6, [sp, #13, mul vl]
+; CHECK-NEXT: str z4, [sp, #12, mul vl]
; CHECK-NEXT: str z24, [sp, #1, mul vl]
-; CHECK-NEXT: ld1w { z24.s }, p0/z, [x14]
-; CHECK-NEXT: cnth x14
-; CHECK-NEXT: add x14, x15, x14
-; CHECK-NEXT: add x15, x9, x10
-; CHECK-NEXT: add x10, x16, x10
-; CHECK-NEXT: st1w { z24.s }, p0, [x14]
-; CHECK-NEXT: str z0, [x15, #1, mul vl]
-; CHECK-NEXT: str z1, [x9, #2, mul vl]
-; CHECK-NEXT: str z2, [x9, #1, mul vl]
-; CHECK-NEXT: mov w9, #84 // =0x54
-; CHECK-NEXT: str z3, [x10, #1, mul vl]
-; CHECK-NEXT: madd x9, x11, x9, x16
+; CHECK-NEXT: ld1w { z24.s }, p0/z, [x10]
+; CHECK-NEXT: cnth x10
+; CHECK-NEXT: add x10, x11, x10
+; CHECK-NEXT: addvl x11, sp, #4
+; CHECK-NEXT: st1w { z24.s }, p0, [x10]
+; CHECK-NEXT: mov w10, #84 // =0x54
+; CHECK-NEXT: madd x8, x8, x10, x11
+; CHECK-NEXT: str z0, [sp, #11, mul vl]
; CHECK-NEXT: ldr z0, [sp, #2, mul vl]
-; CHECK-NEXT: str z6, [sp, #8, mul vl]
+; CHECK-NEXT: str z1, [sp, #10, mul vl]
+; CHECK-NEXT: str z2, [sp, #9, mul vl]
+; CHECK-NEXT: str z3, [sp, #8, mul vl]
+; CHECK-NEXT: str z5, [sp, #7, mul vl]
; CHECK-NEXT: str z7, [sp, #6, mul vl]
-; CHECK-NEXT: str z5, [sp, #5, mul vl]
+; CHECK-NEXT: str z6, [sp, #5, mul vl]
; CHECK-NEXT: str z4, [sp, #4, mul vl]
; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: ld1w { z0.s }, p0/z, [x9]
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: add x9, x9, x12
-; CHECK-NEXT: st1w { z0.s }, p0, [x9]
-; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: ldr z0, [x13, #1, mul vl]
+; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8]
+; CHECK-NEXT: mov x8, sp
+; CHECK-NEXT: add x8, x8, x9
+; CHECK-NEXT: st1w { z0.s }, p0, [x8]
+; CHECK-NEXT: ldr z0, [sp, #61, mul vl]
; CHECK-NEXT: ldr z1, [sp, #60, mul vl]
-; CHECK-NEXT: add x10, x9, x8
; CHECK-NEXT: ldr z2, [sp]
; CHECK-NEXT: ldr z3, [sp, #59, mul vl]
; CHECK-NEXT: ldr z4, [sp, #1, mul vl]
; CHECK-NEXT: ldr z5, [sp, #58, mul vl]
-; CHECK-NEXT: str z0, [x10, #1, mul vl]
-; CHECK-NEXT: str z1, [x9, #1, mul vl]
-; CHECK-NEXT: add x9, x10, x8
-; CHECK-NEXT: add x8, x9, x8
+; CHECK-NEXT: str z0, [x0, #3, mul vl]
+; CHECK-NEXT: str z1, [x0, #2, mul vl]
; CHECK-NEXT: str z3, [x0, #1, mul vl]
; CHECK-NEXT: str z5, [x0]
-; CHECK-NEXT: str z4, [x9, #1, mul vl]
-; CHECK-NEXT: str z2, [x8, #1, mul vl]
+; CHECK-NEXT: str z4, [x0, #4, mul vl]
+; CHECK-NEXT: str z2, [x0, #5, mul vl]
; CHECK-NEXT: addvl sp, sp, #31
; CHECK-NEXT: addvl sp, sp, #31
; CHECK-NEXT: addvl sp, sp, #4
diff --git a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
index 1556c5f6ec8fc..b066abd0a19d5 100644
--- a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
@@ -1963,28 +1963,24 @@ define void @zext_v16i8_to_v16i32_in_loop_scalable_vectors(ptr %src, ptr %dst) {
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: ptrue p0.s
; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: cnth x9
-; CHECK-NEXT: rdvl x10, #2
; CHECK-NEXT: LBB19_1: ; %loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add x11, x0, x8
; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, x8]
-; CHECK-NEXT: ld1b { z1.s }, p0/z, [x11, #2, mul vl]
-; CHECK-NEXT: ld1b { z2.s }, p0/z, [x11, #1, mul vl]
-; CHECK-NEXT: add x11, x11, x9
-; CHECK-NEXT: ld1b { z3.s }, p0/z, [x11, #1, mul vl]
-; CHECK-NEXT: add x11, x1, x8, lsl #2
+; CHECK-NEXT: add x9, x0, x8
+; CHECK-NEXT: ld1b { z1.s }, p0/z, [x9, #2, mul vl]
+; CHECK-NEXT: ld1b { z2.s }, p0/z, [x9, #3, mul vl]
+; CHECK-NEXT: ld1b { z3.s }, p0/z, [x9, #1, mul vl]
+; CHECK-NEXT: add x9, x1, x8, lsl #2
; CHECK-NEXT: add z0.s, z0.s, z0.s
; CHECK-NEXT: add z1.s, z1.s, z1.s
; CHECK-NEXT: add z2.s, z2.s, z2.s
; CHECK-NEXT: add z3.s, z3.s, z3.s
; CHECK-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]
; CHECK-NEXT: add x8, x8, #16
-; CHECK-NEXT: str z1, [x11, #2, mul vl]
; CHECK-NEXT: cmp x8, #128
-; CHECK-NEXT: str z2, [x11, #1, mul vl]
-; CHECK-NEXT: add x11, x11, x10
-; CHECK-NEXT: str z3, [x11, #1, mul vl]
+; CHECK-NEXT: str z1, [x9, #2, mul vl]
+; CHECK-NEXT: str z2, [x9, #3, mul vl]
+; CHECK-NEXT: str z3, [x9, #1, mul vl]
; CHECK-NEXT: b.ne LBB19_1
; CHECK-NEXT: ; %bb.2: ; %exit
; CHECK-NEXT: ret
@@ -1993,28 +1989,24 @@ define void @zext_v16i8_to_v16i32_in_loop_scalable_vectors(ptr %src, ptr %dst) {
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: ptrue p0.s
; CHECK-BE-NEXT: mov x8, xzr
-; CHECK-BE-NEXT: cnth x9
-; CHECK-BE-NEXT: rdvl x10, #2
; CHECK-BE-NEXT: .LBB19_1: // %loop
; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT: add x11, x0, x8
; CHECK-BE-NEXT: ld1b { z0.s }, p0/z, [x0, x8]
-; CHECK-BE-NEXT: ld1b { z1.s }, p0/z, [x11, #2, mul vl]
-; CHECK-BE-NEXT: ld1b { z2.s }, p0/z, [x11, #1, mul vl]
-; CHECK-BE-NEXT: add x11, x11, x9
-; CHECK-BE-NEXT: ld1b { z3.s }, p0/z, [x11, #1, mul vl]
-; CHECK-BE-NEXT: add x11, x1, x8, lsl #2
+; CHECK-BE-NEXT: add x9, x0, x8
+; CHECK-BE-NEXT: ld1b { z1.s }, p0/z, [x9, #2, mul vl]
+; CHECK-BE-NEXT: ld1b { z2.s }, p0/z, [x9, #3, mul vl]
+; CHECK-BE-NEXT: ld1b { z3.s }, p0/z, [x9, #1, mul vl]
+; CHECK-BE-NEXT: add x9, x1, x8, lsl #2
; CHECK-BE-NEXT: add z0.s, z0.s, z0.s
; CHECK-BE-NEXT: add z1.s, z1.s, z1.s
; CHECK-BE-NEXT: add z2.s, z2.s, z2.s
; CHECK-BE-NEXT: add z3.s, z3.s, z3.s
; CHECK-BE-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]
; CHECK-BE-NEXT: add x8, x8, #16
-; CHECK-BE-NEXT: st1w { z1.s }, p0, [x11, #2, mul vl]
; CHECK-BE-NEXT: cmp x8, #128
-; CHECK-BE-NEXT: st1w { z2.s }, p0, [x11, #1, mul vl]
-; CHECK-BE-NEXT: add x11, x11, x10
-; CHECK-BE-NEXT: st1w { z3.s }, p0, [x11, #1, mul vl]
+; CHECK-BE-NEXT: st1w { z1.s }, p0, [x9, #2, mul vl]
+; CHECK-BE-NEXT: st1w { z2.s }, p0, [x9, #3, mul vl]
+; CHECK-BE-NEXT: st1w { z3.s }, p0, [x9, #1, mul vl]
; CHECK-BE-NEXT: b.ne .LBB19_1
; CHECK-BE-NEXT: // %bb.2: // %exit
; CHECK-BE-NEXT: ret
More information about the llvm-commits
mailing list