[llvm] [LLVM][CodeGen][SVE] Make use of predicate load/store "mul vl" addressing mode. (PR #206997)
Paul Walker via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 1 07:30:59 PDT 2026
https://github.com/paulwalker-arm created https://github.com/llvm/llvm-project/pull/206997
None
>From bb43c46a06d4c103487cc0ad96fce16a51d76a84 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker at arm.com>
Date: Wed, 1 Jul 2026 12:03:20 +0100
Subject: [PATCH 1/2] Add tests for predicate loads and stores.
---
llvm/test/CodeGen/AArch64/sve-pred-ldst.ll | 225 +++++++++++++++++++++
1 file changed, 225 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve-pred-ldst.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-pred-ldst.ll b/llvm/test/CodeGen/AArch64/sve-pred-ldst.ll
new file mode 100644
index 0000000000000..abc56087dfa1b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-pred-ldst.ll
@@ -0,0 +1,225 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,SVE
+; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,SME
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define <vscale x 16 x i1> @load_no_offset(ptr %base) {
+; CHECK-LABEL: load_no_offset:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr p0, [x0]
+; CHECK-NEXT: ret
+ %val = load <vscale x 16 x i1>, ptr %base, align 2
+ ret <vscale x 16 x i1> %val
+}
+
+define void @store_no_offset(ptr %base, <vscale x 16 x i1> %val) {
+; CHECK-LABEL: store_no_offset:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str p0, [x0]
+; CHECK-NEXT: ret
+ store <vscale x 16 x i1> %val, ptr %base, align 2
+ ret void
+}
+
+define <vscale x 16 x i1> @load_offset(ptr %base) {
+; SVE-LABEL: load_offset:
+; SVE: // %bb.0:
+; SVE-NEXT: cntd x8, all, mul #7
+; SVE-NEXT: add x8, x0, x8
+; SVE-NEXT: ldr p0, [x8]
+; SVE-NEXT: ret
+;
+; SME-LABEL: load_offset:
+; SME: // %bb.0:
+; SME-NEXT: incd x0, all, mul #7
+; SME-NEXT: ldr p0, [x0]
+; SME-NEXT: ret
+ %addr = getelementptr <vscale x 16 x i1>, ptr %base, i64 7
+ %val = load <vscale x 16 x i1>, ptr %addr, align 2
+ ret <vscale x 16 x i1> %val
+}
+
+define void @store_offset(ptr %base, <vscale x 16 x i1> %val) {
+; SVE-LABEL: store_offset:
+; SVE: // %bb.0:
+; SVE-NEXT: cntd x8, all, mul #7
+; SVE-NEXT: sub x8, x0, x8
+; SVE-NEXT: str p0, [x8]
+; SVE-NEXT: ret
+;
+; SME-LABEL: store_offset:
+; SME: // %bb.0:
+; SME-NEXT: decd x0, all, mul #7
+; SME-NEXT: str p0, [x0]
+; SME-NEXT: ret
+ %addr = getelementptr <vscale x 16 x i1>, ptr %base, i64 -7
+ store <vscale x 16 x i1> %val, ptr %addr, align 2
+ ret void
+}
+
+define <vscale x 16 x i1> @load_offset_min(ptr %base) {
+; SVE-LABEL: load_offset_min:
+; SVE: // %bb.0:
+; SVE-NEXT: rdvl x8, #-32
+; SVE-NEXT: add x8, x0, x8
+; SVE-NEXT: ldr p0, [x8]
+; SVE-NEXT: ret
+;
+; SME-LABEL: load_offset_min:
+; SME: // %bb.0:
+; SME-NEXT: addvl x8, x0, #-32
+; SME-NEXT: ldr p0, [x8]
+; SME-NEXT: ret
+ %addr = getelementptr <vscale x 16 x i1>, ptr %base, i64 -256
+ %val = load <vscale x 16 x i1>, ptr %addr, align 2
+ ret <vscale x 16 x i1> %val
+}
+
+define void @store_offset_max(ptr %base, <vscale x 16 x i1> %val) {
+; CHECK-LABEL: store_offset_max:
+; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: mov w9, #510 // =0x1fe
+; CHECK-NEXT: lsr x8, x8, #4
+; CHECK-NEXT: madd x8, x8, x9, x0
+; CHECK-NEXT: str p0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr <vscale x 16 x i1>, ptr %base, i64 255
+ store <vscale x 16 x i1> %val, ptr %addr, align 2
+ ret void
+}
+
+define <vscale x 16 x i1> @load_offset_too_low(ptr %base) {
+; CHECK-LABEL: load_offset_too_low:
+; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: mov x9, #-514 // =0xfffffffffffffdfe
+; CHECK-NEXT: lsr x8, x8, #4
+; CHECK-NEXT: madd x8, x8, x9, x0
+; CHECK-NEXT: ldr p0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr <vscale x 16 x i1>, ptr %base, i64 -257
+ %val = load <vscale x 16 x i1>, ptr %addr, align 2
+ ret <vscale x 16 x i1> %val
+}
+
+define void @store_offset_too_high(ptr %base, <vscale x 16 x i1> %val) {
+; CHECK-LABEL: store_offset_too_high:
+; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: mov w9, #512 // =0x200
+; CHECK-NEXT: lsr x8, x8, #4
+; CHECK-NEXT: madd x8, x8, x9, x0
+; CHECK-NEXT: str p0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr <vscale x 16 x i1>, ptr %base, i64 256
+ store <vscale x 16 x i1> %val, ptr %addr, align 2
+ ret void
+}
+
+define <vscale x 16 x i1> @load_fixed_offset_unknown_vscale(ptr %base) {
+; CHECK-LABEL: load_fixed_offset_unknown_vscale:
+; CHECK: // %bb.0:
+; CHECK-NEXT: add x8, x0, #2
+; CHECK-NEXT: ldr p0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %base, i64 2
+ %val = load <vscale x 16 x i1>, ptr %addr, align 2
+ ret <vscale x 16 x i1> %val
+}
+
+define void @store_fixed_offset_unknown_vscale(ptr %base, <vscale x 16 x i1> %val) {
+; CHECK-LABEL: store_fixed_offset_unknown_vscale:
+; CHECK: // %bb.0:
+; CHECK-NEXT: add x8, x0, #2
+; CHECK-NEXT: str p0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %base, i64 2
+ store <vscale x 16 x i1> %val, ptr %addr, align 2
+ ret void
+}
+
+; 1 * sizeof(<vscale x 16 x i1>) == 2
+define <vscale x 16 x i1> @load_fixed_offset_vscale_one(ptr %base) vscale_range(1,1) {
+; CHECK-LABEL: load_fixed_offset_vscale_one:
+; CHECK: // %bb.0:
+; CHECK-NEXT: add x8, x0, #2
+; CHECK-NEXT: ldr p0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %base, i64 2
+ %val = load <vscale x 16 x i1>, ptr %addr, align 2
+ ret <vscale x 16 x i1> %val
+}
+
+; -2 * sizeof(<vscale x 16 x i1>) == -8
+define void @store_fixed_offset_vscale_two(ptr %base, <vscale x 16 x i1> %val) vscale_range(2,2) {
+; CHECK-LABEL: store_fixed_offset_vscale_two:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub x8, x0, #8
+; CHECK-NEXT: str p0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %base, i64 -8
+ store <vscale x 16 x i1> %val, ptr %addr, align 2
+ ret void
+}
+
+; 255 * sizeof(<vscale x 16 x i1>) == -1020
+define <vscale x 16 x i1> @load_fixed_offset_max_vscale_two(ptr %base) vscale_range(2,2) {
+; CHECK-LABEL: load_fixed_offset_max_vscale_two:
+; CHECK: // %bb.0:
+; CHECK-NEXT: add x8, x0, #1020
+; CHECK-NEXT: ldr p0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %base, i64 1020
+ %val = load <vscale x 16 x i1>, ptr %addr, align 2
+ ret <vscale x 16 x i1> %val
+}
+
+; -256 * sizeof(<vscale x 16 x i1>) == -512
+define void @store_fixed_offset_min_vscale_one(ptr %base, <vscale x 16 x i1> %val) vscale_range(1,1) {
+; CHECK-LABEL: store_fixed_offset_min_vscale_one:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub x8, x0, #512
+; CHECK-NEXT: str p0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %base, i64 -512
+ store <vscale x 16 x i1> %val, ptr %addr, align 2
+ ret void
+}
+
+; Offsets must be a multiple of sizeof(<vscale x 16 x i1>)
+define void @store_fixed_no_divisible_offset_vscale_one(ptr %base, <vscale x 16 x i1> %val) vscale_range(1,1) {
+; CHECK-LABEL: store_fixed_no_divisible_offset_vscale_one:
+; CHECK: // %bb.0:
+; CHECK-NEXT: add x8, x0, #3
+; CHECK-NEXT: str p0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %base, i64 3
+ store <vscale x 16 x i1> %val, ptr %addr, align 2
+ ret void
+}
+
+; -257 * sizeof(<vscale x 16 x i1>) == -514
+define <vscale x 16 x i1> @load_fixed_offset_too_low_vscale_one(ptr %base) vscale_range(1,1) {
+; CHECK-LABEL: load_fixed_offset_too_low_vscale_one:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub x8, x0, #514
+; CHECK-NEXT: ldr p0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %base, i64 -514
+ %val = load <vscale x 16 x i1>, ptr %addr, align 2
+ ret <vscale x 16 x i1> %val
+}
+
+; 256 * sizeof(<vscale x 16 x i1>) == 1024
+define void @store_fixed_offset_too_high_vscale_two(ptr %base, <vscale x 16 x i1> %val) vscale_range(2,2) {
+; CHECK-LABEL: store_fixed_offset_too_high_vscale_two:
+; CHECK: // %bb.0:
+; CHECK-NEXT: add x8, x0, #1024
+; CHECK-NEXT: str p0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %base, i64 1024
+ store <vscale x 16 x i1> %val, ptr %addr, align 2
+ ret void
+}
>From e507be1398ced49e2201fefc35eae289de580ac3 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker at arm.com>
Date: Tue, 30 Jun 2026 17:49:31 +0100
Subject: [PATCH 2/2] [LLVM][CodeGen][SVE] Make use of predicate load/store
"mul vl" addressing mode.
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 18 -----
.../lib/Target/AArch64/AArch64InstrFormats.td | 2 -
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 20 ------
llvm/lib/Target/AArch64/SVEInstrFormats.td | 41 +++++++----
.../AArch64/sve-calling-convention-byref.ll | 64 +++++------------
llvm/test/CodeGen/AArch64/sve-pred-ldst.ll | 70 +++++--------------
6 files changed, 64 insertions(+), 151 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 95f82f180e58c..6c6da03fe2dde 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -402,7 +402,6 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
void SelectMultiVectorMoveZ(SDNode *N, unsigned NumVecs,
unsigned Op, unsigned MaxIdx, unsigned Scale,
unsigned BaseReg = 0);
- bool SelectAddrModeFrameIndexSVE(SDValue N, SDValue &Base, SDValue &OffImm);
/// SVE Reg+Imm addressing mode.
template <int64_t Min, int64_t Max>
bool SelectAddrModeIndexedSVE(SDNode *Root, SDValue N, SDValue &Base,
@@ -2534,23 +2533,6 @@ void AArch64DAGToDAGISel::SelectPredicatedStore(SDNode *N, unsigned NumVecs,
ReplaceNode(N, St);
}
-bool AArch64DAGToDAGISel::SelectAddrModeFrameIndexSVE(SDValue N, SDValue &Base,
- SDValue &OffImm) {
- SDLoc dl(N);
- const DataLayout &DL = CurDAG->getDataLayout();
- const TargetLowering *TLI = getTargetLowering();
-
- // Try to match it for the frame address
- if (auto FINode = dyn_cast<FrameIndexSDNode>(N)) {
- int FI = FINode->getIndex();
- Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
- OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
- return true;
- }
-
- return false;
-}
-
void AArch64DAGToDAGISel::SelectPostStore(SDNode *N, unsigned NumVecs,
unsigned Opc) {
SDLoc dl(N);
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index daf53c99988b0..9861c8490797e 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -599,8 +599,6 @@ def simm7s16 : Operand<i32> {
let OperandType = "OPERAND_IMMEDIATE";
}
-def am_sve_fi : ComplexPattern<iPTR, 2, "SelectAddrModeFrameIndexSVE", []>;
-
def am_indexed7s8 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed7S8", []>;
def am_indexed7s16 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed7S16", []>;
def am_indexed7s32 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed7S32", []>;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index ec7cb3361635c..8482f88e1ce74 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -3305,26 +3305,6 @@ let Predicates = [HasSVE_or_SME] in {
(STR_ZXI ZPR:$val, GPR64sp:$base, (i64 0))>;
}
- multiclass unpred_store_predicate<ValueType Ty, Instruction Store> {
- def _fi : Pat<(store (Ty PPR:$val), (am_sve_fi GPR64sp:$base, simm9:$offset)),
- (Store PPR:$val, GPR64sp:$base, simm9:$offset)>;
-
- def _default : Pat<(store (Ty PPR:$Val), GPR64:$base),
- (Store PPR:$Val, GPR64:$base, (i64 0))>;
- }
-
- defm Pat_Store_P16 : unpred_store_predicate<nxv16i1, STR_PXI>;
-
- multiclass unpred_load_predicate<ValueType Ty, Instruction Load> {
- def _fi : Pat<(Ty (load (am_sve_fi GPR64sp:$base, simm9:$offset))),
- (Load GPR64sp:$base, simm9:$offset)>;
-
- def _default : Pat<(Ty (load GPR64:$base)),
- (Load GPR64:$base, (i64 0))>;
- }
-
- defm Pat_Load_P16 : unpred_load_predicate<nxv16i1, LDR_PXI>;
-
multiclass ld1<Instruction RegRegInst, Instruction RegImmInst, ValueType Ty,
SDPatternOperator Load, ValueType PredTy, ValueType MemVT, ComplexPattern AddrCP> {
// reg + reg
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 93052a2255ff9..c2f9490aa2f64 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -376,6 +376,18 @@ def SVEShiftImmR64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<1, 64, true>", []
def SVEShiftSplatImmR : ComplexPattern<iAny, 1, "SelectSVEShiftSplatImmR", []>;
+let WantsRoot = true in {
+ def am_sve_indexed_s4 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-8, 7>">;
+ def am_sve_indexed_s6 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-32, 31>">;
+ def am_sve_indexed_s9 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-256, 255>">;
+}
+
+def am_sve_regreg_lsl0 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<0>", []>;
+def am_sve_regreg_lsl1 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<1>", []>;
+def am_sve_regreg_lsl2 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<2>", []>;
+def am_sve_regreg_lsl3 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<3>", []>;
+def am_sve_regreg_lsl4 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<4>", []>;
+
def SVEAllActive : ComplexPattern<untyped, 0, "SelectAllActivePredicate", []>;
def SVEAnyPredicate : ComplexPattern<untyped, 0, "SelectAnyPredicate", []>;
@@ -7374,6 +7386,14 @@ multiclass sve_mem_p_spill<string asm> {
def : InstAlias<asm # "\t$Pt, [$Rn]",
(!cast<Instruction>(NAME) PPRorPNRAny:$Pt, GPR64sp:$Rn, 0), 1>;
+
+ // base + immediate (mul vl)
+ def : Pat<(store nxv16i1:$val, (am_sve_indexed_s9 GPR64sp:$base, simm9:$offset)),
+ (!cast<Instruction>(NAME) PPR:$val, GPR64sp:$base, simm9:$offset)>;
+
+ // base
+ def : Pat<(store nxv16i1:$val, GPR64:$base),
+ (!cast<Instruction>(NAME) PPR:$val, GPR64:$base, (i64 0))>;
}
//===----------------------------------------------------------------------===//
@@ -8697,6 +8717,14 @@ multiclass sve_mem_p_fill<string asm> {
def : InstAlias<asm # "\t$Pt, [$Rn]",
(!cast<Instruction>(NAME) PPRorPNRAny:$Pt, GPR64sp:$Rn, 0), 1>;
+
+ // base + immediate (mul vl)
+ def : Pat<(nxv16i1 (load (am_sve_indexed_s9 GPR64sp:$base, simm9:$offset))),
+ (!cast<Instruction>(NAME) GPR64sp:$base, simm9:$offset)>;
+
+ // base
+ def : Pat<(nxv16i1 (load GPR64:$base)),
+ (!cast<Instruction>(NAME) GPR64:$base, (i64 0))>;
}
class sve2_mem_gldnt_vs_base<bits<5> opc, dag iops, string asm,
@@ -9898,19 +9926,6 @@ multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatter
def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>;
}
-/// Addressing modes
-let WantsRoot = true in {
- def am_sve_indexed_s4 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-8, 7>">;
- def am_sve_indexed_s6 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-32, 31>">;
- def am_sve_indexed_s9 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-256, 255>">;
-}
-
-def am_sve_regreg_lsl0 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<0>", []>;
-def am_sve_regreg_lsl1 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<1>", []>;
-def am_sve_regreg_lsl2 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<2>", []>;
-def am_sve_regreg_lsl3 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<3>", []>;
-def am_sve_regreg_lsl4 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<4>", []>;
-
// Predicated pseudo floating point two operand instructions.
multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> {
def _H_UNDEF : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>;
diff --git a/llvm/test/CodeGen/AArch64/sve-calling-convention-byref.ll b/llvm/test/CodeGen/AArch64/sve-calling-convention-byref.ll
index cc63c7ffc0c1e..5e925b6d9fd12 100644
--- a/llvm/test/CodeGen/AArch64/sve-calling-convention-byref.ll
+++ b/llvm/test/CodeGen/AArch64/sve-calling-convention-byref.ll
@@ -111,16 +111,10 @@ define aarch64_sve_vector_pcs <vscale x 16 x i1> @caller_with_svepred_arg_1xv16i
define aarch64_sve_vector_pcs [4 x <vscale x 16 x i1>] @callee_with_svepred_arg_4xv16i1_4xv16i1([4 x <vscale x 16 x i1>] %arg1, [4 x <vscale x 16 x i1>] %arg2) {
; CHECK: name: callee_with_svepred_arg_4xv16i1_4xv16i1
; CHECK: [[BASE:%[0-9]+]]:gpr64common = COPY $x0
-; CHECK: [[OFFSET1:%[0-9]+]]:gpr64 = CNTD_XPiI 31, 1, implicit $vg
-; CHECK: [[ADDR1:%[0-9]+]]:gpr64common = nuw ADDXrr [[BASE]], killed [[OFFSET1]]
-; CHECK: [[PRED1:%[0-9]+]]:ppr = LDR_PXI killed [[ADDR1]], 0 :: (load (<vscale x 1 x s16>))
-; CHECK: [[OFFSET2:%[0-9]+]]:gpr64 = CNTW_XPiI 31, 1, implicit $vg
-; CHECK: [[ADDR2:%[0-9]+]]:gpr64common = ADDXrr [[BASE]], killed [[OFFSET2]]
-; CHECK: [[PRED2:%[0-9]+]]:ppr = LDR_PXI killed [[ADDR2]], 0 :: (load (<vscale x 1 x s16>))
-; CHECK: [[OFFSET3:%[0-9]+]]:gpr64 = CNTD_XPiI 31, 3, implicit $vg
-; CHECK: [[ADDR3:%[0-9]+]]:gpr64common = ADDXrr [[BASE]], killed [[OFFSET3]]
-; CHECK: [[PRED3:%[0-9]+]]:ppr = LDR_PXI killed [[ADDR3]], 0 :: (load (<vscale x 1 x s16>))
; CHECK: [[PRED0:%[0-9]+]]:ppr = LDR_PXI [[BASE]], 0 :: (load (<vscale x 1 x s16>))
+; CHECK: [[PRED1:%[0-9]+]]:ppr = LDR_PXI [[BASE]], 1 :: (load (<vscale x 1 x s16>))
+; CHECK: [[PRED2:%[0-9]+]]:ppr = LDR_PXI [[BASE]], 2 :: (load (<vscale x 1 x s16>))
+; CHECK: [[PRED3:%[0-9]+]]:ppr = LDR_PXI [[BASE]], 3 :: (load (<vscale x 1 x s16>))
; CHECK: $p0 = COPY [[PRED0]]
; CHECK: $p1 = COPY [[PRED1]]
; CHECK: $p2 = COPY [[PRED2]]
@@ -144,18 +138,12 @@ define [4 x <vscale x 16 x i1>] @caller_with_svepred_arg_4xv16i1_4xv16i1([4 x <v
; CHECK: [[PRED2:%[0-9]+]]:ppr = COPY $p2
; CHECK: [[PRED1:%[0-9]+]]:ppr = COPY $p1
; CHECK: [[PRED0:%[0-9]+]]:ppr = COPY $p0
-; CHECK: [[OFFSET1:%[0-9]+]]:gpr64 = CNTD_XPiI 31, 1, implicit $vg
-; CHECK: [[OFFSET2:%[0-9]+]]:gpr64 = CNTW_XPiI 31, 1, implicit $vg
-; CHECK: [[OFFSET3:%[0-9]+]]:gpr64 = CNTD_XPiI 31, 3, implicit $vg
-; CHECK: [[STACK:%[0-9]+]]:gpr64common = ADDXri %stack.0, 0, 0
-; CHECK: [[ADDR3:%[0-9]+]]:gpr64common = ADDXrr [[STACK]], [[OFFSET3]]
; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
-; CHECK: STR_PXI [[PRED3]], killed [[ADDR3]], 0 :: (store (<vscale x 1 x s16>))
-; CHECK: [[ADDR2:%[0-9]+]]:gpr64common = ADDXrr [[STACK]], [[OFFSET2]]
-; CHECK: STR_PXI [[PRED2]], killed [[ADDR2]], 0 :: (store (<vscale x 1 x s16>))
-; CHECK: [[ADDR1:%[0-9]+]]:gpr64common = nuw ADDXrr [[STACK]], [[OFFSET1]]
-; CHECK: STR_PXI [[PRED1]], killed [[ADDR1]], 0 :: (store (<vscale x 1 x s16>))
+; CHECK: STR_PXI [[PRED3]], %stack.0, 3 :: (store (<vscale x 1 x s16>))
+; CHECK: STR_PXI [[PRED2]], %stack.0, 2 :: (store (<vscale x 1 x s16>))
+; CHECK: STR_PXI [[PRED1]], %stack.0, 1 :: (store (<vscale x 1 x s16>))
; CHECK: STR_PXI [[PRED0]], %stack.0, 0 :: (store (<vscale x 1 x s16>) into %stack.0)
+; CHECK: [[STACK:%[0-9]+]]:gpr64sp = ADDXri %stack.0, 0, 0
; CHECK: $x0 = COPY [[STACK]]
; LINUX: BL @callee_with_svepred_arg_4xv16i1_4xv16i1, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit $p0, implicit $p1, implicit $p2, implicit $p3, implicit $x0, implicit-def $sp, implicit-def $p0, implicit-def $p1, implicit-def $p2, implicit-def $p3
; DARWIN: BL @callee_with_svepred_arg_4xv16i1_4xv16i1, csr_darwin_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit $p0, implicit $p1, implicit $p2, implicit $p3, implicit $x0, implicit-def $sp, implicit-def $p0, implicit-def $p1, implicit-def $p2, implicit-def $p3
@@ -172,16 +160,10 @@ define [4 x <vscale x 16 x i1>] @caller_with_svepred_arg_4xv16i1_4xv16i1([4 x <v
define aarch64_sve_vector_pcs [2 x <vscale x 32 x i1>] @callee_with_svepred_arg_1xv16i1_2xv32i1([1 x <vscale x 16 x i1>] %arg1, [2 x <vscale x 32 x i1>] %arg2) {
; CHECK: name: callee_with_svepred_arg_1xv16i1_2xv32i1
; CHECK: [[BASE:%[0-9]+]]:gpr64common = COPY $x0
-; CHECK: [[OFFSET1:%[0-9]+]]:gpr64 = CNTD_XPiI 31, 1, implicit $vg
-; CHECK: [[ADDR1:%[0-9]+]]:gpr64common = nuw ADDXrr [[BASE]], killed [[OFFSET1]]
-; CHECK: [[PRED1:%[0-9]+]]:ppr = LDR_PXI killed [[ADDR1]], 0 :: (load (<vscale x 1 x s16>))
-; CHECK: [[OFFSET2:%[0-9]+]]:gpr64 = CNTW_XPiI 31, 1, implicit $vg
-; CHECK: [[ADDR2:%[0-9]+]]:gpr64common = ADDXrr [[BASE]], killed [[OFFSET2]]
-; CHECK: [[PRED2:%[0-9]+]]:ppr = LDR_PXI killed [[ADDR2]], 0 :: (load (<vscale x 1 x s16>))
-; CHECK: [[OFFSET3:%[0-9]+]]:gpr64 = CNTD_XPiI 31, 3, implicit $vg
-; CHECK: [[ADDR3:%[0-9]+]]:gpr64common = ADDXrr [[BASE]], killed [[OFFSET3]]
-; CHECK: [[PRED3:%[0-9]+]]:ppr = LDR_PXI killed [[ADDR3]], 0 :: (load (<vscale x 1 x s16>))
; CHECK: [[PRED0:%[0-9]+]]:ppr = LDR_PXI [[BASE]], 0 :: (load (<vscale x 1 x s16>))
+; CHECK: [[PRED1:%[0-9]+]]:ppr = LDR_PXI [[BASE]], 1 :: (load (<vscale x 1 x s16>))
+; CHECK: [[PRED2:%[0-9]+]]:ppr = LDR_PXI [[BASE]], 2 :: (load (<vscale x 1 x s16>))
+; CHECK: [[PRED3:%[0-9]+]]:ppr = LDR_PXI [[BASE]], 3 :: (load (<vscale x 1 x s16>))
; CHECK: $p0 = COPY [[PRED0]]
; CHECK: $p1 = COPY [[PRED1]]
; CHECK: $p2 = COPY [[PRED2]]
@@ -205,18 +187,12 @@ define [2 x <vscale x 32 x i1>] @caller_with_svepred_arg_2xv32i1_1xv16i1([2 x <v
; CHECK: [[PRED2:%[0-9]+]]:ppr = COPY $p2
; CHECK: [[PRED1:%[0-9]+]]:ppr = COPY $p1
; CHECK: [[PRED0:%[0-9]+]]:ppr = COPY $p0
-; CHECK: [[OFFSET3:%[0-9]+]]:gpr64 = CNTD_XPiI 31, 3, implicit $vg
-; CHECK: [[STACK:%[0-9]+]]:gpr64common = ADDXri %stack.0, 0, 0
-; CHECK: [[ADDR3:%[0-9]+]]:gpr64common = ADDXrr [[STACK]], killed [[OFFSET3]]
; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
-; CHECK: STR_PXI [[PRED3]], killed [[ADDR3]], 0 :: (store (<vscale x 1 x s16>))
-; CHECK: [[OFFSET2:%[0-9]+]]:gpr64 = CNTW_XPiI 31, 1, implicit $vg
-; CHECK: [[ADDR2:%[0-9]+]]:gpr64common = ADDXrr [[STACK]], killed [[OFFSET2]]
-; CHECK: STR_PXI [[PRED2]], killed [[ADDR2]], 0 :: (store (<vscale x 1 x s16>))
-; CHECK: [[OFFSET1:%[0-9]+]]:gpr64 = CNTD_XPiI 31, 1, implicit $vg
-; CHECK: [[ADDR1:%[0-9]+]]:gpr64common = nuw ADDXrr [[STACK]], killed [[OFFSET1]]
-; CHECK: STR_PXI [[PRED1]], killed [[ADDR1]], 0 :: (store (<vscale x 1 x s16>))
+; CHECK: STR_PXI [[PRED3]], %stack.0, 3 :: (store (<vscale x 1 x s16>))
+; CHECK: STR_PXI [[PRED2]], %stack.0, 2 :: (store (<vscale x 1 x s16>))
+; CHECK: STR_PXI [[PRED1]], %stack.0, 1 :: (store (<vscale x 1 x s16>))
; CHECK: STR_PXI [[PRED0]], %stack.0, 0 :: (store (<vscale x 1 x s16>) into %stack.0)
+; CHECK: [[STACK:%[0-9]+]]:gpr64sp = ADDXri %stack.0, 0, 0
; CHECK: $x0 = COPY [[STACK]]
; LINUX: BL @callee_with_svepred_arg_1xv16i1_2xv32i1, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit $p0, implicit $x0, implicit-def $sp, implicit-def $p0, implicit-def $p1, implicit-def $p2, implicit-def $p3
; DARWIN: BL @callee_with_svepred_arg_1xv16i1_2xv32i1, csr_darwin_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit $p0, implicit $x0, implicit-def $sp, implicit-def $p0, implicit-def $p1, implicit-def $p2, implicit-def $p3
@@ -233,15 +209,9 @@ define aarch64_sve_vector_pcs [4 x <vscale x 16 x i1>] @callee_with_svepred_arg_
; CHECK: [[X0:%[0-9]+]]:gpr64common = COPY $x0
; CHECK: [[P1:%[0-9]+]]:ppr = COPY $p1
; CHECK: [[P0:%[0-9]+]]:ppr = COPY $p0
-; CHECK: [[OFFSET3:%[0-9]+]]:gpr64 = CNTD_XPiI 31, 3, implicit $vg
-; CHECK: [[ADDR3:%[0-9]+]]:gpr64common = ADDXrr [[X0]], killed [[OFFSET3]]
-; CHECK: [[P7:%[0-9]+]]:ppr = LDR_PXI killed [[ADDR3]], 0 :: (load (<vscale x 1 x s16>))
-; CHECK: [[OFFSET2:%[0-9]+]]:gpr64 = CNTW_XPiI 31, 1, implicit $vg
-; CHECK: [[ADDR2:%[0-9]+]]:gpr64common = ADDXrr [[X0]], killed [[OFFSET2]]
-; CHECK: [[P6:%[0-9]+]]:ppr = LDR_PXI killed [[ADDR2]], 0 :: (load (<vscale x 1 x s16>))
-; CHECK: [[OFFSET1:%[0-9]+]]:gpr64 = CNTD_XPiI 31, 1, implicit $vg
-; CHECK: [[ADDR1:%[0-9]+]]:gpr64common = nuw ADDXrr [[X0]], killed [[OFFSET1]]
-; CHECK: [[P5:%[0-9]+]]:ppr = LDR_PXI killed [[ADDR1]], 0 :: (load (<vscale x 1 x s16>))
+; CHECK: [[P7:%[0-9]+]]:ppr = LDR_PXI [[X0]], 3 :: (load (<vscale x 1 x s16>))
+; CHECK: [[P6:%[0-9]+]]:ppr = LDR_PXI [[X0]], 2 :: (load (<vscale x 1 x s16>))
+; CHECK: [[P5:%[0-9]+]]:ppr = LDR_PXI [[X0]], 1 :: (load (<vscale x 1 x s16>))
; CHECK: [[P4:%[0-9]+]]:ppr = LDR_PXI [[X0]], 0 :: (load (<vscale x 1 x s16>))
; CHECK: [[RES0:%[0-9]+]]:ppr = AND_PPzPP [[P0]], [[P0]], killed [[P4]]
; CHECK: [[RES1:%[0-9]+]]:ppr = AND_PPzPP [[P1]], [[P1]], killed [[P5]]
diff --git a/llvm/test/CodeGen/AArch64/sve-pred-ldst.ll b/llvm/test/CodeGen/AArch64/sve-pred-ldst.ll
index abc56087dfa1b..37e20e6c13c0f 100644
--- a/llvm/test/CodeGen/AArch64/sve-pred-ldst.ll
+++ b/llvm/test/CodeGen/AArch64/sve-pred-ldst.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,SVE
-; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,SME
+; RUN: llc -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s
target triple = "aarch64-unknown-linux-gnu"
@@ -23,54 +23,30 @@ define void @store_no_offset(ptr %base, <vscale x 16 x i1> %val) {
}
define <vscale x 16 x i1> @load_offset(ptr %base) {
-; SVE-LABEL: load_offset:
-; SVE: // %bb.0:
-; SVE-NEXT: cntd x8, all, mul #7
-; SVE-NEXT: add x8, x0, x8
-; SVE-NEXT: ldr p0, [x8]
-; SVE-NEXT: ret
-;
-; SME-LABEL: load_offset:
-; SME: // %bb.0:
-; SME-NEXT: incd x0, all, mul #7
-; SME-NEXT: ldr p0, [x0]
-; SME-NEXT: ret
+; CHECK-LABEL: load_offset:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr p0, [x0, #7, mul vl]
+; CHECK-NEXT: ret
%addr = getelementptr <vscale x 16 x i1>, ptr %base, i64 7
%val = load <vscale x 16 x i1>, ptr %addr, align 2
ret <vscale x 16 x i1> %val
}
define void @store_offset(ptr %base, <vscale x 16 x i1> %val) {
-; SVE-LABEL: store_offset:
-; SVE: // %bb.0:
-; SVE-NEXT: cntd x8, all, mul #7
-; SVE-NEXT: sub x8, x0, x8
-; SVE-NEXT: str p0, [x8]
-; SVE-NEXT: ret
-;
-; SME-LABEL: store_offset:
-; SME: // %bb.0:
-; SME-NEXT: decd x0, all, mul #7
-; SME-NEXT: str p0, [x0]
-; SME-NEXT: ret
+; CHECK-LABEL: store_offset:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str p0, [x0, #-7, mul vl]
+; CHECK-NEXT: ret
%addr = getelementptr <vscale x 16 x i1>, ptr %base, i64 -7
store <vscale x 16 x i1> %val, ptr %addr, align 2
ret void
}
define <vscale x 16 x i1> @load_offset_min(ptr %base) {
-; SVE-LABEL: load_offset_min:
-; SVE: // %bb.0:
-; SVE-NEXT: rdvl x8, #-32
-; SVE-NEXT: add x8, x0, x8
-; SVE-NEXT: ldr p0, [x8]
-; SVE-NEXT: ret
-;
-; SME-LABEL: load_offset_min:
-; SME: // %bb.0:
-; SME-NEXT: addvl x8, x0, #-32
-; SME-NEXT: ldr p0, [x8]
-; SME-NEXT: ret
+; CHECK-LABEL: load_offset_min:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr p0, [x0, #-256, mul vl]
+; CHECK-NEXT: ret
%addr = getelementptr <vscale x 16 x i1>, ptr %base, i64 -256
%val = load <vscale x 16 x i1>, ptr %addr, align 2
ret <vscale x 16 x i1> %val
@@ -79,11 +55,7 @@ define <vscale x 16 x i1> @load_offset_min(ptr %base) {
define void @store_offset_max(ptr %base, <vscale x 16 x i1> %val) {
; CHECK-LABEL: store_offset_max:
; CHECK: // %bb.0:
-; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: mov w9, #510 // =0x1fe
-; CHECK-NEXT: lsr x8, x8, #4
-; CHECK-NEXT: madd x8, x8, x9, x0
-; CHECK-NEXT: str p0, [x8]
+; CHECK-NEXT: str p0, [x0, #255, mul vl]
; CHECK-NEXT: ret
%addr = getelementptr <vscale x 16 x i1>, ptr %base, i64 255
store <vscale x 16 x i1> %val, ptr %addr, align 2
@@ -144,8 +116,7 @@ define void @store_fixed_offset_unknown_vscale(ptr %base, <vscale x 16 x i1> %va
define <vscale x 16 x i1> @load_fixed_offset_vscale_one(ptr %base) vscale_range(1,1) {
; CHECK-LABEL: load_fixed_offset_vscale_one:
; CHECK: // %bb.0:
-; CHECK-NEXT: add x8, x0, #2
-; CHECK-NEXT: ldr p0, [x8]
+; CHECK-NEXT: ldr p0, [x0, #1, mul vl]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %base, i64 2
%val = load <vscale x 16 x i1>, ptr %addr, align 2
@@ -156,8 +127,7 @@ define <vscale x 16 x i1> @load_fixed_offset_vscale_one(ptr %base) vscale_range(
define void @store_fixed_offset_vscale_two(ptr %base, <vscale x 16 x i1> %val) vscale_range(2,2) {
; CHECK-LABEL: store_fixed_offset_vscale_two:
; CHECK: // %bb.0:
-; CHECK-NEXT: sub x8, x0, #8
-; CHECK-NEXT: str p0, [x8]
+; CHECK-NEXT: str p0, [x0, #-2, mul vl]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %base, i64 -8
store <vscale x 16 x i1> %val, ptr %addr, align 2
@@ -168,8 +138,7 @@ define void @store_fixed_offset_vscale_two(ptr %base, <vscale x 16 x i1> %val) v
define <vscale x 16 x i1> @load_fixed_offset_max_vscale_two(ptr %base) vscale_range(2,2) {
; CHECK-LABEL: load_fixed_offset_max_vscale_two:
; CHECK: // %bb.0:
-; CHECK-NEXT: add x8, x0, #1020
-; CHECK-NEXT: ldr p0, [x8]
+; CHECK-NEXT: ldr p0, [x0, #255, mul vl]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %base, i64 1020
%val = load <vscale x 16 x i1>, ptr %addr, align 2
@@ -180,8 +149,7 @@ define <vscale x 16 x i1> @load_fixed_offset_max_vscale_two(ptr %base) vscale_ra
define void @store_fixed_offset_min_vscale_one(ptr %base, <vscale x 16 x i1> %val) vscale_range(1,1) {
; CHECK-LABEL: store_fixed_offset_min_vscale_one:
; CHECK: // %bb.0:
-; CHECK-NEXT: sub x8, x0, #512
-; CHECK-NEXT: str p0, [x8]
+; CHECK-NEXT: str p0, [x0, #-256, mul vl]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %base, i64 -512
store <vscale x 16 x i1> %val, ptr %addr, align 2
More information about the llvm-commits
mailing list