[llvm] [AArch64][SVE] Select immediate offsets for multi-vector instructions (PR #201637)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 8 09:26:30 PDT 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/201637
>From 5fd5ed912eae74aedb518584aa1c43eed5df8cc2 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 4 Jun 2026 16:52:28 +0000
Subject: [PATCH 1/3] Precommit tests
---
.../AArch64/sve-multivector-fold-imms.ll | 166 ++++++++++++++++++
1 file changed, 166 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll b/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll
new file mode 100644
index 0000000000000..5e7c1de98fea4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll
@@ -0,0 +1,166 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -enable-subreg-liveness=true < %s | FileCheck %s
+
+;; Normal multi-vector loads/stores:
+
+define void @store_2x_vectors_offset_mul_vl(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: store_2x_vectors_offset_mul_vl:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: addvl x8, x0, #14
+; CHECK-NEXT: st1b { z0.b, z1.b }, pn8, [x8]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %mul_vl_14 = mul i64 %vscale, 224 ; #14, mul vl
+ %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_14
+ call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %offset.addr)
+ ret void
+}
+
+define void @store_4x_vectors_offset_mul_vl(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: store_4x_vectors_offset_mul_vl:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: incb x0, all, mul #4
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: st1b { z0.b - z3.b }, pn8, [x0]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %mul_vl_4 = mul i64 %vscale, 64 ; #4, mul vl
+ %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_4
+ call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %offset.addr)
+ ret void
+}
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @load_x2_vectors_offset_mul_vl(target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: load_x2_vectors_offset_mul_vl:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: rdvl x8, #-2
+; CHECK-NEXT: ld1b { z0.b, z1.b }, pn8/z, [x0, x8]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %mul_vl_-2 = mul i64 %vscale, -32 ; #-2, mul vl
+ %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_-2
+ %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %pn, ptr %offset.addr);
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @load_x4_vectors_offset_mul_vl(target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: load_x4_vectors_offset_mul_vl:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: rdvl x8, #-4
+; CHECK-NEXT: ld1b { z0.b - z3.b }, pn8/z, [x0, x8]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %mul_vl_-4 = mul i64 %vscale, -64 ; #-4, mul vl
+ %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_-4
+ %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %pn, ptr %offset.addr);
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
+;; Non-temporal multi-vector loads/stores:
+
+define void @store_2x_vectors_offset_mul_vl_nt(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: store_2x_vectors_offset_mul_vl_nt:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: addvl x8, x0, #14
+; CHECK-NEXT: stnt1b { z0.b, z1.b }, pn8, [x8]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %mul_vl_14 = mul i64 %vscale, 224 ; #14, mul vl
+ %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_14
+ call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %offset.addr)
+ ret void
+}
+
+define void @store_4x_vectors_offset_mul_vl_nt(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: store_4x_vectors_offset_mul_vl_nt:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: incb x0, all, mul #4
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: stnt1b { z0.b - z3.b }, pn8, [x0]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %mul_vl_4 = mul i64 %vscale, 64 ; #4, mul vl
+ %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_4
+ call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %offset.addr)
+ ret void
+}
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @load_x2_vectors_offset_mul_vl_nt(target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: load_x2_vectors_offset_mul_vl_nt:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: rdvl x8, #-2
+; CHECK-NEXT: ldnt1b { z0.b, z1.b }, pn8/z, [x0, x8]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %mul_vl_-2 = mul i64 %vscale, -32 ; #-2, mul vl
+ %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_-2
+ %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") %pn, ptr %offset.addr);
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @load_x4_vectors_offset_mul_vl_nt(target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: load_x4_vectors_offset_mul_vl_nt:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: rdvl x8, #-4
+; CHECK-NEXT: ldnt1b { z0.b - z3.b }, pn8/z, [x0, x8]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %mul_vl_-4 = mul i64 %vscale, -64 ; #-4, mul vl
+ %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_-4
+ %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount") %pn, ptr %offset.addr);
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
>From dc5586a9c66487503d91c0816c5a84bf13d4b4b8 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 4 Jun 2026 16:55:18 +0000
Subject: [PATCH 2/3] [AArch64][SVE] Select immediate offsets for multi-vector
instructions
This handles multi-vector intrinsics in getMemVTFromNode() and
Implements the missing ISEL patterns needed to select the immediate
(mul vl) offsets.
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 24 +++++++++++++++++++
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 19 +++++++++++++++
.../AArch64/sve-multivector-fold-imms.ll | 24 +++++++------------
3 files changed, 51 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index b619f525ae9a8..0fac7fdfed2a1 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -7677,6 +7677,14 @@ static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT,
return MemVT;
}
+/// Builds an integer vector type large enough to hold \p NumVec instances
+/// of \p VecVT.
+static EVT getPackedMultipleVectorType(LLVMContext &Ctx, EVT VecVT,
+ unsigned NumVec) {
+ return EVT::getVectorVT(Ctx, VecVT.getScalarType().changeTypeToInteger(),
+ VecVT.getVectorElementCount() * NumVec);
+}
+
/// Return the EVT of the data associated to a memory operation in \p
/// Root. If such EVT cannot be retrieved, it returns an invalid EVT.
static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
@@ -7751,6 +7759,22 @@ static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
case Intrinsic::aarch64_sve_st4q:
return getPackedVectorTypeFromPredicateType(
Ctx, Root->getOperand(6)->getValueType(0), /*NumVec=*/4);
+ case Intrinsic::aarch64_sve_ld1_pn_x2:
+ case Intrinsic::aarch64_sve_ldnt1_pn_x2:
+ return getPackedMultipleVectorType(Ctx, Root->getValueType(0),
+ /*NumVec=*/2);
+ case Intrinsic::aarch64_sve_ld1_pn_x4:
+ case Intrinsic::aarch64_sve_ldnt1_pn_x4:
+ return getPackedMultipleVectorType(Ctx, Root->getValueType(0),
+ /*NumVec=*/4);
+ case Intrinsic::aarch64_sve_st1_pn_x2:
+ case Intrinsic::aarch64_sve_stnt1_pn_x2:
+ return getPackedMultipleVectorType(Ctx, Root->getOperand(2).getValueType(),
+ /*NumVec=*/2);
+ case Intrinsic::aarch64_sve_st1_pn_x4:
+ case Intrinsic::aarch64_sve_stnt1_pn_x4:
+ return getPackedMultipleVectorType(Ctx, Root->getOperand(2).getValueType(),
+ /*NumVec=*/4);
case Intrinsic::aarch64_sve_ld1udq:
case Intrinsic::aarch64_sve_st1dq:
return EVT(MVT::nxv1i64);
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 63f5a680bbf1a..0cc788d12bae0 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4539,6 +4539,15 @@ defm STNT1D_4Z_IMM : sve2p1_mem_cst_si_4z<"stnt1d", 0b11, 0b1, ZZZZ_d_mul_r>;
multiclass store_pn_x2<ValueType Ty, SDPatternOperator Store,
Instruction RegImmInst> {
+ let AddedComplexity = 1 in {
+ // scalar + immediate (mul vl)
+ def : Pat<(Store Ty:$vec0, Ty:$vec1, aarch64svcount:$PNg,
+ (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
+ (RegImmInst (REG_SEQUENCE ZPR2Mul2, Ty:$vec0, zsub0, Ty:$vec1, zsub1),
+ PNR:$PNg, GPR64:$base, simm4s1:$offset)>;
+ }
+
+ // base
def : Pat<(Store Ty:$vec0, Ty:$vec1, aarch64svcount:$PNg, GPR64:$base),
(RegImmInst (REG_SEQUENCE ZPR2Mul2, Ty:$vec0, zsub0, Ty:$vec1, zsub1),
PNR:$PNg, GPR64:$base, (i64 0))>;
@@ -4564,6 +4573,16 @@ defm : store_pn_x2<nxv2f64, int_aarch64_sve_stnt1_pn_x2, STNT1D_2Z_IMM>;
multiclass store_pn_x4<ValueType Ty, SDPatternOperator Store,
Instruction RegImmInst> {
+ let AddedComplexity = 1 in {
+ // scalar + immediate (mul vl)
+ def : Pat<(Store Ty:$vec0, Ty:$vec1, Ty:$vec2, Ty:$vec3, aarch64svcount:$PNg,
+ (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
+ (RegImmInst (REG_SEQUENCE ZPR4Mul4, Ty:$vec0, zsub0, Ty:$vec1, zsub1,
+ Ty:$vec2, zsub2, Ty:$vec3, zsub3),
+ PNR:$PNg, GPR64:$base, simm4s1:$offset)>;
+ }
+
+ // base
def : Pat<(Store Ty:$vec0, Ty:$vec1, Ty:$vec2, Ty:$vec3, aarch64svcount:$PNg, GPR64:$base),
(RegImmInst (REG_SEQUENCE ZPR4Mul4, Ty:$vec0, zsub0, Ty:$vec1, zsub1,
Ty:$vec2, zsub2, Ty:$vec3, zsub3),
diff --git a/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll b/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll
index 5e7c1de98fea4..5bcd415878edf 100644
--- a/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll
+++ b/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll
@@ -10,8 +10,7 @@ define void @store_2x_vectors_offset_mul_vl(<vscale x 16 x i8> %zn0, <vscale x 1
; CHECK-NEXT: addvl sp, sp, #-1
; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; CHECK-NEXT: mov p8.b, p0.b
-; CHECK-NEXT: addvl x8, x0, #14
-; CHECK-NEXT: st1b { z0.b, z1.b }, pn8, [x8]
+; CHECK-NEXT: st1b { z0.b, z1.b }, pn8, [x0, #14, mul vl]
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -28,10 +27,9 @@ define void @store_4x_vectors_offset_mul_vl(<vscale x 16 x i8> %zn0, <vscale x 1
; CHECK: // %bb.0:
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-1
-; CHECK-NEXT: incb x0, all, mul #4
; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; CHECK-NEXT: mov p8.b, p0.b
-; CHECK-NEXT: st1b { z0.b - z3.b }, pn8, [x0]
+; CHECK-NEXT: st1b { z0.b - z3.b }, pn8, [x0, #4, mul vl]
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -50,8 +48,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8> } @load_x2_vectors_offset_mul_vl
; CHECK-NEXT: addvl sp, sp, #-1
; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; CHECK-NEXT: mov p8.b, p0.b
-; CHECK-NEXT: rdvl x8, #-2
-; CHECK-NEXT: ld1b { z0.b, z1.b }, pn8/z, [x0, x8]
+; CHECK-NEXT: ld1b { z0.b, z1.b }, pn8/z, [x0, #-2, mul vl]
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -70,8 +67,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
; CHECK-NEXT: addvl sp, sp, #-1
; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; CHECK-NEXT: mov p8.b, p0.b
-; CHECK-NEXT: rdvl x8, #-4
-; CHECK-NEXT: ld1b { z0.b - z3.b }, pn8/z, [x0, x8]
+; CHECK-NEXT: ld1b { z0.b - z3.b }, pn8/z, [x0, #-4, mul vl]
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -92,8 +88,7 @@ define void @store_2x_vectors_offset_mul_vl_nt(<vscale x 16 x i8> %zn0, <vscale
; CHECK-NEXT: addvl sp, sp, #-1
; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; CHECK-NEXT: mov p8.b, p0.b
-; CHECK-NEXT: addvl x8, x0, #14
-; CHECK-NEXT: stnt1b { z0.b, z1.b }, pn8, [x8]
+; CHECK-NEXT: stnt1b { z0.b, z1.b }, pn8, [x0, #14, mul vl]
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -110,10 +105,9 @@ define void @store_4x_vectors_offset_mul_vl_nt(<vscale x 16 x i8> %zn0, <vscale
; CHECK: // %bb.0:
; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: addvl sp, sp, #-1
-; CHECK-NEXT: incb x0, all, mul #4
; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; CHECK-NEXT: mov p8.b, p0.b
-; CHECK-NEXT: stnt1b { z0.b - z3.b }, pn8, [x0]
+; CHECK-NEXT: stnt1b { z0.b - z3.b }, pn8, [x0, #4, mul vl]
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -132,8 +126,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8> } @load_x2_vectors_offset_mul_vl
; CHECK-NEXT: addvl sp, sp, #-1
; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; CHECK-NEXT: mov p8.b, p0.b
-; CHECK-NEXT: rdvl x8, #-2
-; CHECK-NEXT: ldnt1b { z0.b, z1.b }, pn8/z, [x0, x8]
+; CHECK-NEXT: ldnt1b { z0.b, z1.b }, pn8/z, [x0, #-2, mul vl]
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -152,8 +145,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
; CHECK-NEXT: addvl sp, sp, #-1
; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; CHECK-NEXT: mov p8.b, p0.b
-; CHECK-NEXT: rdvl x8, #-4
-; CHECK-NEXT: ldnt1b { z0.b - z3.b }, pn8/z, [x0, x8]
+; CHECK-NEXT: ldnt1b { z0.b - z3.b }, pn8/z, [x0, #-4, mul vl]
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
>From c0d4601663433a95b498518800da4a077d4f0eb4 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 8 Jun 2026 16:24:58 +0000
Subject: [PATCH 3/3] Fixups
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 19 ++++++-------
.../AArch64/sve-multivector-fold-imms.ll | 28 +++++++------------
2 files changed, 19 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 0fac7fdfed2a1..5691482c52bb9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -7679,8 +7679,7 @@ static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT,
/// Builds an integer vector type large enough to hold \p NumVec instances
/// of \p VecVT.
-static EVT getPackedMultipleVectorType(LLVMContext &Ctx, EVT VecVT,
- unsigned NumVec) {
+static EVT getMultipleVectorType(LLVMContext &Ctx, EVT VecVT, unsigned NumVec) {
return EVT::getVectorVT(Ctx, VecVT.getScalarType().changeTypeToInteger(),
VecVT.getVectorElementCount() * NumVec);
}
@@ -7761,20 +7760,20 @@ static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
Ctx, Root->getOperand(6)->getValueType(0), /*NumVec=*/4);
case Intrinsic::aarch64_sve_ld1_pn_x2:
case Intrinsic::aarch64_sve_ldnt1_pn_x2:
- return getPackedMultipleVectorType(Ctx, Root->getValueType(0),
- /*NumVec=*/2);
+ return getMultipleVectorType(Ctx, Root->getValueType(0),
+ /*NumVec=*/2);
case Intrinsic::aarch64_sve_ld1_pn_x4:
case Intrinsic::aarch64_sve_ldnt1_pn_x4:
- return getPackedMultipleVectorType(Ctx, Root->getValueType(0),
- /*NumVec=*/4);
+ return getMultipleVectorType(Ctx, Root->getValueType(0),
+ /*NumVec=*/4);
case Intrinsic::aarch64_sve_st1_pn_x2:
case Intrinsic::aarch64_sve_stnt1_pn_x2:
- return getPackedMultipleVectorType(Ctx, Root->getOperand(2).getValueType(),
- /*NumVec=*/2);
+ return getMultipleVectorType(Ctx, Root->getOperand(2).getValueType(),
+ /*NumVec=*/2);
case Intrinsic::aarch64_sve_st1_pn_x4:
case Intrinsic::aarch64_sve_stnt1_pn_x4:
- return getPackedMultipleVectorType(Ctx, Root->getOperand(2).getValueType(),
- /*NumVec=*/4);
+ return getMultipleVectorType(Ctx, Root->getOperand(2).getValueType(),
+ /*NumVec=*/4);
case Intrinsic::aarch64_sve_ld1udq:
case Intrinsic::aarch64_sve_st1dq:
return EVT(MVT::nxv1i64);
diff --git a/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll b/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll
index 5bcd415878edf..864216bdc7efc 100644
--- a/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll
+++ b/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll
@@ -16,8 +16,7 @@ define void @store_2x_vectors_offset_mul_vl(<vscale x 16 x i8> %zn0, <vscale x 1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
%vscale = call i64 @llvm.vscale.i64()
- %mul_vl_14 = mul i64 %vscale, 224 ; #14, mul vl
- %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_14
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 14
call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %offset.addr)
ret void
}
@@ -29,14 +28,13 @@ define void @store_4x_vectors_offset_mul_vl(<vscale x 16 x i8> %zn0, <vscale x 1
; CHECK-NEXT: addvl sp, sp, #-1
; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; CHECK-NEXT: mov p8.b, p0.b
-; CHECK-NEXT: st1b { z0.b - z3.b }, pn8, [x0, #4, mul vl]
+; CHECK-NEXT: st1b { z0.b - z3.b }, pn8, [x0, #28, mul vl]
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
%vscale = call i64 @llvm.vscale.i64()
- %mul_vl_4 = mul i64 %vscale, 64 ; #4, mul vl
- %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_4
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 28
call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %offset.addr)
ret void
}
@@ -54,8 +52,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8> } @load_x2_vectors_offset_mul_vl
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
%vscale = call i64 @llvm.vscale.i64()
- %mul_vl_-2 = mul i64 %vscale, -32 ; #-2, mul vl
- %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_-2
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -2
%res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %pn, ptr %offset.addr);
ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
}
@@ -73,8 +70,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
%vscale = call i64 @llvm.vscale.i64()
- %mul_vl_-4 = mul i64 %vscale, -64 ; #-4, mul vl
- %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_-4
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -4
%res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %pn, ptr %offset.addr);
ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
}
@@ -94,8 +90,7 @@ define void @store_2x_vectors_offset_mul_vl_nt(<vscale x 16 x i8> %zn0, <vscale
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
%vscale = call i64 @llvm.vscale.i64()
- %mul_vl_14 = mul i64 %vscale, 224 ; #14, mul vl
- %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_14
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 14
call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %offset.addr)
ret void
}
@@ -107,14 +102,13 @@ define void @store_4x_vectors_offset_mul_vl_nt(<vscale x 16 x i8> %zn0, <vscale
; CHECK-NEXT: addvl sp, sp, #-1
; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
; CHECK-NEXT: mov p8.b, p0.b
-; CHECK-NEXT: stnt1b { z0.b - z3.b }, pn8, [x0, #4, mul vl]
+; CHECK-NEXT: stnt1b { z0.b - z3.b }, pn8, [x0, #28, mul vl]
; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
%vscale = call i64 @llvm.vscale.i64()
- %mul_vl_4 = mul i64 %vscale, 64 ; #4, mul vl
- %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_4
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 28
call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %offset.addr)
ret void
}
@@ -132,8 +126,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8> } @load_x2_vectors_offset_mul_vl
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
%vscale = call i64 @llvm.vscale.i64()
- %mul_vl_-2 = mul i64 %vscale, -32 ; #-2, mul vl
- %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_-2
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -2
%res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") %pn, ptr %offset.addr);
ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
}
@@ -151,8 +144,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
%vscale = call i64 @llvm.vscale.i64()
- %mul_vl_-4 = mul i64 %vscale, -64 ; #-4, mul vl
- %offset.addr = getelementptr i8, ptr %addr, i64 %mul_vl_-4
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -4
%res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount") %pn, ptr %offset.addr);
ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
}
More information about the llvm-commits
mailing list