[llvm] c155183 - [AArch64][SVE] Select immediate offsets for multi-vector instructions (#201637)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 01:28:55 PDT 2026
Author: Benjamin Maxwell
Date: 2026-06-09T09:28:50+01:00
New Revision: c155183957c7d4a0a53cf781c266b02dd6f1a377
URL: https://github.com/llvm/llvm-project/commit/c155183957c7d4a0a53cf781c266b02dd6f1a377
DIFF: https://github.com/llvm/llvm-project/commit/c155183957c7d4a0a53cf781c266b02dd6f1a377.diff
LOG: [AArch64][SVE] Select immediate offsets for multi-vector instructions (#201637)
This handles multi-vector intrinsics in getMemVTFromNode() and
implements the missing ISEL patterns needed to select the immediate
(mul vl) offsets.
Added:
llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll
Modified:
llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index b83ff4aa550c3..43f97dd3b60e3 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -7715,6 +7715,13 @@ static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT,
return MemVT;
}
+/// Builds an integer vector type large enough to hold \p NumVec instances
+/// of \p VecVT.
+static EVT getMultipleVectorType(LLVMContext &Ctx, EVT VecVT, unsigned NumVec) {
+ return EVT::getVectorVT(Ctx, VecVT.getScalarType().changeTypeToInteger(),
+ VecVT.getVectorElementCount() * NumVec);
+}
+
/// Return the EVT of the data associated to a memory operation in \p
/// Root. If such EVT cannot be retrieved, it returns an invalid EVT.
static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
@@ -7789,6 +7796,22 @@ static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
case Intrinsic::aarch64_sve_st4q:
return getPackedVectorTypeFromPredicateType(
Ctx, Root->getOperand(6)->getValueType(0), /*NumVec=*/4);
+ case Intrinsic::aarch64_sve_ld1_pn_x2:
+ case Intrinsic::aarch64_sve_ldnt1_pn_x2:
+ return getMultipleVectorType(Ctx, Root->getValueType(0),
+ /*NumVec=*/2);
+ case Intrinsic::aarch64_sve_ld1_pn_x4:
+ case Intrinsic::aarch64_sve_ldnt1_pn_x4:
+ return getMultipleVectorType(Ctx, Root->getValueType(0),
+ /*NumVec=*/4);
+ case Intrinsic::aarch64_sve_st1_pn_x2:
+ case Intrinsic::aarch64_sve_stnt1_pn_x2:
+ return getMultipleVectorType(Ctx, Root->getOperand(2).getValueType(),
+ /*NumVec=*/2);
+ case Intrinsic::aarch64_sve_st1_pn_x4:
+ case Intrinsic::aarch64_sve_stnt1_pn_x4:
+ return getMultipleVectorType(Ctx, Root->getOperand(2).getValueType(),
+ /*NumVec=*/4);
case Intrinsic::aarch64_sve_ld1udq:
case Intrinsic::aarch64_sve_st1dq:
return EVT(MVT::nxv1i64);
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 63f5a680bbf1a..0cc788d12bae0 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4539,6 +4539,15 @@ defm STNT1D_4Z_IMM : sve2p1_mem_cst_si_4z<"stnt1d", 0b11, 0b1, ZZZZ_d_mul_r>;
multiclass store_pn_x2<ValueType Ty, SDPatternOperator Store,
Instruction RegImmInst> {
+ let AddedComplexity = 1 in {
+ // scalar + immediate (mul vl)
+ def : Pat<(Store Ty:$vec0, Ty:$vec1, aarch64svcount:$PNg,
+ (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
+ (RegImmInst (REG_SEQUENCE ZPR2Mul2, Ty:$vec0, zsub0, Ty:$vec1, zsub1),
+ PNR:$PNg, GPR64:$base, simm4s1:$offset)>;
+ }
+
+ // base
def : Pat<(Store Ty:$vec0, Ty:$vec1, aarch64svcount:$PNg, GPR64:$base),
(RegImmInst (REG_SEQUENCE ZPR2Mul2, Ty:$vec0, zsub0, Ty:$vec1, zsub1),
PNR:$PNg, GPR64:$base, (i64 0))>;
@@ -4564,6 +4573,16 @@ defm : store_pn_x2<nxv2f64, int_aarch64_sve_stnt1_pn_x2, STNT1D_2Z_IMM>;
multiclass store_pn_x4<ValueType Ty, SDPatternOperator Store,
Instruction RegImmInst> {
+ let AddedComplexity = 1 in {
+ // scalar + immediate (mul vl)
+ def : Pat<(Store Ty:$vec0, Ty:$vec1, Ty:$vec2, Ty:$vec3, aarch64svcount:$PNg,
+ (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
+ (RegImmInst (REG_SEQUENCE ZPR4Mul4, Ty:$vec0, zsub0, Ty:$vec1, zsub1,
+ Ty:$vec2, zsub2, Ty:$vec3, zsub3),
+ PNR:$PNg, GPR64:$base, simm4s1:$offset)>;
+ }
+
+ // base
def : Pat<(Store Ty:$vec0, Ty:$vec1, Ty:$vec2, Ty:$vec3, aarch64svcount:$PNg, GPR64:$base),
(RegImmInst (REG_SEQUENCE ZPR4Mul4, Ty:$vec0, zsub0, Ty:$vec1, zsub1,
Ty:$vec2, zsub2, Ty:$vec3, zsub3),
diff --git a/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll b/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll
new file mode 100644
index 0000000000000..864216bdc7efc
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-multivector-fold-imms.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -enable-subreg-liveness=true < %s | FileCheck %s
+
+;; Normal multi-vector loads/stores:
+
+define void @store_2x_vectors_offset_mul_vl(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: store_2x_vectors_offset_mul_vl:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: st1b { z0.b, z1.b }, pn8, [x0, #14, mul vl]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 14
+ call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %offset.addr)
+ ret void
+}
+
+define void @store_4x_vectors_offset_mul_vl(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: store_4x_vectors_offset_mul_vl:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: st1b { z0.b - z3.b }, pn8, [x0, #28, mul vl]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 28
+ call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %offset.addr)
+ ret void
+}
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @load_x2_vectors_offset_mul_vl(target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: load_x2_vectors_offset_mul_vl:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: ld1b { z0.b, z1.b }, pn8/z, [x0, #-2, mul vl]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -2
+ %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %pn, ptr %offset.addr);
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @load_x4_vectors_offset_mul_vl(target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: load_x4_vectors_offset_mul_vl:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: ld1b { z0.b - z3.b }, pn8/z, [x0, #-4, mul vl]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -4
+ %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %pn, ptr %offset.addr);
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
+;; Non-temporal multi-vector loads/stores:
+
+define void @store_2x_vectors_offset_mul_vl_nt(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: store_2x_vectors_offset_mul_vl_nt:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: stnt1b { z0.b, z1.b }, pn8, [x0, #14, mul vl]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 14
+ call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, target("aarch64.svcount") %pn, ptr %offset.addr)
+ ret void
+}
+
+define void @store_4x_vectors_offset_mul_vl_nt(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: store_4x_vectors_offset_mul_vl_nt:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: stnt1b { z0.b - z3.b }, pn8, [x0, #28, mul vl]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 28
+ call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, target("aarch64.svcount") %pn, ptr %offset.addr)
+ ret void
+}
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @load_x2_vectors_offset_mul_vl_nt(target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: load_x2_vectors_offset_mul_vl_nt:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: ldnt1b { z0.b, z1.b }, pn8/z, [x0, #-2, mul vl]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -2
+ %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") %pn, ptr %offset.addr);
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @load_x4_vectors_offset_mul_vl_nt(target("aarch64.svcount") %pn, ptr %addr) nounwind {
+; CHECK-LABEL: load_x4_vectors_offset_mul_vl_nt:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-1
+; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: mov p8.b, p0.b
+; CHECK-NEXT: ldnt1b { z0.b - z3.b }, pn8/z, [x0, #-4, mul vl]
+; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #1
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %vscale = call i64 @llvm.vscale.i64()
+ %offset.addr = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -4
+ %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount") %pn, ptr %offset.addr);
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
More information about the llvm-commits
mailing list