[llvm] [AArch64][SDAG] Legalise nxv1 gather/scatter nodes (PR #204620)
Gaƫtan Bossu via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 19 01:24:00 PDT 2026
https://github.com/gbossu updated https://github.com/llvm/llvm-project/pull/204620
>From 2de87264cc0afec3caa34c0cfc02bdaa02c91840 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Wed, 6 May 2026 07:47:19 +0000
Subject: [PATCH 1/2] [AArch64][SDAG] Legalise nxv1 gather/scatter nodes
This updates WidenVecRes_MGATHER and WidenVecOp_MSCATTER to support
scalable vector types.
---
.../SelectionDAG/LegalizeVectorTypes.cpp | 20 +++--
.../AArch64/sve-masked-gather-64b-scaled.ll | 17 ++++
.../AArch64/sve-masked-gather-64b-unscaled.ll | 77 +++++++++++++++++++
.../AArch64/sve-masked-gather-legalize.ll | 18 +++++
.../test/CodeGen/AArch64/sve-masked-gather.ll | 73 ++++++++++++++++++
.../sve-masked-scatter-64b-unscaled.ll | 54 +++++++++++++
.../CodeGen/AArch64/sve-masked-scatter.ll | 66 ++++++++++++++++
7 files changed, 314 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 382b0e2395da3..181cf7bd48b02 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -7021,27 +7021,25 @@ SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
EVT MaskVT = Mask.getValueType();
SDValue PassThru = GetWidenedVector(N->getPassThru());
SDValue Scale = N->getScale();
- unsigned NumElts = WideVT.getVectorNumElements();
+ ElementCount WideEC = WideVT.getVectorElementCount();
SDLoc dl(N);
// The mask should be widened as well
EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
- MaskVT.getVectorElementType(),
- WideVT.getVectorNumElements());
+ MaskVT.getVectorElementType(), WideEC);
Mask = ModifyToType(Mask, WideMaskVT, true);
// Widen the Index operand
SDValue Index = N->getIndex();
- EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
- Index.getValueType().getScalarType(),
- NumElts);
+ EVT WideIndexVT = EVT::getVectorVT(
+ *DAG.getContext(), Index.getValueType().getScalarType(), WideEC);
Index = ModifyToType(Index, WideIndexVT);
SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
Scale };
// Widen the MemoryType
EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
- N->getMemoryVT().getScalarType(), NumElts);
+ N->getMemoryVT().getScalarType(), WideEC);
SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
WideMemVT, dl, Ops, N->getMemOperand(),
N->getIndexType(), N->getExtensionType());
@@ -8373,23 +8371,23 @@ SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
if (OpNo == 1) {
DataOp = GetWidenedVector(DataOp);
- unsigned NumElts = DataOp.getValueType().getVectorNumElements();
+ ElementCount WideEC = DataOp.getValueType().getVectorElementCount();
// Widen index.
EVT IndexVT = Index.getValueType();
EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
- IndexVT.getVectorElementType(), NumElts);
+ IndexVT.getVectorElementType(), WideEC);
Index = ModifyToType(Index, WideIndexVT);
// The mask should be widened as well.
EVT MaskVT = Mask.getValueType();
EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
- MaskVT.getVectorElementType(), NumElts);
+ MaskVT.getVectorElementType(), WideEC);
Mask = ModifyToType(Mask, WideMaskVT, true);
// Widen the MemoryType
WideMemVT = EVT::getVectorVT(*DAG.getContext(),
- MSC->getMemoryVT().getScalarType(), NumElts);
+ MSC->getMemoryVT().getScalarType(), WideEC);
} else if (OpNo == 4) {
// Just widen the index. It's allowed to have extra elements.
Index = GetWidenedVector(Index);
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
index 624541766c957..72b8496c8d8b7 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
@@ -96,6 +96,23 @@ define <vscale x 2 x i64> @masked_sgather_nxv2i32(ptr %base, <vscale x 2 x i64>
ret <vscale x 2 x i64> %vals.sext
}
+define <vscale x 2 x i64> @masked_gather_nxv1i64(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d, lsl #3]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+ <vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i64, ptr %base, <vscale x 1 x i64> %offsets
+ %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
+ <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+ %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
+ <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+ ret <vscale x 2 x i64> %r.legal
+}
+
declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
index 0ed4dd1e4136e..b6552968a97f3 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
@@ -127,6 +127,83 @@ define <vscale x 2 x i64> @masked_sgather_nxv2i32(ptr %base, <vscale x 2 x i64>
ret <vscale x 2 x i64> %vals.sext
}
+define <vscale x 16 x i8> @masked_gather_nxv1i8(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+ <vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(
+ <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
+ %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(
+ <vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
+ ret <vscale x 16 x i8> %r.legal
+}
+
+define <vscale x 8 x i16> @masked_gather_nxv1i16(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+ <vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(
+ <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
+ %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(
+ <vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
+ ret <vscale x 8 x i16> %r.legal
+}
+
+define <vscale x 4 x i32> @masked_gather_nxv1i32(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+ <vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(
+ <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
+ %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(
+ <vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
+ ret <vscale x 4 x i32> %r.legal
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv1i64(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+ <vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
+ <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+ %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
+ <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+ ret <vscale x 2 x i64> %r.legal
+}
+
declare <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
index adc95e9515e2e..dfe1ba0a0f05c 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
@@ -201,6 +201,24 @@ define <vscale x 16 x i8> @masked_gather_nxv16i8(ptr %base, <vscale x 16 x i8> %
ret <vscale x 16 x i8> %data
}
+; Similar as above but only a fourth of the mask is defined and the other lanes are "false".
+; Expect a single ld1b.
+define <vscale x 16 x i8> @masked_gather_nxv16i8_undef_hi_mask(ptr %base, <vscale x 16 x i8> %indices, <vscale x 4 x i1> %mask) #0 {
+; CHECK-LABEL: masked_gather_nxv16i8_undef_hi_mask:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sunpklo z0.h, z0.b
+; CHECK-NEXT: sunpklo z0.s, z0.h
+; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT: uzp1 z1.h, z0.h, z0.h
+; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+ %ptrs = getelementptr i8, ptr %base, <vscale x 16 x i8> %indices
+ %mask.false.hi = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1> splat (i1 false), <vscale x 4 x i1> %mask, i64 0)
+ %data = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8(<vscale x 16 x ptr> %ptrs, i32 1, <vscale x 16 x i1> %mask.false.hi, <vscale x 16 x i8> poison)
+ ret <vscale x 16 x i8> %data
+}
+
; Code generate the worst case scenario when all vector types are illegal.
define <vscale x 32 x i32> @masked_gather_nxv32i32(ptr %base, <vscale x 32 x i32> %indices, <vscale x 32 x i1> %mask) #0 {
; CHECK-LABEL: masked_gather_nxv32i32:
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
index 5a7865e92415f..77329371c8e2c 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
@@ -96,6 +96,79 @@ define <vscale x 2 x i64> @masked_sgather_nxv2i16(<vscale x 2 x ptr> %ptrs, <vsc
ret <vscale x 2 x i64> %vals.sext
}
+define <vscale x 16 x i8> @masked_gather_nxv1i8(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1b { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(
+ <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
+ %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(
+ <vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
+ ret <vscale x 16 x i8> %r.legal
+}
+
+define <vscale x 8 x i16> @masked_gather_nxv1i16(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1h { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(
+ <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
+ %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(
+ <vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
+ ret <vscale x 8 x i16> %r.legal
+}
+
+define <vscale x 4 x i32> @masked_gather_nxv1i32(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1w { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(
+ <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
+ %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(
+ <vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
+ ret <vscale x 4 x i32> %r.legal
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv1i64(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
+ <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+ %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
+ <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+ ret <vscale x 2 x i64> %r.legal
+}
+
define <vscale x 2 x i64> @masked_sgather_nxv2i32(<vscale x 2 x ptr> %ptrs, <vscale x 2 x i1> %mask) {
; CHECK-LABEL: masked_sgather_nxv2i32:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
index 67acf8618809b..60eb3edf4d31c 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
@@ -93,6 +93,60 @@ define void @masked_scatter_nxv2f64_unscaled_64bit_offsets(<vscale x 2 x double>
ret void
}
+define void @masked_scatter_nxv1i8_unscaled_64bit_offsets(<vscale x 16 x i8> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i8_unscaled_64bit_offsets:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: st1b { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+ <vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(
+ <vscale x 16 x i8> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @masked_scatter_nxv1i16_unscaled_64bit_offsets(<vscale x 8 x i16> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i16_unscaled_64bit_offsets:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+ <vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(
+ <vscale x 8 x i16> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @masked_scatter_nxv1i32_unscaled_64bit_offsets(<vscale x 4 x i32> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i32_unscaled_64bit_offsets:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1w { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+ <vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(
+ <vscale x 4 x i32> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask)
+ ret void
+}
+
declare void @llvm.masked.scatter.nxv2f16(<vscale x 2 x half>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
declare void @llvm.masked.scatter.nxv4f16(<vscale x 4 x half>, <vscale x 4 x ptr>, i32, <vscale x 4 x i1>)
declare void @llvm.masked.scatter.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
index 8f2cbbdb55636..86bb90e0587f1 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
@@ -73,6 +73,72 @@ define void @masked_scatter_nxv2f64(<vscale x 2 x double> %data, <vscale x 2 x p
ret void
}
+define void @masked_scatter_nxv1i8(<vscale x 16 x i8> %data.wide, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: st1b { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(
+ <vscale x 16 x i8> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @masked_scatter_nxv1i16(<vscale x 8 x i16> %data.wide, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: st1h { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(
+ <vscale x 8 x i16> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @masked_scatter_nxv1i32(<vscale x 4 x i32> %data.wide, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1w { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(
+ <vscale x 4 x i32> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @masked_scatter_nxv1i64(<vscale x 2 x i64> %data.wide, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+ <vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+ <vscale x 2 x i64> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask)
+ ret void
+}
+
define void @masked_scatter_splat_constant_pointer (<vscale x 4 x i1> %pg) {
; CHECK-LABEL: masked_scatter_splat_constant_pointer:
; CHECK: // %bb.0: // %vector.body
>From 3c7a8dfef610ca446ddea9c2d1c4242321e6c824 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Fri, 19 Jun 2026 08:22:24 +0000
Subject: [PATCH 2/2] Tidy up tests
---
.../AArch64/sve-masked-gather-64b-scaled.ll | 9 ++---
.../AArch64/sve-masked-gather-64b-unscaled.ll | 36 ++++++------------
.../test/CodeGen/AArch64/sve-masked-gather.ll | 36 ++++++------------
.../AArch64/sve-masked-scatter-64b-scaled.ll | 14 +++++++
.../sve-masked-scatter-64b-unscaled.ll | 38 +++++++++++--------
.../CodeGen/AArch64/sve-masked-scatter.ll | 32 ++++++----------
6 files changed, 76 insertions(+), 89 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
index 72b8496c8d8b7..42838d4ee6d65 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
@@ -103,13 +103,10 @@ define <vscale x 2 x i64> @masked_gather_nxv1i64(ptr %base, <vscale x 2 x i64> %
; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d, lsl #3]
; CHECK-NEXT: ret
- %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
- <vscale x 2 x i64> %wide.offsets, i64 0)
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
%ptrs = getelementptr i64, ptr %base, <vscale x 1 x i64> %offsets
- %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
- <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
- %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
- <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+ %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+ %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(<vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
ret <vscale x 2 x i64> %r.legal
}
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
index b6552968a97f3..03c2194e91a2b 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
@@ -139,13 +139,10 @@ define <vscale x 16 x i8> @masked_gather_nxv1i8(ptr %base, <vscale x 2 x i64> %w
; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h
; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
; CHECK-NEXT: ret
- %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
- <vscale x 2 x i64> %wide.offsets, i64 0)
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
%ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
- %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(
- <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
- %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(
- <vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
+ %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(<vscale x 1 x ptr> align 1 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
+ %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(<vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
ret <vscale x 16 x i8> %r.legal
}
@@ -159,13 +156,10 @@ define <vscale x 8 x i16> @masked_gather_nxv1i16(ptr %base, <vscale x 2 x i64> %
; CHECK-NEXT: uzp1 z1.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: ret
- %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
- <vscale x 2 x i64> %wide.offsets, i64 0)
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
%ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
- %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(
- <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
- %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(
- <vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
+ %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(<vscale x 1 x ptr> align 2 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
+ %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(<vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
ret <vscale x 8 x i16> %r.legal
}
@@ -177,13 +171,10 @@ define <vscale x 4 x i32> @masked_gather_nxv1i32(ptr %base, <vscale x 2 x i64> %
; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0, z0.d]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: ret
- %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
- <vscale x 2 x i64> %wide.offsets, i64 0)
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
%ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
- %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(
- <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
- %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(
- <vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
+ %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(<vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
+ %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(<vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
ret <vscale x 4 x i32> %r.legal
}
@@ -194,13 +185,10 @@ define <vscale x 2 x i64> @masked_gather_nxv1i64(ptr %base, <vscale x 2 x i64> %
; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d]
; CHECK-NEXT: ret
- %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
- <vscale x 2 x i64> %wide.offsets, i64 0)
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
%ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
- %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
- <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
- %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
- <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+ %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+ %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(<vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
ret <vscale x 2 x i64> %r.legal
}
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
index 77329371c8e2c..8ae74188ec939 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
@@ -108,12 +108,9 @@ define <vscale x 16 x i8> @masked_gather_nxv1i8(<vscale x 2 x ptr> %wide.ptrs, <
; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h
; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
; CHECK-NEXT: ret
- %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
- <vscale x 2 x ptr> %wide.ptrs, i64 0)
- %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(
- <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
- %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(
- <vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(<vscale x 1 x ptr> align 1 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
+ %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(<vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
ret <vscale x 16 x i8> %r.legal
}
@@ -127,12 +124,9 @@ define <vscale x 8 x i16> @masked_gather_nxv1i16(<vscale x 2 x ptr> %wide.ptrs,
; CHECK-NEXT: uzp1 z1.s, z0.s, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: ret
- %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
- <vscale x 2 x ptr> %wide.ptrs, i64 0)
- %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(
- <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
- %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(
- <vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(<vscale x 1 x ptr> align 2 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
+ %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(<vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
ret <vscale x 8 x i16> %r.legal
}
@@ -144,12 +138,9 @@ define <vscale x 4 x i32> @masked_gather_nxv1i32(<vscale x 2 x ptr> %wide.ptrs,
; CHECK-NEXT: ld1w { z0.d }, p0/z, [z0.d]
; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
; CHECK-NEXT: ret
- %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
- <vscale x 2 x ptr> %wide.ptrs, i64 0)
- %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(
- <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
- %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(
- <vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(<vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
+ %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(<vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
ret <vscale x 4 x i32> %r.legal
}
@@ -160,12 +151,9 @@ define <vscale x 2 x i64> @masked_gather_nxv1i64(<vscale x 2 x ptr> %wide.ptrs,
; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
; CHECK-NEXT: ld1d { z0.d }, p0/z, [z0.d]
; CHECK-NEXT: ret
- %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
- <vscale x 2 x ptr> %wide.ptrs, i64 0)
- %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
- <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
- %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
- <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+ %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(<vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
ret <vscale x 2 x i64> %r.legal
}
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll
index 9244f2c81e799..beae9876c2636 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll
@@ -65,6 +65,20 @@ define void @masked_scatter_nxv2f64(<vscale x 2 x double> %data, ptr %base, <vsc
ret void
}
+define void @masked_scatter_nxv1i64(<vscale x 2 x i64> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d, lsl #3]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i64, ptr %base, <vscale x 1 x i64> %offsets
+ %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
declare void @llvm.masked.scatter.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
declare void @llvm.masked.scatter.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
declare void @llvm.masked.scatter.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
index 60eb3edf4d31c..536ddabd5d43b 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
@@ -103,12 +103,10 @@ define void @masked_scatter_nxv1i8_unscaled_64bit_offsets(<vscale x 16 x i8> %da
; CHECK-NEXT: uunpklo z0.d, z0.s
; CHECK-NEXT: st1b { z0.d }, p0, [x0, z1.d]
; CHECK-NEXT: ret
- %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
- <vscale x 2 x i64> %wide.offsets, i64 0)
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
%ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
- %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(
- <vscale x 16 x i8> %data.wide, i64 0)
- call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask)
+ %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(<vscale x 16 x i8> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> align 1 %ptrs, <vscale x 1 x i1> %mask)
ret void
}
@@ -121,12 +119,10 @@ define void @masked_scatter_nxv1i16_unscaled_64bit_offsets(<vscale x 8 x i16> %d
; CHECK-NEXT: uunpklo z0.d, z0.s
; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d]
; CHECK-NEXT: ret
- %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
- <vscale x 2 x i64> %wide.offsets, i64 0)
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
%ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
- %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(
- <vscale x 8 x i16> %data.wide, i64 0)
- call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask)
+ %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(<vscale x 8 x i16> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> align 2 %ptrs, <vscale x 1 x i1> %mask)
ret void
}
@@ -138,12 +134,24 @@ define void @masked_scatter_nxv1i32_unscaled_64bit_offsets(<vscale x 4 x i32> %d
; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
; CHECK-NEXT: st1w { z0.d }, p0, [x0, z1.d]
; CHECK-NEXT: ret
- %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
- <vscale x 2 x i64> %wide.offsets, i64 0)
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
%ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
- %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(
- <vscale x 4 x i32> %data.wide, i64 0)
- call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask)
+ %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(<vscale x 4 x i32> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @masked_scatter_nxv1i64_unscaled_64bit_offsets(<vscale x 2 x i64> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i64_unscaled_64bit_offsets:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
ret void
}
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
index 86bb90e0587f1..483953c2b23d4 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
@@ -83,11 +83,9 @@ define void @masked_scatter_nxv1i8(<vscale x 16 x i8> %data.wide, <vscale x 2 x
; CHECK-NEXT: uunpklo z0.d, z0.s
; CHECK-NEXT: st1b { z0.d }, p0, [z1.d]
; CHECK-NEXT: ret
- %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
- <vscale x 2 x ptr> %wide.ptrs, i64 0)
- %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(
- <vscale x 16 x i8> %data.wide, i64 0)
- call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask)
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(<vscale x 16 x i8> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> align 1 %ptrs, <vscale x 1 x i1> %mask)
ret void
}
@@ -100,11 +98,9 @@ define void @masked_scatter_nxv1i16(<vscale x 8 x i16> %data.wide, <vscale x 2 x
; CHECK-NEXT: uunpklo z0.d, z0.s
; CHECK-NEXT: st1h { z0.d }, p0, [z1.d]
; CHECK-NEXT: ret
- %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
- <vscale x 2 x ptr> %wide.ptrs, i64 0)
- %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(
- <vscale x 8 x i16> %data.wide, i64 0)
- call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask)
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(<vscale x 8 x i16> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> align 2 %ptrs, <vscale x 1 x i1> %mask)
ret void
}
@@ -116,11 +112,9 @@ define void @masked_scatter_nxv1i32(<vscale x 4 x i32> %data.wide, <vscale x 2 x
; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
; CHECK-NEXT: st1w { z0.d }, p0, [z1.d]
; CHECK-NEXT: ret
- %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
- <vscale x 2 x ptr> %wide.ptrs, i64 0)
- %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(
- <vscale x 4 x i32> %data.wide, i64 0)
- call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask)
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(<vscale x 4 x i32> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask)
ret void
}
@@ -131,11 +125,9 @@ define void @masked_scatter_nxv1i64(<vscale x 2 x i64> %data.wide, <vscale x 2 x
; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
; CHECK-NEXT: st1d { z0.d }, p0, [z1.d]
; CHECK-NEXT: ret
- %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
- <vscale x 2 x ptr> %wide.ptrs, i64 0)
- %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
- <vscale x 2 x i64> %data.wide, i64 0)
- call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask)
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
ret void
}
More information about the llvm-commits
mailing list