[llvm] 40cbc98 - [AArch64][SDAG] Legalise nxv1 gather/scatter nodes (#204620)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 19 02:50:35 PDT 2026
Author: Gaƫtan Bossu
Date: 2026-06-19T10:50:30+01:00
New Revision: 40cbc98bb17718eef5e0b1f47e3b58900aac354a
URL: https://github.com/llvm/llvm-project/commit/40cbc98bb17718eef5e0b1f47e3b58900aac354a
DIFF: https://github.com/llvm/llvm-project/commit/40cbc98bb17718eef5e0b1f47e3b58900aac354a.diff
LOG: [AArch64][SDAG] Legalise nxv1 gather/scatter nodes (#204620)
This updates WidenVecRes_MGATHER and WidenVecOp_MSCATTER to support
scalable vector types.
Added:
Modified:
llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
llvm/test/CodeGen/AArch64/sve-masked-gather.ll
llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll
llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 382b0e2395da3..181cf7bd48b02 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -7021,27 +7021,25 @@ SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
EVT MaskVT = Mask.getValueType();
SDValue PassThru = GetWidenedVector(N->getPassThru());
SDValue Scale = N->getScale();
- unsigned NumElts = WideVT.getVectorNumElements();
+ ElementCount WideEC = WideVT.getVectorElementCount();
SDLoc dl(N);
// The mask should be widened as well
EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
- MaskVT.getVectorElementType(),
- WideVT.getVectorNumElements());
+ MaskVT.getVectorElementType(), WideEC);
Mask = ModifyToType(Mask, WideMaskVT, true);
// Widen the Index operand
SDValue Index = N->getIndex();
- EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
- Index.getValueType().getScalarType(),
- NumElts);
+ EVT WideIndexVT = EVT::getVectorVT(
+ *DAG.getContext(), Index.getValueType().getScalarType(), WideEC);
Index = ModifyToType(Index, WideIndexVT);
SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
Scale };
// Widen the MemoryType
EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
- N->getMemoryVT().getScalarType(), NumElts);
+ N->getMemoryVT().getScalarType(), WideEC);
SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
WideMemVT, dl, Ops, N->getMemOperand(),
N->getIndexType(), N->getExtensionType());
@@ -8373,23 +8371,23 @@ SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
if (OpNo == 1) {
DataOp = GetWidenedVector(DataOp);
- unsigned NumElts = DataOp.getValueType().getVectorNumElements();
+ ElementCount WideEC = DataOp.getValueType().getVectorElementCount();
// Widen index.
EVT IndexVT = Index.getValueType();
EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
- IndexVT.getVectorElementType(), NumElts);
+ IndexVT.getVectorElementType(), WideEC);
Index = ModifyToType(Index, WideIndexVT);
// The mask should be widened as well.
EVT MaskVT = Mask.getValueType();
EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
- MaskVT.getVectorElementType(), NumElts);
+ MaskVT.getVectorElementType(), WideEC);
Mask = ModifyToType(Mask, WideMaskVT, true);
// Widen the MemoryType
WideMemVT = EVT::getVectorVT(*DAG.getContext(),
- MSC->getMemoryVT().getScalarType(), NumElts);
+ MSC->getMemoryVT().getScalarType(), WideEC);
} else if (OpNo == 4) {
// Just widen the index. It's allowed to have extra elements.
Index = GetWidenedVector(Index);
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
index 624541766c957..42838d4ee6d65 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
@@ -96,6 +96,20 @@ define <vscale x 2 x i64> @masked_sgather_nxv2i32(ptr %base, <vscale x 2 x i64>
ret <vscale x 2 x i64> %vals.sext
}
+define <vscale x 2 x i64> @masked_gather_nxv1i64(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d, lsl #3]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i64, ptr %base, <vscale x 1 x i64> %offsets
+ %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+ %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(<vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+ ret <vscale x 2 x i64> %r.legal
+}
+
declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
index 0ed4dd1e4136e..03c2194e91a2b 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
@@ -127,6 +127,71 @@ define <vscale x 2 x i64> @masked_sgather_nxv2i32(ptr %base, <vscale x 2 x i64>
ret <vscale x 2 x i64> %vals.sext
}
+define <vscale x 16 x i8> @masked_gather_nxv1i8(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(<vscale x 1 x ptr> align 1 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
+ %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(<vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
+ ret <vscale x 16 x i8> %r.legal
+}
+
+define <vscale x 8 x i16> @masked_gather_nxv1i16(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(<vscale x 1 x ptr> align 2 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
+ %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(<vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
+ ret <vscale x 8 x i16> %r.legal
+}
+
+define <vscale x 4 x i32> @masked_gather_nxv1i32(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(<vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
+ %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(<vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
+ ret <vscale x 4 x i32> %r.legal
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv1i64(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+ %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(<vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+ ret <vscale x 2 x i64> %r.legal
+}
+
declare <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
index adc95e9515e2e..d40aa5bc7f31f 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
@@ -201,6 +201,24 @@ define <vscale x 16 x i8> @masked_gather_nxv16i8(ptr %base, <vscale x 16 x i8> %
ret <vscale x 16 x i8> %data
}
+; Similar as above but only a fourth of the mask is defined and the other lanes are "false".
+; Expect a single ld1b.
+define <vscale x 16 x i8> @masked_gather_nxv16i8_undef_hi_mask(ptr %base, <vscale x 16 x i8> %indices, <vscale x 4 x i1> %mask) #0 {
+; CHECK-LABEL: masked_gather_nxv16i8_undef_hi_mask:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sunpklo z0.h, z0.b
+; CHECK-NEXT: sunpklo z0.s, z0.h
+; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT: uzp1 z1.h, z0.h, z0.h
+; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+ %ptrs = getelementptr i8, ptr %base, <vscale x 16 x i8> %indices
+ %mask.false.hi = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1> splat (i1 false), <vscale x 4 x i1> %mask, i64 0)
+ %data = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8(<vscale x 16 x ptr> align 1 %ptrs, <vscale x 16 x i1> %mask.false.hi, <vscale x 16 x i8> poison)
+ ret <vscale x 16 x i8> %data
+}
+
; Code generate the worst case scenario when all vector types are illegal.
define <vscale x 32 x i32> @masked_gather_nxv32i32(ptr %base, <vscale x 32 x i32> %indices, <vscale x 32 x i1> %mask) #0 {
; CHECK-LABEL: masked_gather_nxv32i32:
@@ -252,17 +270,3 @@ define <vscale x 4 x i32> @masked_sgather_nxv4i8(<vscale x 4 x ptr> %ptrs, <vsca
}
attributes #0 = { nounwind "target-features"="+sve,+bf16" }
-
-declare <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
-declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
-declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
-declare <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x i8>)
-declare <vscale x 16 x i8> @llvm.masked.gather.nxv16i8(<vscale x 16 x ptr>, i32, <vscale x 16 x i1>, <vscale x 16 x i8>)
-declare <vscale x 32 x i32> @llvm.masked.gather.nxv32i32(<vscale x 32 x ptr>, i32, <vscale x 32 x i1>, <vscale x 32 x i32>)
-
-declare <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x half>)
-declare <vscale x 8 x half> @llvm.masked.gather.nxv8f16(<vscale x 8 x ptr>, i32, <vscale x 8 x i1>, <vscale x 8 x half>)
-declare <vscale x 8 x bfloat> @llvm.masked.gather.nxv8bf16(<vscale x 8 x ptr>, i32, <vscale x 8 x i1>, <vscale x 8 x bfloat>)
-declare <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x float>)
-declare <vscale x 8 x float> @llvm.masked.gather.nxv8f32(<vscale x 8 x ptr>, i32, <vscale x 8 x i1>, <vscale x 8 x float>)
-declare <vscale x 4 x double> @llvm.masked.gather.nxv4f64(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
index 5a7865e92415f..8ae74188ec939 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
@@ -96,6 +96,67 @@ define <vscale x 2 x i64> @masked_sgather_nxv2i16(<vscale x 2 x ptr> %ptrs, <vsc
ret <vscale x 2 x i64> %vals.sext
}
+define <vscale x 16 x i8> @masked_gather_nxv1i8(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1b { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(<vscale x 1 x ptr> align 1 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
+ %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(<vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
+ ret <vscale x 16 x i8> %r.legal
+}
+
+define <vscale x 8 x i16> @masked_gather_nxv1i16(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1h { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(<vscale x 1 x ptr> align 2 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
+ %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(<vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
+ ret <vscale x 8 x i16> %r.legal
+}
+
+define <vscale x 4 x i32> @masked_gather_nxv1i32(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1w { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(<vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
+ %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(<vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
+ ret <vscale x 4 x i32> %r.legal
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv1i64(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+ %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(<vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+ ret <vscale x 2 x i64> %r.legal
+}
+
define <vscale x 2 x i64> @masked_sgather_nxv2i32(<vscale x 2 x ptr> %ptrs, <vscale x 2 x i1> %mask) {
; CHECK-LABEL: masked_sgather_nxv2i32:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll
index 9244f2c81e799..beae9876c2636 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll
@@ -65,6 +65,20 @@ define void @masked_scatter_nxv2f64(<vscale x 2 x double> %data, ptr %base, <vsc
ret void
}
+define void @masked_scatter_nxv1i64(<vscale x 2 x i64> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d, lsl #3]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i64, ptr %base, <vscale x 1 x i64> %offsets
+ %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
declare void @llvm.masked.scatter.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
declare void @llvm.masked.scatter.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
declare void @llvm.masked.scatter.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
index 67acf8618809b..536ddabd5d43b 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
@@ -93,6 +93,68 @@ define void @masked_scatter_nxv2f64_unscaled_64bit_offsets(<vscale x 2 x double>
ret void
}
+define void @masked_scatter_nxv1i8_unscaled_64bit_offsets(<vscale x 16 x i8> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i8_unscaled_64bit_offsets:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: st1b { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(<vscale x 16 x i8> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> align 1 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @masked_scatter_nxv1i16_unscaled_64bit_offsets(<vscale x 8 x i16> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i16_unscaled_64bit_offsets:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: st1h { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(<vscale x 8 x i16> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> align 2 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @masked_scatter_nxv1i32_unscaled_64bit_offsets(<vscale x 4 x i32> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i32_unscaled_64bit_offsets:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1w { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(<vscale x 4 x i32> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @masked_scatter_nxv1i64_unscaled_64bit_offsets(<vscale x 2 x i64> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i64_unscaled_64bit_offsets:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT: ret
+ %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+ %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+ %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
declare void @llvm.masked.scatter.nxv2f16(<vscale x 2 x half>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
declare void @llvm.masked.scatter.nxv4f16(<vscale x 4 x half>, <vscale x 4 x ptr>, i32, <vscale x 4 x i1>)
declare void @llvm.masked.scatter.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
index 8f2cbbdb55636..483953c2b23d4 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
@@ -73,6 +73,64 @@ define void @masked_scatter_nxv2f64(<vscale x 2 x double> %data, <vscale x 2 x p
ret void
}
+define void @masked_scatter_nxv1i8(<vscale x 16 x i8> %data.wide, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpklo z0.h, z0.b
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: st1b { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(<vscale x 16 x i8> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> align 1 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @masked_scatter_nxv1i16(<vscale x 8 x i16> %data.wide, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: st1h { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(<vscale x 8 x i16> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> align 2 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @masked_scatter_nxv1i32(<vscale x 4 x i32> %data.wide, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1w { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(<vscale x 4 x i32> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
+define void @masked_scatter_nxv1i64(<vscale x 2 x i64> %data.wide, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: pfalse p1.b
+; CHECK-NEXT: uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT: st1d { z0.d }, p0, [z1.d]
+; CHECK-NEXT: ret
+ %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+ %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %data.wide, i64 0)
+ call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
+ ret void
+}
+
define void @masked_scatter_splat_constant_pointer (<vscale x 4 x i1> %pg) {
; CHECK-LABEL: masked_scatter_splat_constant_pointer:
; CHECK: // %bb.0: // %vector.body
More information about the llvm-commits
mailing list