[llvm] [AArch64][SDAG] Legalise nxv1 gather/scatter nodes (PR #204620)

Gaƫtan Bossu via llvm-commits llvm-commits at lists.llvm.org
Fri Jun 19 01:24:00 PDT 2026


https://github.com/gbossu updated https://github.com/llvm/llvm-project/pull/204620

>From 2de87264cc0afec3caa34c0cfc02bdaa02c91840 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Wed, 6 May 2026 07:47:19 +0000
Subject: [PATCH 1/2] [AArch64][SDAG] Legalise nxv1 gather/scatter nodes

This updates WidenVecRes_MGATHER and WidenVecOp_MSCATTER to support
scalable vector types.
---
 .../SelectionDAG/LegalizeVectorTypes.cpp      | 20 +++--
 .../AArch64/sve-masked-gather-64b-scaled.ll   | 17 ++++
 .../AArch64/sve-masked-gather-64b-unscaled.ll | 77 +++++++++++++++++++
 .../AArch64/sve-masked-gather-legalize.ll     | 18 +++++
 .../test/CodeGen/AArch64/sve-masked-gather.ll | 73 ++++++++++++++++++
 .../sve-masked-scatter-64b-unscaled.ll        | 54 +++++++++++++
 .../CodeGen/AArch64/sve-masked-scatter.ll     | 66 ++++++++++++++++
 7 files changed, 314 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 382b0e2395da3..181cf7bd48b02 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -7021,27 +7021,25 @@ SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
   EVT MaskVT = Mask.getValueType();
   SDValue PassThru = GetWidenedVector(N->getPassThru());
   SDValue Scale = N->getScale();
-  unsigned NumElts = WideVT.getVectorNumElements();
+  ElementCount WideEC = WideVT.getVectorElementCount();
   SDLoc dl(N);
 
   // The mask should be widened as well
   EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
-                                    MaskVT.getVectorElementType(),
-                                    WideVT.getVectorNumElements());
+                                    MaskVT.getVectorElementType(), WideEC);
   Mask = ModifyToType(Mask, WideMaskVT, true);
 
   // Widen the Index operand
   SDValue Index = N->getIndex();
-  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
-                                     Index.getValueType().getScalarType(),
-                                     NumElts);
+  EVT WideIndexVT = EVT::getVectorVT(
+      *DAG.getContext(), Index.getValueType().getScalarType(), WideEC);
   Index = ModifyToType(Index, WideIndexVT);
   SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
                     Scale };
 
   // Widen the MemoryType
   EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
-                                   N->getMemoryVT().getScalarType(), NumElts);
+                                   N->getMemoryVT().getScalarType(), WideEC);
   SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
                                     WideMemVT, dl, Ops, N->getMemOperand(),
                                     N->getIndexType(), N->getExtensionType());
@@ -8373,23 +8371,23 @@ SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
 
   if (OpNo == 1) {
     DataOp = GetWidenedVector(DataOp);
-    unsigned NumElts = DataOp.getValueType().getVectorNumElements();
+    ElementCount WideEC = DataOp.getValueType().getVectorElementCount();
 
     // Widen index.
     EVT IndexVT = Index.getValueType();
     EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
-                                       IndexVT.getVectorElementType(), NumElts);
+                                       IndexVT.getVectorElementType(), WideEC);
     Index = ModifyToType(Index, WideIndexVT);
 
     // The mask should be widened as well.
     EVT MaskVT = Mask.getValueType();
     EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
-                                      MaskVT.getVectorElementType(), NumElts);
+                                      MaskVT.getVectorElementType(), WideEC);
     Mask = ModifyToType(Mask, WideMaskVT, true);
 
     // Widen the MemoryType
     WideMemVT = EVT::getVectorVT(*DAG.getContext(),
-                                 MSC->getMemoryVT().getScalarType(), NumElts);
+                                 MSC->getMemoryVT().getScalarType(), WideEC);
   } else if (OpNo == 4) {
     // Just widen the index. It's allowed to have extra elements.
     Index = GetWidenedVector(Index);
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
index 624541766c957..72b8496c8d8b7 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
@@ -96,6 +96,23 @@ define <vscale x 2 x i64> @masked_sgather_nxv2i32(ptr %base, <vscale x 2 x i64>
   ret <vscale x 2 x i64> %vals.sext
 }
 
+define <vscale x 2 x i64> @masked_gather_nxv1i64(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d, lsl #3]
+; CHECK-NEXT:    ret
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %ptrs = getelementptr i64, ptr %base, <vscale x 1 x i64> %offsets
+  %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
+      <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+  %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
+      <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+  ret <vscale x 2 x i64> %r.legal
+}
+
 declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
 declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
 declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
index 0ed4dd1e4136e..b6552968a97f3 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
@@ -127,6 +127,83 @@ define <vscale x 2 x i64> @masked_sgather_nxv2i32(ptr %base, <vscale x 2 x i64>
   ret <vscale x 2 x i64> %vals.sext
 }
 
+define <vscale x 16 x i8> @masked_gather_nxv1i8(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
+; CHECK-NEXT:    uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT:    ret
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+  %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(
+      <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
+  %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(
+      <vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
+  ret <vscale x 16 x i8> %r.legal
+}
+
+define <vscale x 8 x i16> @masked_gather_nxv1i16(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+  %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(
+      <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
+  %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(
+      <vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
+  ret <vscale x 8 x i16> %r.legal
+}
+
+define <vscale x 4 x i32> @masked_gather_nxv1i32(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT:    ret
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+  %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(
+      <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
+  %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(
+      <vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
+  ret <vscale x 4 x i32> %r.legal
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv1i64(ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ret
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+  %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
+      <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+  %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
+      <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+  ret <vscale x 2 x i64> %r.legal
+}
+
 declare <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
 declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
 declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
index adc95e9515e2e..dfe1ba0a0f05c 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
@@ -201,6 +201,24 @@ define <vscale x 16 x i8> @masked_gather_nxv16i8(ptr %base, <vscale x 16 x i8> %
   ret <vscale x 16 x i8> %data
 }
 
+; Similar as above but only a fourth of the mask is defined and the other lanes are "false".
+; Expect a single ld1b.
+define <vscale x 16 x i8> @masked_gather_nxv16i8_undef_hi_mask(ptr %base, <vscale x 16 x i8> %indices, <vscale x 4 x i1> %mask) #0 {
+; CHECK-LABEL: masked_gather_nxv16i8_undef_hi_mask:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sunpklo z0.h, z0.b
+; CHECK-NEXT:    sunpklo z0.s, z0.h
+; CHECK-NEXT:    ld1b { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z1.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i8, ptr %base, <vscale x 16 x i8> %indices
+  %mask.false.hi = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv4i1(<vscale x 16 x i1> splat (i1 false), <vscale x 4 x i1> %mask, i64 0)
+  %data = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8(<vscale x 16 x ptr> %ptrs, i32 1, <vscale x 16 x i1> %mask.false.hi, <vscale x 16 x i8> poison)
+  ret <vscale x 16 x i8> %data
+}
+
 ; Code generate the worst case scenario when all vector types are illegal.
 define <vscale x 32 x i32> @masked_gather_nxv32i32(ptr %base, <vscale x 32 x i32> %indices, <vscale x 32 x i1> %mask) #0 {
 ; CHECK-LABEL: masked_gather_nxv32i32:
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
index 5a7865e92415f..77329371c8e2c 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
@@ -96,6 +96,79 @@ define <vscale x 2 x i64> @masked_sgather_nxv2i16(<vscale x 2 x ptr> %ptrs, <vsc
   ret <vscale x 2 x i64> %vals.sext
 }
 
+define <vscale x 16 x i8> @masked_gather_nxv1i8(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
+; CHECK-NEXT:    uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT:    ret
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+      <vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(
+      <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
+  %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(
+      <vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
+  ret <vscale x 16 x i8> %r.legal
+}
+
+define <vscale x 8 x i16> @masked_gather_nxv1i16(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+      <vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(
+      <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
+  %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(
+      <vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
+  ret <vscale x 8 x i16> %r.legal
+}
+
+define <vscale x 4 x i32> @masked_gather_nxv1i32(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT:    ret
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+      <vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(
+      <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
+  %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(
+      <vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
+  ret <vscale x 4 x i32> %r.legal
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv1i64(<vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv1i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [z0.d]
+; CHECK-NEXT:    ret
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+      <vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
+      <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+  %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
+      <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+  ret <vscale x 2 x i64> %r.legal
+}
+
 define <vscale x 2 x i64> @masked_sgather_nxv2i32(<vscale x 2 x ptr> %ptrs, <vscale x 2 x i1> %mask) {
 ; CHECK-LABEL: masked_sgather_nxv2i32:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
index 67acf8618809b..60eb3edf4d31c 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
@@ -93,6 +93,60 @@ define void @masked_scatter_nxv2f64_unscaled_64bit_offsets(<vscale x 2 x double>
   ret void
 }
 
+define void @masked_scatter_nxv1i8_unscaled_64bit_offsets(<vscale x 16 x i8> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i8_unscaled_64bit_offsets:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uunpklo z0.h, z0.b
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    uunpklo z0.d, z0.s
+; CHECK-NEXT:    st1b { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT:    ret
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+  %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(
+      <vscale x 16 x i8> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask)
+  ret void
+}
+
+define void @masked_scatter_nxv1i16_unscaled_64bit_offsets(<vscale x 8 x i16> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i16_unscaled_64bit_offsets:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    uunpklo z0.d, z0.s
+; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT:    ret
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+  %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(
+      <vscale x 8 x i16> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask)
+  ret void
+}
+
+define void @masked_scatter_nxv1i32_unscaled_64bit_offsets(<vscale x 4 x i32> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i32_unscaled_64bit_offsets:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uunpklo z0.d, z0.s
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    st1w { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT:    ret
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+  %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(
+      <vscale x 4 x i32> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask)
+  ret void
+}
+
 declare void @llvm.masked.scatter.nxv2f16(<vscale x 2 x half>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
 declare void @llvm.masked.scatter.nxv4f16(<vscale x 4 x half>, <vscale x 4 x ptr>, i32, <vscale x 4 x i1>)
 declare void @llvm.masked.scatter.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
index 8f2cbbdb55636..86bb90e0587f1 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
@@ -73,6 +73,72 @@ define void @masked_scatter_nxv2f64(<vscale x 2 x double> %data, <vscale x 2 x p
   ret void
 }
 
+define void @masked_scatter_nxv1i8(<vscale x 16 x i8> %data.wide, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uunpklo z0.h, z0.b
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    uunpklo z0.d, z0.s
+; CHECK-NEXT:    st1b { z0.d }, p0, [z1.d]
+; CHECK-NEXT:    ret
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+      <vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(
+      <vscale x 16 x i8> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask)
+  ret void
+}
+
+define void @masked_scatter_nxv1i16(<vscale x 8 x i16> %data.wide, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    uunpklo z0.d, z0.s
+; CHECK-NEXT:    st1h { z0.d }, p0, [z1.d]
+; CHECK-NEXT:    ret
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+      <vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(
+      <vscale x 8 x i16> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask)
+  ret void
+}
+
+define void @masked_scatter_nxv1i32(<vscale x 4 x i32> %data.wide, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uunpklo z0.d, z0.s
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    st1w { z0.d }, p0, [z1.d]
+; CHECK-NEXT:    ret
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+      <vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(
+      <vscale x 4 x i32> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask)
+  ret void
+}
+
+define void @masked_scatter_nxv1i64(<vscale x 2 x i64> %data.wide, <vscale x 2 x ptr> %wide.ptrs, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    st1d { z0.d }, p0, [z1.d]
+; CHECK-NEXT:    ret
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
+      <vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
+      <vscale x 2 x i64> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask)
+  ret void
+}
+
 define void @masked_scatter_splat_constant_pointer (<vscale x 4 x i1> %pg) {
 ; CHECK-LABEL: masked_scatter_splat_constant_pointer:
 ; CHECK:       // %bb.0: // %vector.body

>From 3c7a8dfef610ca446ddea9c2d1c4242321e6c824 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Fri, 19 Jun 2026 08:22:24 +0000
Subject: [PATCH 2/2] Tidy up tests

---
 .../AArch64/sve-masked-gather-64b-scaled.ll   |  9 ++---
 .../AArch64/sve-masked-gather-64b-unscaled.ll | 36 ++++++------------
 .../test/CodeGen/AArch64/sve-masked-gather.ll | 36 ++++++------------
 .../AArch64/sve-masked-scatter-64b-scaled.ll  | 14 +++++++
 .../sve-masked-scatter-64b-unscaled.ll        | 38 +++++++++++--------
 .../CodeGen/AArch64/sve-masked-scatter.ll     | 32 ++++++----------
 6 files changed, 76 insertions(+), 89 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
index 72b8496c8d8b7..42838d4ee6d65 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
@@ -103,13 +103,10 @@ define <vscale x 2 x i64> @masked_gather_nxv1i64(ptr %base, <vscale x 2 x i64> %
 ; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d, lsl #3]
 ; CHECK-NEXT:    ret
-  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
-      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
   %ptrs = getelementptr i64, ptr %base, <vscale x 1 x i64> %offsets
-  %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
-      <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
-  %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
-      <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+  %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+  %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(<vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
   ret <vscale x 2 x i64> %r.legal
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
index b6552968a97f3..03c2194e91a2b 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
@@ -139,13 +139,10 @@ define <vscale x 16 x i8> @masked_gather_nxv1i8(ptr %base, <vscale x 2 x i64> %w
 ; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z1.b
 ; CHECK-NEXT:    ret
-  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
-      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
   %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
-  %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(
-      <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
-  %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(
-      <vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
+  %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(<vscale x 1 x ptr> align 1 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
+  %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(<vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
   ret <vscale x 16 x i8> %r.legal
 }
 
@@ -159,13 +156,10 @@ define <vscale x 8 x i16> @masked_gather_nxv1i16(ptr %base, <vscale x 2 x i64> %
 ; CHECK-NEXT:    uzp1 z1.s, z0.s, z0.s
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
-      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
   %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
-  %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(
-      <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
-  %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(
-      <vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
+  %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(<vscale x 1 x ptr> align 2 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
+  %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(<vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
   ret <vscale x 8 x i16> %r.legal
 }
 
@@ -177,13 +171,10 @@ define <vscale x 4 x i32> @masked_gather_nxv1i32(ptr %base, <vscale x 2 x i64> %
 ; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d]
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    ret
-  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
-      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
   %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
-  %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(
-      <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
-  %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(
-      <vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
+  %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(<vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
+  %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(<vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
   ret <vscale x 4 x i32> %r.legal
 }
 
@@ -194,13 +185,10 @@ define <vscale x 2 x i64> @masked_gather_nxv1i64(ptr %base, <vscale x 2 x i64> %
 ; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d]
 ; CHECK-NEXT:    ret
-  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
-      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
   %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
-  %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
-      <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
-  %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
-      <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+  %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+  %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(<vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
   ret <vscale x 2 x i64> %r.legal
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
index 77329371c8e2c..8ae74188ec939 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather.ll
@@ -108,12 +108,9 @@ define <vscale x 16 x i8> @masked_gather_nxv1i8(<vscale x 2 x ptr> %wide.ptrs, <
 ; CHECK-NEXT:    uzp1 z1.h, z1.h, z1.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z1.b
 ; CHECK-NEXT:    ret
-  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
-      <vscale x 2 x ptr> %wide.ptrs, i64 0)
-  %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(
-      <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
-  %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(
-      <vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %r = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8(<vscale x 1 x ptr> align 1 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i8> poison)
+  %r.legal = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.nxv1i8(<vscale x 16 x i8> poison, <vscale x 1 x i8> %r, i64 0)
   ret <vscale x 16 x i8> %r.legal
 }
 
@@ -127,12 +124,9 @@ define <vscale x 8 x i16> @masked_gather_nxv1i16(<vscale x 2 x ptr> %wide.ptrs,
 ; CHECK-NEXT:    uzp1 z1.s, z0.s, z0.s
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
 ; CHECK-NEXT:    ret
-  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
-      <vscale x 2 x ptr> %wide.ptrs, i64 0)
-  %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(
-      <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
-  %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(
-      <vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %r = call <vscale x 1 x i16> @llvm.masked.gather.nxv1i16(<vscale x 1 x ptr> align 2 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i16> poison)
+  %r.legal = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.nxv1i16(<vscale x 8 x i16> poison, <vscale x 1 x i16> %r, i64 0)
   ret <vscale x 8 x i16> %r.legal
 }
 
@@ -144,12 +138,9 @@ define <vscale x 4 x i32> @masked_gather_nxv1i32(<vscale x 2 x ptr> %wide.ptrs,
 ; CHECK-NEXT:    ld1w { z0.d }, p0/z, [z0.d]
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    ret
-  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
-      <vscale x 2 x ptr> %wide.ptrs, i64 0)
-  %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(
-      <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
-  %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(
-      <vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %r = call <vscale x 1 x i32> @llvm.masked.gather.nxv1i32(<vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i32> poison)
+  %r.legal = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.nxv1i32(<vscale x 4 x i32> poison, <vscale x 1 x i32> %r, i64 0)
   ret <vscale x 4 x i32> %r.legal
 }
 
@@ -160,12 +151,9 @@ define <vscale x 2 x i64> @masked_gather_nxv1i64(<vscale x 2 x ptr> %wide.ptrs,
 ; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [z0.d]
 ; CHECK-NEXT:    ret
-  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
-      <vscale x 2 x ptr> %wide.ptrs, i64 0)
-  %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(
-      <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
-  %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(
-      <vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %r = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64(<vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask, <vscale x 1 x i64> poison)
+  %r.legal = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.nxv1i64(<vscale x 2 x i64> poison, <vscale x 1 x i64> %r, i64 0)
   ret <vscale x 2 x i64> %r.legal
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll
index 9244f2c81e799..beae9876c2636 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-scaled.ll
@@ -65,6 +65,20 @@ define void @masked_scatter_nxv2f64(<vscale x 2 x double> %data, ptr %base, <vsc
   ret void
 }
 
+define void @masked_scatter_nxv1i64(<vscale x 2 x i64> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0, z1.d, lsl #3]
+; CHECK-NEXT:    ret
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+  %ptrs = getelementptr i64, ptr %base, <vscale x 1 x i64> %offsets
+  %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
+  ret void
+}
+
 declare void @llvm.masked.scatter.nxv2i16(<vscale x 2 x i16>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
 declare void @llvm.masked.scatter.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
 declare void @llvm.masked.scatter.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x ptr>, i32, <vscale x 2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
index 60eb3edf4d31c..536ddabd5d43b 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter-64b-unscaled.ll
@@ -103,12 +103,10 @@ define void @masked_scatter_nxv1i8_unscaled_64bit_offsets(<vscale x 16 x i8> %da
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
 ; CHECK-NEXT:    st1b { z0.d }, p0, [x0, z1.d]
 ; CHECK-NEXT:    ret
-  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
-      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
   %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
-  %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(
-      <vscale x 16 x i8> %data.wide, i64 0)
-  call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask)
+  %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(<vscale x 16 x i8> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> align 1 %ptrs, <vscale x 1 x i1> %mask)
   ret void
 }
 
@@ -121,12 +119,10 @@ define void @masked_scatter_nxv1i16_unscaled_64bit_offsets(<vscale x 8 x i16> %d
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
 ; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z1.d]
 ; CHECK-NEXT:    ret
-  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
-      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
   %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
-  %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(
-      <vscale x 8 x i16> %data.wide, i64 0)
-  call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask)
+  %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(<vscale x 8 x i16> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> align 2 %ptrs, <vscale x 1 x i1> %mask)
   ret void
 }
 
@@ -138,12 +134,24 @@ define void @masked_scatter_nxv1i32_unscaled_64bit_offsets(<vscale x 4 x i32> %d
 ; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
 ; CHECK-NEXT:    st1w { z0.d }, p0, [x0, z1.d]
 ; CHECK-NEXT:    ret
-  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
-      <vscale x 2 x i64> %wide.offsets, i64 0)
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
   %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
-  %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(
-      <vscale x 4 x i32> %data.wide, i64 0)
-  call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask)
+  %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(<vscale x 4 x i32> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask)
+  ret void
+}
+
+define void @masked_scatter_nxv1i64_unscaled_64bit_offsets(<vscale x 2 x i64> %data.wide, ptr %base, <vscale x 2 x i64> %wide.offsets, <vscale x 1 x i1> %mask) {
+; CHECK-LABEL: masked_scatter_nxv1i64_unscaled_64bit_offsets:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pfalse p1.b
+; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0, z1.d]
+; CHECK-NEXT:    ret
+  %offsets = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %wide.offsets, i64 0)
+  %ptrs = getelementptr i8, ptr %base, <vscale x 1 x i64> %offsets
+  %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
index 86bb90e0587f1..483953c2b23d4 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll
@@ -83,11 +83,9 @@ define void @masked_scatter_nxv1i8(<vscale x 16 x i8> %data.wide, <vscale x 2 x
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
 ; CHECK-NEXT:    st1b { z0.d }, p0, [z1.d]
 ; CHECK-NEXT:    ret
-  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
-      <vscale x 2 x ptr> %wide.ptrs, i64 0)
-  %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(
-      <vscale x 16 x i8> %data.wide, i64 0)
-  call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %mask)
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %data = call <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv16i8(<vscale x 16 x i8> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i8(<vscale x 1 x i8> %data, <vscale x 1 x ptr> align 1 %ptrs, <vscale x 1 x i1> %mask)
   ret void
 }
 
@@ -100,11 +98,9 @@ define void @masked_scatter_nxv1i16(<vscale x 8 x i16> %data.wide, <vscale x 2 x
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
 ; CHECK-NEXT:    st1h { z0.d }, p0, [z1.d]
 ; CHECK-NEXT:    ret
-  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
-      <vscale x 2 x ptr> %wide.ptrs, i64 0)
-  %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(
-      <vscale x 8 x i16> %data.wide, i64 0)
-  call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> %ptrs, i32 2, <vscale x 1 x i1> %mask)
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %data = call <vscale x 1 x i16> @llvm.vector.extract.nxv1i16.nxv8i16(<vscale x 8 x i16> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i16(<vscale x 1 x i16> %data, <vscale x 1 x ptr> align 2 %ptrs, <vscale x 1 x i1> %mask)
   ret void
 }
 
@@ -116,11 +112,9 @@ define void @masked_scatter_nxv1i32(<vscale x 4 x i32> %data.wide, <vscale x 2 x
 ; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
 ; CHECK-NEXT:    st1w { z0.d }, p0, [z1.d]
 ; CHECK-NEXT:    ret
-  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
-      <vscale x 2 x ptr> %wide.ptrs, i64 0)
-  %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(
-      <vscale x 4 x i32> %data.wide, i64 0)
-  call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> %ptrs, i32 4, <vscale x 1 x i1> %mask)
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %data = call <vscale x 1 x i32> @llvm.vector.extract.nxv1i32.nxv4i32(<vscale x 4 x i32> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i32(<vscale x 1 x i32> %data, <vscale x 1 x ptr> align 4 %ptrs, <vscale x 1 x i1> %mask)
   ret void
 }
 
@@ -131,11 +125,9 @@ define void @masked_scatter_nxv1i64(<vscale x 2 x i64> %data.wide, <vscale x 2 x
 ; CHECK-NEXT:    uzp1 p0.d, p0.d, p1.d
 ; CHECK-NEXT:    st1d { z0.d }, p0, [z1.d]
 ; CHECK-NEXT:    ret
-  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(
-      <vscale x 2 x ptr> %wide.ptrs, i64 0)
-  %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(
-      <vscale x 2 x i64> %data.wide, i64 0)
-  call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> %ptrs, i32 8, <vscale x 1 x i1> %mask)
+  %ptrs = call <vscale x 1 x ptr> @llvm.vector.extract.nxv1p0.nxv2p0(<vscale x 2 x ptr> %wide.ptrs, i64 0)
+  %data = call <vscale x 1 x i64> @llvm.vector.extract.nxv1i64.nxv2i64(<vscale x 2 x i64> %data.wide, i64 0)
+  call void @llvm.masked.scatter.nxv1i64(<vscale x 1 x i64> %data, <vscale x 1 x ptr> align 8 %ptrs, <vscale x 1 x i1> %mask)
   ret void
 }
 



More information about the llvm-commits mailing list