[llvm] [AArch64] Use sve for 64 wide masked load/store (PR #203480)
Yashwant Singh via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 01:13:28 PDT 2026
https://github.com/yashssh created https://github.com/llvm/llvm-project/pull/203480
None
>From cc064f64d5de19b8661b290e0f06d6f99e5f2a21 Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Wed, 10 Jun 2026 23:52:26 -0700
Subject: [PATCH 1/2] [NFC] Add tests for SVE fixed-length 64 wide masked
load/store
---
.../sve-fixed-length-masked-128bit-loads.ll | 85 ------
.../sve-fixed-length-masked-128bit-stores.ll | 69 -----
...sve-fixed-length-masked-64-128bit-loads.ll | 243 ++++++++++++++++++
...ve-fixed-length-masked-64-128bit-stores.ll | 219 ++++++++++++++++
4 files changed, 462 insertions(+), 154 deletions(-)
delete mode 100644 llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll
delete mode 100644 llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll
create mode 100644 llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll
create mode 100644 llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll
deleted file mode 100644
index b2af99bd96ab6..0000000000000
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll
+++ /dev/null
@@ -1,85 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mattr=+sve < %s | FileCheck %s
-
-
-target triple = "aarch64-unknown-linux-gnu"
-
-;
-; Masked Load
-;
-
-define <16 x i8> @masked_load_v16i8(ptr %src, <16 x i1> %mask) {
-; CHECK-LABEL: masked_load_v16i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: shl v0.16b, v0.16b, #7
-; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
-; CHECK-NEXT: ld1b { z0.b }, p1/z, [x0]
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT: ret
- %load = call <16 x i8> @llvm.masked.load.v16i8(ptr %src, i32 8, <16 x i1> %mask, <16 x i8> zeroinitializer)
- ret <16 x i8> %load
-}
-
-define <8 x half> @masked_load_v8f16(ptr %src, <8 x i1> %mask) {
-; CHECK-LABEL: masked_load_v8f16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: shl v0.8h, v0.8h, #15
-; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
-; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0]
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT: ret
- %load = call <8 x half> @llvm.masked.load.v8f16(ptr %src, i32 8, <8 x i1> %mask, <8 x half> zeroinitializer)
- ret <8 x half> %load
-}
-
-define <4 x float> @masked_load_v4f32(ptr %src, <4 x i1> %mask) {
-; CHECK-LABEL: masked_load_v4f32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: shl v0.4s, v0.4s, #31
-; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
-; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0]
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT: ret
- %load = call <4 x float> @llvm.masked.load.v4f32(ptr %src, i32 8, <4 x i1> %mask, <4 x float> zeroinitializer)
- ret <4 x float> %load
-}
-
-define <2 x double> @masked_load_v2f64(ptr %src, <2 x i1> %mask) {
-; CHECK-LABEL: masked_load_v2f64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: shl v0.2d, v0.2d, #63
-; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
-; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0]
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT: ret
- %load = call <2 x double> @llvm.masked.load.v2f64(ptr %src, i32 8, <2 x i1> %mask, <2 x double> zeroinitializer)
- ret <2 x double> %load
-}
-
-define <2 x double> @masked_load_passthru_v2f64(ptr %src, <2 x i1> %mask, <2 x double> %passthru) {
-; CHECK-LABEL: masked_load_passthru_v2f64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: shl v0.2d, v0.2d, #63
-; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
-; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0]
-; CHECK-NEXT: sel z0.d, p1, z0.d, z1.d
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT: ret
- %load = call <2 x double> @llvm.masked.load.v2f64(ptr %src, i32 8, <2 x i1> %mask, <2 x double> %passthru)
- ret <2 x double> %load
-}
-
-declare <16 x i8> @llvm.masked.load.v16i8(ptr, i32, <16 x i1>, <16 x i8>)
-declare <8 x half> @llvm.masked.load.v8f16(ptr, i32, <8 x i1>, <8 x half>)
-declare <4 x float> @llvm.masked.load.v4f32(ptr, i32, <4 x i1>, <4 x float>)
-declare <2 x double> @llvm.masked.load.v2f64(ptr, i32, <2 x i1>, <2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll
deleted file mode 100644
index 660213d1c7f33..0000000000000
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll
+++ /dev/null
@@ -1,69 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mattr=+sve < %s | FileCheck %s
-
-
-target triple = "aarch64-unknown-linux-gnu"
-
-;
-; Masked Store
-;
-
-define void @masked_store_v16i8(ptr %dst, <16 x i1> %mask) {
-; CHECK-LABEL: masked_store_v16i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: shl v0.16b, v0.16b, #7
-; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: st1b { z0.b }, p1, [x0]
-; CHECK-NEXT: ret
- call void @llvm.masked.store.v16i8(<16 x i8> zeroinitializer, ptr %dst, i32 8, <16 x i1> %mask)
- ret void
-}
-
-define void @masked_store_v8f16(ptr %dst, <8 x i1> %mask) {
-; CHECK-LABEL: masked_store_v8f16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: shl v0.8h, v0.8h, #15
-; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: st1h { z0.h }, p1, [x0]
-; CHECK-NEXT: ret
- call void @llvm.masked.store.v8f16(<8 x half> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask)
- ret void
-}
-
-define void @masked_store_v4f32(ptr %dst, <4 x i1> %mask) {
-; CHECK-LABEL: masked_store_v4f32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: shl v0.4s, v0.4s, #31
-; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: st1w { z0.s }, p1, [x0]
-; CHECK-NEXT: ret
- call void @llvm.masked.store.v4f32(<4 x float> zeroinitializer, ptr %dst, i32 8, <4 x i1> %mask)
- ret void
-}
-
-define void @masked_store_v2f64(ptr %dst, <2 x i1> %mask) {
-; CHECK-LABEL: masked_store_v2f64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ushll v0.2d, v0.2s, #0
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: shl v0.2d, v0.2d, #63
-; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: st1d { z0.d }, p1, [x0]
-; CHECK-NEXT: ret
- call void @llvm.masked.store.v2f64(<2 x double> zeroinitializer, ptr %dst, i32 8, <2 x i1> %mask)
- ret void
-}
-
-declare void @llvm.masked.store.v16i8(<16 x i8>, ptr, i32, <16 x i1>)
-declare void @llvm.masked.store.v8f16(<8 x half>, ptr, i32, <8 x i1>)
-declare void @llvm.masked.store.v4f32(<4 x float>, ptr, i32, <4 x i1>)
-declare void @llvm.masked.store.v2f64(<2 x double>, ptr, i32, <2 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll
new file mode 100644
index 0000000000000..e59e44bde90da
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll
@@ -0,0 +1,243 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mattr=+sve < %s | FileCheck %s
+
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;
+; Masked Load
+;
+
+define <16 x i8> @masked_load_v16i8(ptr %src, <16 x i1> %mask) {
+; CHECK-LABEL: masked_load_v16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: ld1b { z0.b }, p1/z, [x0]
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %load = call <16 x i8> @llvm.masked.load.v16i8(ptr %src, i32 8, <16 x i1> %mask, <16 x i8> zeroinitializer)
+ ret <16 x i8> %load
+}
+
+define <8 x half> @masked_load_v8f16(ptr %src, <8 x i1> %mask) {
+; CHECK-LABEL: masked_load_v8f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: shl v0.8h, v0.8h, #15
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0]
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %load = call <8 x half> @llvm.masked.load.v8f16(ptr %src, i32 8, <8 x i1> %mask, <8 x half> zeroinitializer)
+ ret <8 x half> %load
+}
+
+define <4 x float> @masked_load_v4f32(ptr %src, <4 x i1> %mask) {
+; CHECK-LABEL: masked_load_v4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: shl v0.4s, v0.4s, #31
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0]
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %load = call <4 x float> @llvm.masked.load.v4f32(ptr %src, i32 8, <4 x i1> %mask, <4 x float> zeroinitializer)
+ ret <4 x float> %load
+}
+
+define <2 x double> @masked_load_v2f64(ptr %src, <2 x i1> %mask) {
+; CHECK-LABEL: masked_load_v2f64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: shl v0.2d, v0.2d, #63
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0]
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %load = call <2 x double> @llvm.masked.load.v2f64(ptr %src, i32 8, <2 x i1> %mask, <2 x double> zeroinitializer)
+ ret <2 x double> %load
+}
+
+define <2 x double> @masked_load_passthru_v2f64(ptr %src, <2 x i1> %mask, <2 x double> %passthru) {
+; CHECK-LABEL: masked_load_passthru_v2f64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: shl v0.2d, v0.2d, #63
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0]
+; CHECK-NEXT: sel z0.d, p1, z0.d, z1.d
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %load = call <2 x double> @llvm.masked.load.v2f64(ptr %src, i32 8, <2 x i1> %mask, <2 x double> %passthru)
+ ret <2 x double> %load
+}
+
+;
+; 64-bit Masked Loads
+;
+
+define <8 x i8> @masked_load_v8i8(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_load_v8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: adrp x8, .LCPI5_0
+; CHECK-NEXT: cmeq v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI5_0]
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: addv b0, v0.8b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tbz w8, #0, .LBB5_2
+; CHECK-NEXT: // %bb.1: // %cond.load
+; CHECK-NEXT: ldr b0, [x0]
+; CHECK-NEXT: tbnz w8, #1, .LBB5_3
+; CHECK-NEXT: b .LBB5_4
+; CHECK-NEXT: .LBB5_2:
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: tbz w8, #1, .LBB5_4
+; CHECK-NEXT: .LBB5_3: // %cond.load1
+; CHECK-NEXT: add x9, x0, #1
+; CHECK-NEXT: ld1 { v0.b }[1], [x9]
+; CHECK-NEXT: .LBB5_4: // %else2
+; CHECK-NEXT: tbnz w8, #2, .LBB5_11
+; CHECK-NEXT: // %bb.5: // %else5
+; CHECK-NEXT: tbnz w8, #3, .LBB5_12
+; CHECK-NEXT: .LBB5_6: // %else8
+; CHECK-NEXT: tbnz w8, #4, .LBB5_13
+; CHECK-NEXT: .LBB5_7: // %else11
+; CHECK-NEXT: tbnz w8, #5, .LBB5_14
+; CHECK-NEXT: .LBB5_8: // %else14
+; CHECK-NEXT: tbnz w8, #6, .LBB5_15
+; CHECK-NEXT: .LBB5_9: // %else17
+; CHECK-NEXT: tbnz w8, #7, .LBB5_16
+; CHECK-NEXT: .LBB5_10: // %else20
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB5_11: // %cond.load4
+; CHECK-NEXT: add x9, x0, #2
+; CHECK-NEXT: ld1 { v0.b }[2], [x9]
+; CHECK-NEXT: tbz w8, #3, .LBB5_6
+; CHECK-NEXT: .LBB5_12: // %cond.load7
+; CHECK-NEXT: add x9, x0, #3
+; CHECK-NEXT: ld1 { v0.b }[3], [x9]
+; CHECK-NEXT: tbz w8, #4, .LBB5_7
+; CHECK-NEXT: .LBB5_13: // %cond.load10
+; CHECK-NEXT: add x9, x0, #4
+; CHECK-NEXT: ld1 { v0.b }[4], [x9]
+; CHECK-NEXT: tbz w8, #5, .LBB5_8
+; CHECK-NEXT: .LBB5_14: // %cond.load13
+; CHECK-NEXT: add x9, x0, #5
+; CHECK-NEXT: ld1 { v0.b }[5], [x9]
+; CHECK-NEXT: tbz w8, #6, .LBB5_9
+; CHECK-NEXT: .LBB5_15: // %cond.load16
+; CHECK-NEXT: add x9, x0, #6
+; CHECK-NEXT: ld1 { v0.b }[6], [x9]
+; CHECK-NEXT: tbz w8, #7, .LBB5_10
+; CHECK-NEXT: .LBB5_16: // %cond.load19
+; CHECK-NEXT: add x8, x0, #7
+; CHECK-NEXT: ld1 { v0.b }[7], [x8]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
+ %a = load <8 x i8>, ptr %ap
+ %b = load <8 x i8>, ptr %bp
+ %mask = icmp eq <8 x i8> %a, %b
+ %load = call <8 x i8> @llvm.masked.load.v8i8(ptr %ap, i32 1, <8 x i1> %mask, <8 x i8> zeroinitializer)
+ ret <8 x i8> %load
+}
+
+define <4 x i16> @masked_load_v4i16(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_load_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: adrp x8, .LCPI6_0
+; CHECK-NEXT: cmeq v0.4h, v0.4h, v1.4h
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI6_0]
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: addv h0, v0.4h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tbz w8, #0, .LBB6_2
+; CHECK-NEXT: // %bb.1: // %cond.load
+; CHECK-NEXT: ldr h0, [x0]
+; CHECK-NEXT: tbnz w8, #1, .LBB6_3
+; CHECK-NEXT: b .LBB6_4
+; CHECK-NEXT: .LBB6_2:
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: tbz w8, #1, .LBB6_4
+; CHECK-NEXT: .LBB6_3: // %cond.load1
+; CHECK-NEXT: add x9, x0, #2
+; CHECK-NEXT: ld1 { v0.h }[1], [x9]
+; CHECK-NEXT: .LBB6_4: // %else2
+; CHECK-NEXT: tbnz w8, #2, .LBB6_7
+; CHECK-NEXT: // %bb.5: // %else5
+; CHECK-NEXT: tbnz w8, #3, .LBB6_8
+; CHECK-NEXT: .LBB6_6: // %else8
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB6_7: // %cond.load4
+; CHECK-NEXT: add x9, x0, #4
+; CHECK-NEXT: ld1 { v0.h }[2], [x9]
+; CHECK-NEXT: tbz w8, #3, .LBB6_6
+; CHECK-NEXT: .LBB6_8: // %cond.load7
+; CHECK-NEXT: add x8, x0, #6
+; CHECK-NEXT: ld1 { v0.h }[3], [x8]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
+ %a = load <4 x i16>, ptr %ap
+ %b = load <4 x i16>, ptr %bp
+ %mask = icmp eq <4 x i16> %a, %b
+ %load = call <4 x i16> @llvm.masked.load.v4i16(ptr %ap, i32 2, <4 x i1> %mask, <4 x i16> zeroinitializer)
+ ret <4 x i16> %load
+}
+
+define <4 x half> @masked_load_v4f16(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_load_v4f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: adrp x8, .LCPI7_0
+; CHECK-NEXT: fcmeq v0.4h, v0.4h, v1.4h
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI7_0]
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: addv h1, v0.4h
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: fmov w8, s1
+; CHECK-NEXT: tbnz w8, #0, .LBB7_5
+; CHECK-NEXT: // %bb.1: // %else
+; CHECK-NEXT: tbnz w8, #1, .LBB7_6
+; CHECK-NEXT: .LBB7_2: // %else2
+; CHECK-NEXT: tbnz w8, #2, .LBB7_7
+; CHECK-NEXT: .LBB7_3: // %else5
+; CHECK-NEXT: tbnz w8, #3, .LBB7_8
+; CHECK-NEXT: .LBB7_4: // %else8
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB7_5: // %cond.load
+; CHECK-NEXT: ldr h0, [x0]
+; CHECK-NEXT: tbz w8, #1, .LBB7_2
+; CHECK-NEXT: .LBB7_6: // %cond.load1
+; CHECK-NEXT: add x9, x0, #2
+; CHECK-NEXT: ld1 { v0.h }[1], [x9]
+; CHECK-NEXT: tbz w8, #2, .LBB7_3
+; CHECK-NEXT: .LBB7_7: // %cond.load4
+; CHECK-NEXT: add x9, x0, #4
+; CHECK-NEXT: ld1 { v0.h }[2], [x9]
+; CHECK-NEXT: tbz w8, #3, .LBB7_4
+; CHECK-NEXT: .LBB7_8: // %cond.load7
+; CHECK-NEXT: add x8, x0, #6
+; CHECK-NEXT: ld1 { v0.h }[3], [x8]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
+ %a = load <4 x half>, ptr %ap
+ %b = load <4 x half>, ptr %bp
+ %mask = fcmp oeq <4 x half> %a, %b
+ %load = call <4 x half> @llvm.masked.load.v4f16(ptr %ap, i32 2, <4 x i1> %mask, <4 x half> zeroinitializer)
+ ret <4 x half> %load
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll
new file mode 100644
index 0000000000000..22b2f64577025
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll
@@ -0,0 +1,219 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mattr=+sve < %s | FileCheck %s
+
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;
+; Masked Store
+;
+
+define void @masked_store_v16i8(ptr %dst, <16 x i1> %mask) {
+; CHECK-LABEL: masked_store_v16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: st1b { z0.b }, p1, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.v16i8(<16 x i8> zeroinitializer, ptr %dst, i32 8, <16 x i1> %mask)
+ ret void
+}
+
+define void @masked_store_v8f16(ptr %dst, <8 x i1> %mask) {
+; CHECK-LABEL: masked_store_v8f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: shl v0.8h, v0.8h, #15
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: st1h { z0.h }, p1, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.v8f16(<8 x half> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask)
+ ret void
+}
+
+define void @masked_store_v4f32(ptr %dst, <4 x i1> %mask) {
+; CHECK-LABEL: masked_store_v4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: shl v0.4s, v0.4s, #31
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: st1w { z0.s }, p1, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.v4f32(<4 x float> zeroinitializer, ptr %dst, i32 8, <4 x i1> %mask)
+ ret void
+}
+
+define void @masked_store_v2f64(ptr %dst, <2 x i1> %mask) {
+; CHECK-LABEL: masked_store_v2f64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: shl v0.2d, v0.2d, #63
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: st1d { z0.d }, p1, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.v2f64(<2 x double> zeroinitializer, ptr %dst, i32 8, <2 x i1> %mask)
+ ret void
+}
+
+;
+; 64-bit Masked Stores
+;
+
+define void @masked_store_v8i8(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_store_v8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: adrp x8, .LCPI4_0
+; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI4_0]
+; CHECK-NEXT: cmeq v1.8b, v0.8b, v1.8b
+; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: addv b1, v1.8b
+; CHECK-NEXT: fmov w8, s1
+; CHECK-NEXT: tbnz w8, #0, .LBB4_9
+; CHECK-NEXT: // %bb.1: // %else
+; CHECK-NEXT: tbnz w8, #1, .LBB4_10
+; CHECK-NEXT: .LBB4_2: // %else2
+; CHECK-NEXT: tbnz w8, #2, .LBB4_11
+; CHECK-NEXT: .LBB4_3: // %else4
+; CHECK-NEXT: tbnz w8, #3, .LBB4_12
+; CHECK-NEXT: .LBB4_4: // %else6
+; CHECK-NEXT: tbnz w8, #4, .LBB4_13
+; CHECK-NEXT: .LBB4_5: // %else8
+; CHECK-NEXT: tbnz w8, #5, .LBB4_14
+; CHECK-NEXT: .LBB4_6: // %else10
+; CHECK-NEXT: tbnz w8, #6, .LBB4_15
+; CHECK-NEXT: .LBB4_7: // %else12
+; CHECK-NEXT: tbnz w8, #7, .LBB4_16
+; CHECK-NEXT: .LBB4_8: // %else14
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB4_9: // %cond.store
+; CHECK-NEXT: str b0, [x1]
+; CHECK-NEXT: tbz w8, #1, .LBB4_2
+; CHECK-NEXT: .LBB4_10: // %cond.store1
+; CHECK-NEXT: mov b1, v0.b[1]
+; CHECK-NEXT: stur b1, [x1, #1]
+; CHECK-NEXT: tbz w8, #2, .LBB4_3
+; CHECK-NEXT: .LBB4_11: // %cond.store3
+; CHECK-NEXT: mov b1, v0.b[2]
+; CHECK-NEXT: stur b1, [x1, #2]
+; CHECK-NEXT: tbz w8, #3, .LBB4_4
+; CHECK-NEXT: .LBB4_12: // %cond.store5
+; CHECK-NEXT: mov b1, v0.b[3]
+; CHECK-NEXT: stur b1, [x1, #3]
+; CHECK-NEXT: tbz w8, #4, .LBB4_5
+; CHECK-NEXT: .LBB4_13: // %cond.store7
+; CHECK-NEXT: mov b1, v0.b[4]
+; CHECK-NEXT: stur b1, [x1, #4]
+; CHECK-NEXT: tbz w8, #5, .LBB4_6
+; CHECK-NEXT: .LBB4_14: // %cond.store9
+; CHECK-NEXT: mov b1, v0.b[5]
+; CHECK-NEXT: stur b1, [x1, #5]
+; CHECK-NEXT: tbz w8, #6, .LBB4_7
+; CHECK-NEXT: .LBB4_15: // %cond.store11
+; CHECK-NEXT: mov b1, v0.b[6]
+; CHECK-NEXT: stur b1, [x1, #6]
+; CHECK-NEXT: tbz w8, #7, .LBB4_8
+; CHECK-NEXT: .LBB4_16: // %cond.store13
+; CHECK-NEXT: mov b0, v0.b[7]
+; CHECK-NEXT: stur b0, [x1, #7]
+; CHECK-NEXT: ret
+ %a = load <8 x i8>, ptr %ap
+ %b = load <8 x i8>, ptr %bp
+ %mask = icmp eq <8 x i8> %a, %b
+ call void @llvm.masked.store.v8i8(<8 x i8> %a, ptr %bp, i32 1, <8 x i1> %mask)
+ ret void
+}
+
+define void @masked_store_v4i16(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_store_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: adrp x8, .LCPI5_0
+; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI5_0]
+; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h
+; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: addv h1, v1.4h
+; CHECK-NEXT: fmov w8, s1
+; CHECK-NEXT: tbnz w8, #0, .LBB5_5
+; CHECK-NEXT: // %bb.1: // %else
+; CHECK-NEXT: tbnz w8, #1, .LBB5_6
+; CHECK-NEXT: .LBB5_2: // %else2
+; CHECK-NEXT: tbnz w8, #2, .LBB5_7
+; CHECK-NEXT: .LBB5_3: // %else4
+; CHECK-NEXT: tbnz w8, #3, .LBB5_8
+; CHECK-NEXT: .LBB5_4: // %else6
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB5_5: // %cond.store
+; CHECK-NEXT: str h0, [x1]
+; CHECK-NEXT: tbz w8, #1, .LBB5_2
+; CHECK-NEXT: .LBB5_6: // %cond.store1
+; CHECK-NEXT: mov h1, v0.h[1]
+; CHECK-NEXT: str h1, [x1, #2]
+; CHECK-NEXT: tbz w8, #2, .LBB5_3
+; CHECK-NEXT: .LBB5_7: // %cond.store3
+; CHECK-NEXT: mov h1, v0.h[2]
+; CHECK-NEXT: str h1, [x1, #4]
+; CHECK-NEXT: tbz w8, #3, .LBB5_4
+; CHECK-NEXT: .LBB5_8: // %cond.store5
+; CHECK-NEXT: mov h0, v0.h[3]
+; CHECK-NEXT: str h0, [x1, #6]
+; CHECK-NEXT: ret
+ %a = load <4 x i16>, ptr %ap
+ %b = load <4 x i16>, ptr %bp
+ %mask = icmp eq <4 x i16> %a, %b
+ call void @llvm.masked.store.v4i16(<4 x i16> %a, ptr %bp, i32 2, <4 x i1> %mask)
+ ret void
+}
+
+define void @masked_store_v4f16(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_store_v4f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: adrp x8, .LCPI6_0
+; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI6_0]
+; CHECK-NEXT: fcmeq v1.4h, v0.4h, v1.4h
+; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: addv h1, v1.4h
+; CHECK-NEXT: fmov w8, s1
+; CHECK-NEXT: tbnz w8, #0, .LBB6_5
+; CHECK-NEXT: // %bb.1: // %else
+; CHECK-NEXT: tbnz w8, #1, .LBB6_6
+; CHECK-NEXT: .LBB6_2: // %else2
+; CHECK-NEXT: tbnz w8, #2, .LBB6_7
+; CHECK-NEXT: .LBB6_3: // %else4
+; CHECK-NEXT: tbnz w8, #3, .LBB6_8
+; CHECK-NEXT: .LBB6_4: // %else6
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB6_5: // %cond.store
+; CHECK-NEXT: str h0, [x1]
+; CHECK-NEXT: tbz w8, #1, .LBB6_2
+; CHECK-NEXT: .LBB6_6: // %cond.store1
+; CHECK-NEXT: add x9, x1, #2
+; CHECK-NEXT: st1 { v0.h }[1], [x9]
+; CHECK-NEXT: tbz w8, #2, .LBB6_3
+; CHECK-NEXT: .LBB6_7: // %cond.store3
+; CHECK-NEXT: add x9, x1, #4
+; CHECK-NEXT: st1 { v0.h }[2], [x9]
+; CHECK-NEXT: tbz w8, #3, .LBB6_4
+; CHECK-NEXT: .LBB6_8: // %cond.store5
+; CHECK-NEXT: add x8, x1, #6
+; CHECK-NEXT: st1 { v0.h }[3], [x8]
+; CHECK-NEXT: ret
+ %a = load <4 x half>, ptr %ap
+ %b = load <4 x half>, ptr %bp
+ %mask = fcmp oeq <4 x half> %a, %b
+ call void @llvm.masked.store.v4f16(<4 x half> %a, ptr %bp, i32 2, <4 x i1> %mask)
+ ret void
+}
+
>From 437811e6858a14659b4ce6770d6904f5462bd15a Mon Sep 17 00:00:00 2001
From: Yashwant Singh <yashwants at nvidia.com>
Date: Wed, 10 Jun 2026 23:53:57 -0700
Subject: [PATCH 2/2] [AArch64] Use SVE for lowering 64 wide masked load/store
---
.../AArch64/AArch64TargetTransformInfo.h | 9 +-
...sve-fixed-length-masked-64-128bit-loads.ll | 131 ++----------------
...ve-fixed-length-masked-64-128bit-stores.ll | 119 ++--------------
3 files changed, 26 insertions(+), 233 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 200b4c9fba196..68af38aed185f 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -315,10 +315,11 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
if (!ST->isSVEorStreamingSVEAvailable())
return false;
- // For fixed vectors, avoid scalarization if using SVE for them.
- if (isa<FixedVectorType>(DataType) && !ST->useSVEForFixedLengthVectors() &&
- DataType->getPrimitiveSizeInBits() != 128)
- return false; // Fall back to scalarization of masked operations.
+ if (isa<FixedVectorType>(DataType) && !ST->useSVEForFixedLengthVectors()) {
+ unsigned Bits = DataType->getPrimitiveSizeInBits();
+ if (Bits != 64 && Bits != 128)
+ return false; // Fall back to scalarization of masked operations.
+ }
return isElementTypeLegalForScalableVector(DataType->getScalarType());
}
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll
index e59e44bde90da..91af81c617c41 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll
@@ -86,64 +86,12 @@ define <2 x double> @masked_load_passthru_v2f64(ptr %src, <2 x i1> %mask, <2 x d
define <8 x i8> @masked_load_v8i8(ptr %ap, ptr %bp) {
; CHECK-LABEL: masked_load_v8i8:
; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl8
; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ldr d1, [x1]
-; CHECK-NEXT: adrp x8, .LCPI5_0
-; CHECK-NEXT: cmeq v0.8b, v0.8b, v1.8b
-; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI5_0]
-; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT: addv b0, v0.8b
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: tbz w8, #0, .LBB5_2
-; CHECK-NEXT: // %bb.1: // %cond.load
-; CHECK-NEXT: ldr b0, [x0]
-; CHECK-NEXT: tbnz w8, #1, .LBB5_3
-; CHECK-NEXT: b .LBB5_4
-; CHECK-NEXT: .LBB5_2:
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: tbz w8, #1, .LBB5_4
-; CHECK-NEXT: .LBB5_3: // %cond.load1
-; CHECK-NEXT: add x9, x0, #1
-; CHECK-NEXT: ld1 { v0.b }[1], [x9]
-; CHECK-NEXT: .LBB5_4: // %else2
-; CHECK-NEXT: tbnz w8, #2, .LBB5_11
-; CHECK-NEXT: // %bb.5: // %else5
-; CHECK-NEXT: tbnz w8, #3, .LBB5_12
-; CHECK-NEXT: .LBB5_6: // %else8
-; CHECK-NEXT: tbnz w8, #4, .LBB5_13
-; CHECK-NEXT: .LBB5_7: // %else11
-; CHECK-NEXT: tbnz w8, #5, .LBB5_14
-; CHECK-NEXT: .LBB5_8: // %else14
-; CHECK-NEXT: tbnz w8, #6, .LBB5_15
-; CHECK-NEXT: .LBB5_9: // %else17
-; CHECK-NEXT: tbnz w8, #7, .LBB5_16
-; CHECK-NEXT: .LBB5_10: // %else20
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB5_11: // %cond.load4
-; CHECK-NEXT: add x9, x0, #2
-; CHECK-NEXT: ld1 { v0.b }[2], [x9]
-; CHECK-NEXT: tbz w8, #3, .LBB5_6
-; CHECK-NEXT: .LBB5_12: // %cond.load7
-; CHECK-NEXT: add x9, x0, #3
-; CHECK-NEXT: ld1 { v0.b }[3], [x9]
-; CHECK-NEXT: tbz w8, #4, .LBB5_7
-; CHECK-NEXT: .LBB5_13: // %cond.load10
-; CHECK-NEXT: add x9, x0, #4
-; CHECK-NEXT: ld1 { v0.b }[4], [x9]
-; CHECK-NEXT: tbz w8, #5, .LBB5_8
-; CHECK-NEXT: .LBB5_14: // %cond.load13
-; CHECK-NEXT: add x9, x0, #5
-; CHECK-NEXT: ld1 { v0.b }[5], [x9]
-; CHECK-NEXT: tbz w8, #6, .LBB5_9
-; CHECK-NEXT: .LBB5_15: // %cond.load16
-; CHECK-NEXT: add x9, x0, #6
-; CHECK-NEXT: ld1 { v0.b }[6], [x9]
-; CHECK-NEXT: tbz w8, #7, .LBB5_10
-; CHECK-NEXT: .LBB5_16: // %cond.load19
-; CHECK-NEXT: add x8, x0, #7
-; CHECK-NEXT: ld1 { v0.b }[7], [x8]
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ld1b { z0.b }, p1/z, [x0]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%a = load <8 x i8>, ptr %ap
%b = load <8 x i8>, ptr %bp
@@ -155,40 +103,12 @@ define <8 x i8> @masked_load_v8i8(ptr %ap, ptr %bp) {
define <4 x i16> @masked_load_v4i16(ptr %ap, ptr %bp) {
; CHECK-LABEL: masked_load_v4i16:
; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl4
; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ldr d1, [x1]
-; CHECK-NEXT: adrp x8, .LCPI6_0
-; CHECK-NEXT: cmeq v0.4h, v0.4h, v1.4h
-; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI6_0]
-; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT: addv h0, v0.4h
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: tbz w8, #0, .LBB6_2
-; CHECK-NEXT: // %bb.1: // %cond.load
-; CHECK-NEXT: ldr h0, [x0]
-; CHECK-NEXT: tbnz w8, #1, .LBB6_3
-; CHECK-NEXT: b .LBB6_4
-; CHECK-NEXT: .LBB6_2:
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: tbz w8, #1, .LBB6_4
-; CHECK-NEXT: .LBB6_3: // %cond.load1
-; CHECK-NEXT: add x9, x0, #2
-; CHECK-NEXT: ld1 { v0.h }[1], [x9]
-; CHECK-NEXT: .LBB6_4: // %else2
-; CHECK-NEXT: tbnz w8, #2, .LBB6_7
-; CHECK-NEXT: // %bb.5: // %else5
-; CHECK-NEXT: tbnz w8, #3, .LBB6_8
-; CHECK-NEXT: .LBB6_6: // %else8
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB6_7: // %cond.load4
-; CHECK-NEXT: add x9, x0, #4
-; CHECK-NEXT: ld1 { v0.h }[2], [x9]
-; CHECK-NEXT: tbz w8, #3, .LBB6_6
-; CHECK-NEXT: .LBB6_8: // %cond.load7
-; CHECK-NEXT: add x8, x0, #6
-; CHECK-NEXT: ld1 { v0.h }[3], [x8]
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%a = load <4 x i16>, ptr %ap
%b = load <4 x i16>, ptr %bp
@@ -202,38 +122,11 @@ define <4 x half> @masked_load_v4f16(ptr %ap, ptr %bp) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ldr d1, [x1]
-; CHECK-NEXT: adrp x8, .LCPI7_0
+; CHECK-NEXT: ptrue p0.h, vl4
; CHECK-NEXT: fcmeq v0.4h, v0.4h, v1.4h
-; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI7_0]
-; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT: addv h1, v0.4h
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: fmov w8, s1
-; CHECK-NEXT: tbnz w8, #0, .LBB7_5
-; CHECK-NEXT: // %bb.1: // %else
-; CHECK-NEXT: tbnz w8, #1, .LBB7_6
-; CHECK-NEXT: .LBB7_2: // %else2
-; CHECK-NEXT: tbnz w8, #2, .LBB7_7
-; CHECK-NEXT: .LBB7_3: // %else5
-; CHECK-NEXT: tbnz w8, #3, .LBB7_8
-; CHECK-NEXT: .LBB7_4: // %else8
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB7_5: // %cond.load
-; CHECK-NEXT: ldr h0, [x0]
-; CHECK-NEXT: tbz w8, #1, .LBB7_2
-; CHECK-NEXT: .LBB7_6: // %cond.load1
-; CHECK-NEXT: add x9, x0, #2
-; CHECK-NEXT: ld1 { v0.h }[1], [x9]
-; CHECK-NEXT: tbz w8, #2, .LBB7_3
-; CHECK-NEXT: .LBB7_7: // %cond.load4
-; CHECK-NEXT: add x9, x0, #4
-; CHECK-NEXT: ld1 { v0.h }[2], [x9]
-; CHECK-NEXT: tbz w8, #3, .LBB7_4
-; CHECK-NEXT: .LBB7_8: // %cond.load7
-; CHECK-NEXT: add x8, x0, #6
-; CHECK-NEXT: ld1 { v0.h }[3], [x8]
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%a = load <4 x half>, ptr %ap
%b = load <4 x half>, ptr %bp
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll
index 22b2f64577025..439c61339a631 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll
@@ -70,61 +70,11 @@ define void @masked_store_v2f64(ptr %dst, <2 x i1> %mask) {
define void @masked_store_v8i8(ptr %ap, ptr %bp) {
; CHECK-LABEL: masked_store_v8i8:
; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl8
; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ldr d1, [x1]
-; CHECK-NEXT: adrp x8, .LCPI4_0
-; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI4_0]
-; CHECK-NEXT: cmeq v1.8b, v0.8b, v1.8b
-; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
-; CHECK-NEXT: addv b1, v1.8b
-; CHECK-NEXT: fmov w8, s1
-; CHECK-NEXT: tbnz w8, #0, .LBB4_9
-; CHECK-NEXT: // %bb.1: // %else
-; CHECK-NEXT: tbnz w8, #1, .LBB4_10
-; CHECK-NEXT: .LBB4_2: // %else2
-; CHECK-NEXT: tbnz w8, #2, .LBB4_11
-; CHECK-NEXT: .LBB4_3: // %else4
-; CHECK-NEXT: tbnz w8, #3, .LBB4_12
-; CHECK-NEXT: .LBB4_4: // %else6
-; CHECK-NEXT: tbnz w8, #4, .LBB4_13
-; CHECK-NEXT: .LBB4_5: // %else8
-; CHECK-NEXT: tbnz w8, #5, .LBB4_14
-; CHECK-NEXT: .LBB4_6: // %else10
-; CHECK-NEXT: tbnz w8, #6, .LBB4_15
-; CHECK-NEXT: .LBB4_7: // %else12
-; CHECK-NEXT: tbnz w8, #7, .LBB4_16
-; CHECK-NEXT: .LBB4_8: // %else14
-; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB4_9: // %cond.store
-; CHECK-NEXT: str b0, [x1]
-; CHECK-NEXT: tbz w8, #1, .LBB4_2
-; CHECK-NEXT: .LBB4_10: // %cond.store1
-; CHECK-NEXT: mov b1, v0.b[1]
-; CHECK-NEXT: stur b1, [x1, #1]
-; CHECK-NEXT: tbz w8, #2, .LBB4_3
-; CHECK-NEXT: .LBB4_11: // %cond.store3
-; CHECK-NEXT: mov b1, v0.b[2]
-; CHECK-NEXT: stur b1, [x1, #2]
-; CHECK-NEXT: tbz w8, #3, .LBB4_4
-; CHECK-NEXT: .LBB4_12: // %cond.store5
-; CHECK-NEXT: mov b1, v0.b[3]
-; CHECK-NEXT: stur b1, [x1, #3]
-; CHECK-NEXT: tbz w8, #4, .LBB4_5
-; CHECK-NEXT: .LBB4_13: // %cond.store7
-; CHECK-NEXT: mov b1, v0.b[4]
-; CHECK-NEXT: stur b1, [x1, #4]
-; CHECK-NEXT: tbz w8, #5, .LBB4_6
-; CHECK-NEXT: .LBB4_14: // %cond.store9
-; CHECK-NEXT: mov b1, v0.b[5]
-; CHECK-NEXT: stur b1, [x1, #5]
-; CHECK-NEXT: tbz w8, #6, .LBB4_7
-; CHECK-NEXT: .LBB4_15: // %cond.store11
-; CHECK-NEXT: mov b1, v0.b[6]
-; CHECK-NEXT: stur b1, [x1, #6]
-; CHECK-NEXT: tbz w8, #7, .LBB4_8
-; CHECK-NEXT: .LBB4_16: // %cond.store13
-; CHECK-NEXT: mov b0, v0.b[7]
-; CHECK-NEXT: stur b0, [x1, #7]
+; CHECK-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: st1b { z0.b }, p1, [x1]
; CHECK-NEXT: ret
%a = load <8 x i8>, ptr %ap
%b = load <8 x i8>, ptr %bp
@@ -136,37 +86,11 @@ define void @masked_store_v8i8(ptr %ap, ptr %bp) {
define void @masked_store_v4i16(ptr %ap, ptr %bp) {
; CHECK-LABEL: masked_store_v4i16:
; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl4
; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ldr d1, [x1]
-; CHECK-NEXT: adrp x8, .LCPI5_0
-; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI5_0]
-; CHECK-NEXT: cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
-; CHECK-NEXT: addv h1, v1.4h
-; CHECK-NEXT: fmov w8, s1
-; CHECK-NEXT: tbnz w8, #0, .LBB5_5
-; CHECK-NEXT: // %bb.1: // %else
-; CHECK-NEXT: tbnz w8, #1, .LBB5_6
-; CHECK-NEXT: .LBB5_2: // %else2
-; CHECK-NEXT: tbnz w8, #2, .LBB5_7
-; CHECK-NEXT: .LBB5_3: // %else4
-; CHECK-NEXT: tbnz w8, #3, .LBB5_8
-; CHECK-NEXT: .LBB5_4: // %else6
-; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB5_5: // %cond.store
-; CHECK-NEXT: str h0, [x1]
-; CHECK-NEXT: tbz w8, #1, .LBB5_2
-; CHECK-NEXT: .LBB5_6: // %cond.store1
-; CHECK-NEXT: mov h1, v0.h[1]
-; CHECK-NEXT: str h1, [x1, #2]
-; CHECK-NEXT: tbz w8, #2, .LBB5_3
-; CHECK-NEXT: .LBB5_7: // %cond.store3
-; CHECK-NEXT: mov h1, v0.h[2]
-; CHECK-NEXT: str h1, [x1, #4]
-; CHECK-NEXT: tbz w8, #3, .LBB5_4
-; CHECK-NEXT: .LBB5_8: // %cond.store5
-; CHECK-NEXT: mov h0, v0.h[3]
-; CHECK-NEXT: str h0, [x1, #6]
+; CHECK-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: st1h { z0.h }, p1, [x1]
; CHECK-NEXT: ret
%a = load <4 x i16>, ptr %ap
%b = load <4 x i16>, ptr %bp
@@ -180,35 +104,10 @@ define void @masked_store_v4f16(ptr %ap, ptr %bp) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: ldr d1, [x1]
-; CHECK-NEXT: adrp x8, .LCPI6_0
-; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI6_0]
+; CHECK-NEXT: ptrue p0.h, vl4
; CHECK-NEXT: fcmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
-; CHECK-NEXT: addv h1, v1.4h
-; CHECK-NEXT: fmov w8, s1
-; CHECK-NEXT: tbnz w8, #0, .LBB6_5
-; CHECK-NEXT: // %bb.1: // %else
-; CHECK-NEXT: tbnz w8, #1, .LBB6_6
-; CHECK-NEXT: .LBB6_2: // %else2
-; CHECK-NEXT: tbnz w8, #2, .LBB6_7
-; CHECK-NEXT: .LBB6_3: // %else4
-; CHECK-NEXT: tbnz w8, #3, .LBB6_8
-; CHECK-NEXT: .LBB6_4: // %else6
-; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB6_5: // %cond.store
-; CHECK-NEXT: str h0, [x1]
-; CHECK-NEXT: tbz w8, #1, .LBB6_2
-; CHECK-NEXT: .LBB6_6: // %cond.store1
-; CHECK-NEXT: add x9, x1, #2
-; CHECK-NEXT: st1 { v0.h }[1], [x9]
-; CHECK-NEXT: tbz w8, #2, .LBB6_3
-; CHECK-NEXT: .LBB6_7: // %cond.store3
-; CHECK-NEXT: add x9, x1, #4
-; CHECK-NEXT: st1 { v0.h }[2], [x9]
-; CHECK-NEXT: tbz w8, #3, .LBB6_4
-; CHECK-NEXT: .LBB6_8: // %cond.store5
-; CHECK-NEXT: add x8, x1, #6
-; CHECK-NEXT: st1 { v0.h }[3], [x8]
+; CHECK-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-NEXT: st1h { z0.h }, p1, [x1]
; CHECK-NEXT: ret
%a = load <4 x half>, ptr %ap
%b = load <4 x half>, ptr %bp
More information about the llvm-commits
mailing list