[llvm] [AArch64][SVE] Add SubtargetFeature to disable lowering unpredicated loads/stores as… (PR #170256)
Kinoshita Kotaro via llvm-commits
llvm-commits at lists.llvm.org
Mon Dec 8 00:19:06 PST 2025
https://github.com/kinoshita-fj updated https://github.com/llvm/llvm-project/pull/170256
>From 15501024fb8d8dd2a09bda1fab388fda74965a90 Mon Sep 17 00:00:00 2001
From: Kinoshita Kotaro <k.kotaro at fujitsu.com>
Date: Thu, 6 Nov 2025 06:24:30 +0000
Subject: [PATCH 1/2] [AArch64][SVE] Add SubtargetFeature to disable lowering
unpredicated loads/stores as LDR/STR
PR #127837 changed the lowering for unpredicated loads/stores to use LDR/STR instead of LD1/ST1.
However, on some CPUs, such as A64FX, there is a performance difference between LD1/ST1 and LDR/STR.
As a result, the lowering introduced in #127837 can cause a performance regression on these targets.
This patch adds a SubtargetFeature to disable this lowering and prevent the regression.
---
llvm/lib/Target/AArch64/AArch64Features.td | 4 +++
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 2 ++
llvm/lib/Target/AArch64/AArch64Processors.td | 3 +-
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 2 +-
...e-disable-unpredicated-load-store-lower.ll | 29 +++++++++++++++++++
5 files changed, 38 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/sve-disable-unpredicated-load-store-lower.ll
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 066724bea92c9..f1baaf82195f9 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -915,6 +915,10 @@ def FeatureUseWzrToVecMove : SubtargetFeature<"use-wzr-to-vec-move",
"UseWzrToVecMove", "true",
"Move from WZR to insert 0 into vector registers">;
+def FeatureDisableUnpredicatedLdStLower : SubtargetFeature<
+ "disable-unpredicated-ld-st-lower", "DisableUnpredicatedLdStLower",
+ "true", "Disable lowering unpredicated loads/stores as LDR/STR">;
+
//===----------------------------------------------------------------------===//
// Architectures.
//
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index da93a2b13fc11..5490ee7201f3b 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -443,6 +443,8 @@ def AllowMisalignedMemAccesses
def UseWzrToVecMove : Predicate<"Subtarget->useWzrToVecMove()">;
+def AllowUnpredicatedLdStLower
+ : Predicate<"!Subtarget->disableUnpredicatedLdStLower()">;
//===----------------------------------------------------------------------===//
// AArch64-specific DAG Nodes.
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index 120415f91c9ae..72882ac078c55 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -306,7 +306,8 @@ def TuneA64FX : SubtargetFeature<"a64fx", "ARMProcFamily", "A64FX",
FeatureAggressiveFMA,
FeatureArithmeticBccFusion,
FeatureStorePairSuppress,
- FeaturePredictableSelectIsExpensive]>;
+ FeaturePredictableSelectIsExpensive,
+ FeatureDisableUnpredicatedLdStLower]>;
def TuneMONAKA : SubtargetFeature<"fujitsu-monaka", "ARMProcFamily", "MONAKA",
"Fujitsu FUJITSU-MONAKA processors", [
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index e99b3f8ff07e0..4d549c6c55d17 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -3164,7 +3164,7 @@ let Predicates = [HasSVE_or_SME] in {
}
// Allow using LDR/STR to avoid the predicate dependence.
- let Predicates = [HasSVE_or_SME, IsLE, AllowMisalignedMemAccesses] in
+ let Predicates = [HasSVE_or_SME, IsLE, AllowMisalignedMemAccesses, AllowUnpredicatedLdStLower] in
foreach Ty = [ nxv16i8, nxv8i16, nxv4i32, nxv2i64, nxv8f16, nxv4f32, nxv2f64, nxv8bf16 ] in {
let AddedComplexity = 2 in {
def : Pat<(Ty (load (am_sve_indexed_s9 GPR64sp:$base, simm9:$offset))),
diff --git a/llvm/test/CodeGen/AArch64/sve-disable-unpredicated-load-store-lower.ll b/llvm/test/CodeGen/AArch64/sve-disable-unpredicated-load-store-lower.ll
new file mode 100644
index 0000000000000..ef9a09666a424
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-disable-unpredicated-load-store-lower.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+disable-unpredicated-ld-st-lower < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck --check-prefix CHECK-DEFAULT %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mcpu=a64fx < %s | FileCheck --check-prefix CHECK-A64FX %s
+
+define void @nxv2i64(ptr %ldptr, ptr %stptr) {
+; CHECK-LABEL: nxv2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT: st1d { z0.d }, p0, [x1]
+; CHECK-NEXT: ret
+;
+; CHECK-DEFAULT-LABEL: nxv2i64:
+; CHECK-DEFAULT: // %bb.0:
+; CHECK-DEFAULT-NEXT: ldr z0, [x0]
+; CHECK-DEFAULT-NEXT: str z0, [x1]
+; CHECK-DEFAULT-NEXT: ret
+;
+; CHECK-A64FX-LABEL: nxv2i64:
+; CHECK-A64FX: // %bb.0:
+; CHECK-A64FX-NEXT: ptrue p0.d
+; CHECK-A64FX-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-A64FX-NEXT: st1d { z0.d }, p0, [x1]
+; CHECK-A64FX-NEXT: ret
+ %l3 = load <vscale x 2 x i64>, ptr %ldptr, align 8
+ store <vscale x 2 x i64> %l3, ptr %stptr, align 8
+ ret void
+}
>From c8b2b62bec90f30b4650ec102b899d52891a6499 Mon Sep 17 00:00:00 2001
From: Kinoshita Kotaro <k.kotaro at fujitsu.com>
Date: Mon, 8 Dec 2025 08:15:59 +0000
Subject: [PATCH 2/2] fixup! [AArch64][SVE] Add SubtargetFeature to disable
lowering unpredicated loads/stores as LDR/STR
---
...e-disable-unpredicated-load-store-lower.ll | 29 -----
.../sve-ld1-addressing-mode-reg-imm.ll | 100 +++++++++++++++++
.../sve-st1-addressing-mode-reg-imm.ll | 102 ++++++++++++++++++
3 files changed, 202 insertions(+), 29 deletions(-)
delete mode 100644 llvm/test/CodeGen/AArch64/sve-disable-unpredicated-load-store-lower.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-disable-unpredicated-load-store-lower.ll b/llvm/test/CodeGen/AArch64/sve-disable-unpredicated-load-store-lower.ll
deleted file mode 100644
index ef9a09666a424..0000000000000
--- a/llvm/test/CodeGen/AArch64/sve-disable-unpredicated-load-store-lower.ll
+++ /dev/null
@@ -1,29 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+disable-unpredicated-ld-st-lower < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck --check-prefix CHECK-DEFAULT %s
-; RUN: llc -mtriple=aarch64-linux-gnu -mcpu=a64fx < %s | FileCheck --check-prefix CHECK-A64FX %s
-
-define void @nxv2i64(ptr %ldptr, ptr %stptr) {
-; CHECK-LABEL: nxv2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
-; CHECK-NEXT: st1d { z0.d }, p0, [x1]
-; CHECK-NEXT: ret
-;
-; CHECK-DEFAULT-LABEL: nxv2i64:
-; CHECK-DEFAULT: // %bb.0:
-; CHECK-DEFAULT-NEXT: ldr z0, [x0]
-; CHECK-DEFAULT-NEXT: str z0, [x1]
-; CHECK-DEFAULT-NEXT: ret
-;
-; CHECK-A64FX-LABEL: nxv2i64:
-; CHECK-A64FX: // %bb.0:
-; CHECK-A64FX-NEXT: ptrue p0.d
-; CHECK-A64FX-NEXT: ld1d { z0.d }, p0/z, [x0]
-; CHECK-A64FX-NEXT: st1d { z0.d }, p0, [x1]
-; CHECK-A64FX-NEXT: ret
- %l3 = load <vscale x 2 x i64>, ptr %ldptr, align 8
- store <vscale x 2 x i64> %l3, ptr %stptr, align 8
- ret void
-}
diff --git a/llvm/test/CodeGen/AArch64/sve-ld1-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-ld1-addressing-mode-reg-imm.ll
index 523fdea6b2231..3e2aed22764c5 100644
--- a/llvm/test/CodeGen/AArch64/sve-ld1-addressing-mode-reg-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-ld1-addressing-mode-reg-imm.ll
@@ -1,5 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+disable-unpredicated-ld-st-lower < %s | FileCheck --check-prefixes=COMMON-NO-UPLS-LOWER,NO-UPLS-LOWER %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mcpu=a64fx < %s | FileCheck --check-prefixes=COMMON-NO-UPLS-LOWER,A64FX %s
; LD1B
@@ -8,6 +10,12 @@ define <vscale x 16 x i8> @ld1b_lower_bound(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldr z0, [x0, #-8, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: ld1b_lower_bound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.b
+; COMMON-NO-UPLS-LOWER-NEXT: ld1b { z0.b }, p0/z, [x0, #-8, mul vl]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 16 x i8>, ptr %a, i64 -8
%load = load <vscale x 16 x i8>, ptr %base
ret <vscale x 16 x i8> %load
@@ -18,6 +26,12 @@ define <vscale x 16 x i8> @ld1b_inbound(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldr z0, [x0, #2, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: ld1b_inbound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.b
+; COMMON-NO-UPLS-LOWER-NEXT: ld1b { z0.b }, p0/z, [x0, #2, mul vl]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 16 x i8>, ptr %a, i64 2
%load = load <vscale x 16 x i8>, ptr %base
ret <vscale x 16 x i8> %load
@@ -28,6 +42,12 @@ define <vscale x 16 x i8> @ld1b_upper_bound(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldr z0, [x0, #7, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: ld1b_upper_bound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.b
+; COMMON-NO-UPLS-LOWER-NEXT: ld1b { z0.b }, p0/z, [x0, #7, mul vl]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 16 x i8>, ptr %a, i64 7
%load = load <vscale x 16 x i8>, ptr %base
ret <vscale x 16 x i8> %load
@@ -38,6 +58,13 @@ define <vscale x 16 x i8> @ld1b_out_of_upper_bound(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldr z0, [x0, #8, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: ld1b_out_of_upper_bound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.b
+; COMMON-NO-UPLS-LOWER-NEXT: rdvl x8, #8
+; COMMON-NO-UPLS-LOWER-NEXT: ld1b { z0.b }, p0/z, [x0, x8]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 16 x i8>, ptr %a, i64 8
%load = load <vscale x 16 x i8>, ptr %base
ret <vscale x 16 x i8> %load
@@ -48,6 +75,13 @@ define <vscale x 16 x i8> @ld1b_out_of_lower_bound(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldr z0, [x0, #-9, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: ld1b_out_of_lower_bound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.b
+; COMMON-NO-UPLS-LOWER-NEXT: rdvl x8, #-9
+; COMMON-NO-UPLS-LOWER-NEXT: ld1b { z0.b }, p0/z, [x0, x8]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 16 x i8>, ptr %a, i64 -9
%load = load <vscale x 16 x i8>, ptr %base
ret <vscale x 16 x i8> %load
@@ -60,6 +94,12 @@ define <vscale x 8 x i16> @ld1h_inbound(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldr z0, [x0, #-2, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: ld1h_inbound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.h
+; COMMON-NO-UPLS-LOWER-NEXT: ld1h { z0.h }, p0/z, [x0, #-2, mul vl]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 8 x i16>, ptr %a, i64 -2
%load = load <vscale x 8 x i16>, ptr %base
ret <vscale x 8 x i16> %load
@@ -72,6 +112,12 @@ define <vscale x 4 x i32> @ld1s_inbound(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldr z0, [x0, #4, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: ld1s_inbound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.s
+; COMMON-NO-UPLS-LOWER-NEXT: ld1w { z0.s }, p0/z, [x0, #4, mul vl]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 4 x i32>, ptr %a, i64 4
%load = load <vscale x 4 x i32>, ptr %base
ret <vscale x 4 x i32> %load
@@ -84,6 +130,12 @@ define <vscale x 2 x i64> @ld1d_inbound(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldr z0, [x0, #6, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: ld1d_inbound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.d
+; COMMON-NO-UPLS-LOWER-NEXT: ld1d { z0.d }, p0/z, [x0, #6, mul vl]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 2 x i64>, ptr %a, i64 6
%load = load <vscale x 2 x i64>, ptr %base
ret <vscale x 2 x i64> %load
@@ -97,6 +149,22 @@ define void @load_nxv6f16(ptr %a) {
; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, #2, mul vl]
; CHECK-NEXT: ld1h { z0.s }, p1/z, [x0]
; CHECK-NEXT: ret
+;
+; NO-UPLS-LOWER-LABEL: load_nxv6f16:
+; NO-UPLS-LOWER: // %bb.0:
+; NO-UPLS-LOWER-NEXT: ptrue p0.d
+; NO-UPLS-LOWER-NEXT: ptrue p1.s
+; NO-UPLS-LOWER-NEXT: ld1h { z0.d }, p0/z, [x0, #2, mul vl]
+; NO-UPLS-LOWER-NEXT: ld1h { z0.s }, p1/z, [x0]
+; NO-UPLS-LOWER-NEXT: ret
+;
+; A64FX-LABEL: load_nxv6f16:
+; A64FX: // %bb.0:
+; A64FX-NEXT: ptrue p0.d
+; A64FX-NEXT: ld1h { z0.d }, p0/z, [x0, #2, mul vl]
+; A64FX-NEXT: ptrue p0.s
+; A64FX-NEXT: ld1h { z0.s }, p0/z, [x0]
+; A64FX-NEXT: ret
%val = load volatile <vscale x 6 x half>, ptr %a
ret void
}
@@ -108,6 +176,22 @@ define void @load_nxv6f32(ptr %a) {
; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0, #2, mul vl]
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ret
+;
+; NO-UPLS-LOWER-LABEL: load_nxv6f32:
+; NO-UPLS-LOWER: // %bb.0:
+; NO-UPLS-LOWER-NEXT: ptrue p0.d
+; NO-UPLS-LOWER-NEXT: ptrue p1.s
+; NO-UPLS-LOWER-NEXT: ld1w { z0.d }, p0/z, [x0, #2, mul vl]
+; NO-UPLS-LOWER-NEXT: ld1w { z0.s }, p1/z, [x0]
+; NO-UPLS-LOWER-NEXT: ret
+;
+; A64FX-LABEL: load_nxv6f32:
+; A64FX: // %bb.0:
+; A64FX-NEXT: ptrue p0.d
+; A64FX-NEXT: ld1w { z0.d }, p0/z, [x0, #2, mul vl]
+; A64FX-NEXT: ptrue p0.s
+; A64FX-NEXT: ld1w { z0.s }, p0/z, [x0]
+; A64FX-NEXT: ret
%val = load volatile <vscale x 6 x float>, ptr %a
ret void
}
@@ -119,6 +203,22 @@ define void @load_nxv12f16(ptr %a) {
; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, #2, mul vl]
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ret
+;
+; NO-UPLS-LOWER-LABEL: load_nxv12f16:
+; NO-UPLS-LOWER: // %bb.0:
+; NO-UPLS-LOWER-NEXT: ptrue p0.s
+; NO-UPLS-LOWER-NEXT: ptrue p1.h
+; NO-UPLS-LOWER-NEXT: ld1h { z0.s }, p0/z, [x0, #2, mul vl]
+; NO-UPLS-LOWER-NEXT: ld1h { z0.h }, p1/z, [x0]
+; NO-UPLS-LOWER-NEXT: ret
+;
+; A64FX-LABEL: load_nxv12f16:
+; A64FX: // %bb.0:
+; A64FX-NEXT: ptrue p0.s
+; A64FX-NEXT: ld1h { z0.s }, p0/z, [x0, #2, mul vl]
+; A64FX-NEXT: ptrue p0.h
+; A64FX-NEXT: ld1h { z0.h }, p0/z, [x0]
+; A64FX-NEXT: ret
%val = load volatile <vscale x 12 x half>, ptr %a
ret void
}
diff --git a/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll
index 71b883f0ef7ec..35f39ce5069bd 100644
--- a/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll
@@ -1,5 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+disable-unpredicated-ld-st-lower < %s | FileCheck --check-prefixes=COMMON-NO-UPLS-LOWER,NO-UPLS-LOWER %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mcpu=a64fx < %s | FileCheck --check-prefixes=COMMON-NO-UPLS-LOWER,A64FX %s
; ST1B
@@ -8,6 +10,12 @@ define void @st1b_lower_bound(<vscale x 16 x i8> %data, ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: str z0, [x0, #-8, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: st1b_lower_bound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.b
+; COMMON-NO-UPLS-LOWER-NEXT: st1b { z0.b }, p0, [x0, #-8, mul vl]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 16 x i8>, ptr %a, i64 -8
store <vscale x 16 x i8> %data, ptr %base
ret void
@@ -18,6 +26,12 @@ define void @st1b_inbound(<vscale x 16 x i8> %data, ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: str z0, [x0, #1, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: st1b_inbound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.b
+; COMMON-NO-UPLS-LOWER-NEXT: st1b { z0.b }, p0, [x0, #1, mul vl]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 16 x i8>, ptr %a, i64 1
store <vscale x 16 x i8> %data, ptr %base
ret void
@@ -28,6 +42,12 @@ define void @st1b_upper_bound(<vscale x 16 x i8> %data, ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: str z0, [x0, #7, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: st1b_upper_bound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.b
+; COMMON-NO-UPLS-LOWER-NEXT: st1b { z0.b }, p0, [x0, #7, mul vl]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 16 x i8>, ptr %a, i64 7
store <vscale x 16 x i8> %data, ptr %base
ret void
@@ -38,6 +58,13 @@ define void @st1b_out_of_upper_bound(<vscale x 16 x i8> %data, ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: str z0, [x0, #8, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: st1b_out_of_upper_bound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.b
+; COMMON-NO-UPLS-LOWER-NEXT: rdvl x8, #8
+; COMMON-NO-UPLS-LOWER-NEXT: st1b { z0.b }, p0, [x0, x8]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 16 x i8>, ptr %a, i64 8
store <vscale x 16 x i8> %data, ptr %base
ret void
@@ -48,6 +75,13 @@ define void @st1b_out_of_lower_bound(<vscale x 16 x i8> %data, ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: str z0, [x0, #-9, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: st1b_out_of_lower_bound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.b
+; COMMON-NO-UPLS-LOWER-NEXT: rdvl x8, #-9
+; COMMON-NO-UPLS-LOWER-NEXT: st1b { z0.b }, p0, [x0, x8]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 16 x i8>, ptr %a, i64 -9
store <vscale x 16 x i8> %data, ptr %base
ret void
@@ -60,6 +94,12 @@ define void @st1h_inbound(<vscale x 8 x i16> %data, ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: str z0, [x0, #-6, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: st1h_inbound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.h
+; COMMON-NO-UPLS-LOWER-NEXT: st1h { z0.h }, p0, [x0, #-6, mul vl]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 8 x i16>, ptr %a, i64 -6
store <vscale x 8 x i16> %data, ptr %base
ret void
@@ -72,6 +112,12 @@ define void @st1w_inbound(<vscale x 4 x i32> %data, ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: str z0, [x0, #2, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: st1w_inbound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.s
+; COMMON-NO-UPLS-LOWER-NEXT: st1w { z0.s }, p0, [x0, #2, mul vl]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 4 x i32>, ptr %a, i64 2
store <vscale x 4 x i32> %data, ptr %base
ret void
@@ -84,6 +130,12 @@ define void @st1d_inbound(<vscale x 2 x i64> %data, ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: str z0, [x0, #5, mul vl]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: st1d_inbound:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.d
+; COMMON-NO-UPLS-LOWER-NEXT: st1d { z0.d }, p0, [x0, #5, mul vl]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
%base = getelementptr <vscale x 2 x i64>, ptr %a, i64 5
store <vscale x 2 x i64> %data, ptr %base
ret void
@@ -99,6 +151,13 @@ define void @store_nxv2f32(ptr %out) {
; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: st1w { z0.d }, p0, [x0]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: store_nxv2f32:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: fmov z0.s, #1.00000000
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.d
+; COMMON-NO-UPLS-LOWER-NEXT: st1w { z0.d }, p0, [x0]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
store <vscale x 2 x float> splat(float 1.0), ptr %out
ret void
}
@@ -110,6 +169,13 @@ define void @store_nxv4f16(ptr %out) {
; CHECK-NEXT: ptrue p0.s
; CHECK-NEXT: st1h { z0.s }, p0, [x0]
; CHECK-NEXT: ret
+;
+; COMMON-NO-UPLS-LOWER-LABEL: store_nxv4f16:
+; COMMON-NO-UPLS-LOWER: // %bb.0:
+; COMMON-NO-UPLS-LOWER-NEXT: fmov z0.h, #1.00000000
+; COMMON-NO-UPLS-LOWER-NEXT: ptrue p0.s
+; COMMON-NO-UPLS-LOWER-NEXT: st1h { z0.s }, p0, [x0]
+; COMMON-NO-UPLS-LOWER-NEXT: ret
store <vscale x 4 x half> splat(half 1.0), ptr %out
ret void
}
@@ -124,6 +190,24 @@ define void @store_nxv6f32(ptr %out) {
; CHECK-NEXT: st1w { z0.d }, p0, [x0, #2, mul vl]
; CHECK-NEXT: str z0, [x0]
; CHECK-NEXT: ret
+;
+; NO-UPLS-LOWER-LABEL: store_nxv6f32:
+; NO-UPLS-LOWER: // %bb.0:
+; NO-UPLS-LOWER-NEXT: fmov z0.s, #1.00000000
+; NO-UPLS-LOWER-NEXT: ptrue p0.d
+; NO-UPLS-LOWER-NEXT: ptrue p1.s
+; NO-UPLS-LOWER-NEXT: st1w { z0.d }, p0, [x0, #2, mul vl]
+; NO-UPLS-LOWER-NEXT: st1w { z0.s }, p1, [x0]
+; NO-UPLS-LOWER-NEXT: ret
+;
+; A64FX-LABEL: store_nxv6f32:
+; A64FX: // %bb.0:
+; A64FX-NEXT: fmov z0.s, #1.00000000
+; A64FX-NEXT: ptrue p0.d
+; A64FX-NEXT: st1w { z0.d }, p0, [x0, #2, mul vl]
+; A64FX-NEXT: ptrue p0.s
+; A64FX-NEXT: st1w { z0.s }, p0, [x0]
+; A64FX-NEXT: ret
store <vscale x 6 x float> splat(float 1.0), ptr %out
ret void
}
@@ -136,6 +220,24 @@ define void @store_nxv12f16(ptr %out) {
; CHECK-NEXT: st1h { z0.s }, p0, [x0, #2, mul vl]
; CHECK-NEXT: str z0, [x0]
; CHECK-NEXT: ret
+;
+; NO-UPLS-LOWER-LABEL: store_nxv12f16:
+; NO-UPLS-LOWER: // %bb.0:
+; NO-UPLS-LOWER-NEXT: fmov z0.h, #1.00000000
+; NO-UPLS-LOWER-NEXT: ptrue p0.s
+; NO-UPLS-LOWER-NEXT: ptrue p1.h
+; NO-UPLS-LOWER-NEXT: st1h { z0.s }, p0, [x0, #2, mul vl]
+; NO-UPLS-LOWER-NEXT: st1h { z0.h }, p1, [x0]
+; NO-UPLS-LOWER-NEXT: ret
+;
+; A64FX-LABEL: store_nxv12f16:
+; A64FX: // %bb.0:
+; A64FX-NEXT: fmov z0.h, #1.00000000
+; A64FX-NEXT: ptrue p0.s
+; A64FX-NEXT: st1h { z0.s }, p0, [x0, #2, mul vl]
+; A64FX-NEXT: ptrue p0.h
+; A64FX-NEXT: st1h { z0.h }, p0, [x0]
+; A64FX-NEXT: ret
store <vscale x 12 x half> splat(half 1.0), ptr %out
ret void
}
More information about the llvm-commits
mailing list