[llvm] [AArch64] Forward stores of loaded i1 vectors as integer load/store pairs (PR #210535)
Joel Walker via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 20 04:58:33 PDT 2026
https://github.com/Joel-Wwalker updated https://github.com/llvm/llvm-project/pull/210535
>From 7cf2cb41935f53ac920a86e037f8197138c638f1 Mon Sep 17 00:00:00 2001
From: Joel-Wwalker <theagingboy05 at gmail.com>
Date: Mon, 20 Jul 2026 07:57:08 -0400
Subject: [PATCH 1/2] [AArch64] Add tests for stores of loaded i1 vectors. NFC
---
.../AArch64/i1-vector-store-forward.ll | 156 ++++++++++++++++++
1 file changed, 156 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll
diff --git a/llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll b/llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll
new file mode 100644
index 0000000000000..bd0a53b06e94d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s
+
+; A store of a freshly loaded i1 vector should be forwarded as an integer
+; load/store pair instead of unpacking the bits into a byte vector and
+; repacking them with an addv.
+
+define void @copy_v8i1(ptr %s, ptr %d) {
+; CHECK-LABEL: copy_v8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrb w8, [x0]
+; CHECK-NEXT: sbfx w9, w8, #0, #1
+; CHECK-NEXT: sbfx w10, w8, #1, #1
+; CHECK-NEXT: fmov s0, w9
+; CHECK-NEXT: sbfx w9, w8, #2, #1
+; CHECK-NEXT: mov v0.b[1], w10
+; CHECK-NEXT: mov v0.b[2], w9
+; CHECK-NEXT: sbfx w9, w8, #3, #1
+; CHECK-NEXT: mov v0.b[3], w9
+; CHECK-NEXT: sbfx w9, w8, #4, #1
+; CHECK-NEXT: mov v0.b[4], w9
+; CHECK-NEXT: sbfx w9, w8, #5, #1
+; CHECK-NEXT: mov v0.b[5], w9
+; CHECK-NEXT: sbfx w9, w8, #6, #1
+; CHECK-NEXT: sbfx w8, w8, #7, #1
+; CHECK-NEXT: mov v0.b[6], w9
+; CHECK-NEXT: mov v0.b[7], w8
+; CHECK-NEXT: adrp x8, .LCPI0_0
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: addv b0, v0.8b
+; CHECK-NEXT: str b0, [x1]
+; CHECK-NEXT: ret
+ %ld = load <8 x i1>, ptr %s
+ store <8 x i1> %ld, ptr %d
+ ret void
+}
+
+define void @copy_v16i1(ptr %s, ptr %d) {
+; CHECK-LABEL: copy_v16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrh w8, [x0]
+; CHECK-NEXT: sbfx w9, w8, #0, #1
+; CHECK-NEXT: sbfx w10, w8, #1, #1
+; CHECK-NEXT: fmov s0, w9
+; CHECK-NEXT: sbfx w9, w8, #2, #1
+; CHECK-NEXT: mov v0.b[1], w10
+; CHECK-NEXT: mov v0.b[2], w9
+; CHECK-NEXT: sbfx w9, w8, #3, #1
+; CHECK-NEXT: mov v0.b[3], w9
+; CHECK-NEXT: sbfx w9, w8, #4, #1
+; CHECK-NEXT: mov v0.b[4], w9
+; CHECK-NEXT: sbfx w9, w8, #5, #1
+; CHECK-NEXT: mov v0.b[5], w9
+; CHECK-NEXT: sbfx w9, w8, #6, #1
+; CHECK-NEXT: mov v0.b[6], w9
+; CHECK-NEXT: sbfx w9, w8, #7, #1
+; CHECK-NEXT: mov v0.b[7], w9
+; CHECK-NEXT: sbfx w9, w8, #8, #1
+; CHECK-NEXT: mov v0.b[8], w9
+; CHECK-NEXT: sbfx w9, w8, #9, #1
+; CHECK-NEXT: mov v0.b[9], w9
+; CHECK-NEXT: sbfx w9, w8, #10, #1
+; CHECK-NEXT: mov v0.b[10], w9
+; CHECK-NEXT: sbfx w9, w8, #11, #1
+; CHECK-NEXT: mov v0.b[11], w9
+; CHECK-NEXT: sbfx w9, w8, #12, #1
+; CHECK-NEXT: mov v0.b[12], w9
+; CHECK-NEXT: sbfx w9, w8, #13, #1
+; CHECK-NEXT: mov v0.b[13], w9
+; CHECK-NEXT: sbfx w9, w8, #14, #1
+; CHECK-NEXT: sbfx w8, w8, #15, #1
+; CHECK-NEXT: mov v0.b[14], w9
+; CHECK-NEXT: mov v0.b[15], w8
+; CHECK-NEXT: adrp x8, .LCPI1_0
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
+; CHECK-NEXT: str h0, [x1]
+; CHECK-NEXT: ret
+ %ld = load <16 x i1>, ptr %s
+ store <16 x i1> %ld, ptr %d
+ ret void
+}
+
+; Negative: the loaded value has a second use, so the vector value is
+; needed anyway.
+define <8 x i1> @copy_v8i1_multiuse(ptr %s, ptr %d) {
+; CHECK-LABEL: copy_v8i1_multiuse:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrb w9, [x0]
+; CHECK-NEXT: adrp x8, .LCPI2_0
+; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT: and w10, w9, #0x1
+; CHECK-NEXT: ubfx w11, w9, #1, #1
+; CHECK-NEXT: fmov s0, w10
+; CHECK-NEXT: ubfx w10, w9, #2, #1
+; CHECK-NEXT: mov v0.b[1], w11
+; CHECK-NEXT: mov v0.b[2], w10
+; CHECK-NEXT: ubfx w10, w9, #3, #1
+; CHECK-NEXT: mov v0.b[3], w10
+; CHECK-NEXT: ubfx w10, w9, #4, #1
+; CHECK-NEXT: mov v0.b[4], w10
+; CHECK-NEXT: ubfx w10, w9, #5, #1
+; CHECK-NEXT: mov v0.b[5], w10
+; CHECK-NEXT: ubfx w10, w9, #6, #1
+; CHECK-NEXT: lsr w9, w9, #7
+; CHECK-NEXT: mov v0.b[6], w10
+; CHECK-NEXT: mov v0.b[7], w9
+; CHECK-NEXT: shl v1.8b, v0.8b, #7
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: cmlt v1.8b, v1.8b, #0
+; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: addv b1, v1.8b
+; CHECK-NEXT: str b1, [x1]
+; CHECK-NEXT: ret
+ %ld = load <8 x i1>, ptr %s
+ store <8 x i1> %ld, ptr %d
+ ret <8 x i1> %ld
+}
+
+; Negative: volatile accesses are left alone.
+define void @copy_v8i1_volatile(ptr %s, ptr %d) {
+; CHECK-LABEL: copy_v8i1_volatile:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrb w9, [x0]
+; CHECK-NEXT: adrp x8, .LCPI3_0
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT: and w10, w9, #0x1
+; CHECK-NEXT: ubfx w11, w9, #1, #1
+; CHECK-NEXT: fmov s0, w10
+; CHECK-NEXT: ubfx w10, w9, #2, #1
+; CHECK-NEXT: mov v0.b[1], w11
+; CHECK-NEXT: mov v0.b[2], w10
+; CHECK-NEXT: ubfx w10, w9, #3, #1
+; CHECK-NEXT: mov v0.b[3], w10
+; CHECK-NEXT: ubfx w10, w9, #4, #1
+; CHECK-NEXT: mov v0.b[4], w10
+; CHECK-NEXT: ubfx w10, w9, #5, #1
+; CHECK-NEXT: mov v0.b[5], w10
+; CHECK-NEXT: ubfx w10, w9, #6, #1
+; CHECK-NEXT: lsr w9, w9, #7
+; CHECK-NEXT: mov v0.b[6], w10
+; CHECK-NEXT: mov v0.b[7], w9
+; CHECK-NEXT: shl v0.8b, v0.8b, #7
+; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: addv b0, v0.8b
+; CHECK-NEXT: str b0, [x1]
+; CHECK-NEXT: ret
+ %ld = load volatile <8 x i1>, ptr %s
+ store volatile <8 x i1> %ld, ptr %d
+ ret void
+}
>From f12fe806200fa539db30364fb88106e0e1df7110 Mon Sep 17 00:00:00 2001
From: Joel-Wwalker <theagingboy05 at gmail.com>
Date: Mon, 20 Jul 2026 07:58:04 -0400
Subject: [PATCH 2/2] [AArch64] Forward stores of loaded i1 vectors as integer
load/store pairs
A store of a freshly loaded i1 vector currently round-trips through an
unpack to a byte vector and an addv-based repack: 23 instructions for an
8-lane copy that is just a byte load and store. Forward such stores as an
integer load/store pair in performSTORECombine, before type legalization
expands the value. The memory accesses are unchanged, only the value type
is. Volatile accesses and loads with other uses are left alone, and
non-byte-sized types such as v4i1 are excluded.
Noticed while fixing the GlobalISel side of the same shapes in #210514,
where GlobalISel now produces the 3-instruction form.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 24 +++++++
.../AArch64/i1-vector-store-forward.ll | 62 +------------------
2 files changed, 26 insertions(+), 60 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index a1cf0fd66feb1..1f2cfb28a367a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27412,6 +27412,30 @@ static SDValue performSTORECombine(SDNode *N,
ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
ST->getAAInfo());
+ // A store of a freshly loaded i1 vector round-trips through an unpack to
+ // a byte vector and a repack. Forward it as an integer load/store pair
+ // instead. The memory accesses are unchanged, only the value type is.
+ if (DCI.isBeforeLegalize() && ISD::isNormalStore(N) && !ST->isVolatile() &&
+ ValueVT.isFixedLengthVector() &&
+ ValueVT.getVectorElementType() == MVT::i1 &&
+ ValueVT.getSizeInBits().isKnownMultipleOf(8) &&
+ Value.getOpcode() == ISD::LOAD && Value.hasOneUse()) {
+ LoadSDNode *LD = cast<LoadSDNode>(Value);
+ if (ISD::isNormalLoad(LD) && !LD->isVolatile() &&
+ LD->getMemoryVT() == MemVT) {
+ EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
+ SDValue NewLoad =
+ DAG.getLoad(IntVT, SDLoc(LD), LD->getChain(), LD->getBasePtr(),
+ LD->getPointerInfo(), LD->getBaseAlign(),
+ LD->getMemOperand()->getFlags(), LD->getAAInfo());
+ SDValue StoreChain = Chain.getNode() == LD ? NewLoad.getValue(1) : Chain;
+ DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLoad.getValue(1));
+ return DAG.getStore(StoreChain, DL, NewLoad, Ptr, ST->getPointerInfo(),
+ ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
+ ST->getAAInfo());
+ }
+ }
+
if (SDValue Res = combineStoreValueFPToInt(ST, DCI, DAG, Subtarget))
return Res;
diff --git a/llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll b/llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll
index bd0a53b06e94d..8617e291b1ef9 100644
--- a/llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll
+++ b/llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll
@@ -9,27 +9,7 @@ define void @copy_v8i1(ptr %s, ptr %d) {
; CHECK-LABEL: copy_v8i1:
; CHECK: // %bb.0:
; CHECK-NEXT: ldrb w8, [x0]
-; CHECK-NEXT: sbfx w9, w8, #0, #1
-; CHECK-NEXT: sbfx w10, w8, #1, #1
-; CHECK-NEXT: fmov s0, w9
-; CHECK-NEXT: sbfx w9, w8, #2, #1
-; CHECK-NEXT: mov v0.b[1], w10
-; CHECK-NEXT: mov v0.b[2], w9
-; CHECK-NEXT: sbfx w9, w8, #3, #1
-; CHECK-NEXT: mov v0.b[3], w9
-; CHECK-NEXT: sbfx w9, w8, #4, #1
-; CHECK-NEXT: mov v0.b[4], w9
-; CHECK-NEXT: sbfx w9, w8, #5, #1
-; CHECK-NEXT: mov v0.b[5], w9
-; CHECK-NEXT: sbfx w9, w8, #6, #1
-; CHECK-NEXT: sbfx w8, w8, #7, #1
-; CHECK-NEXT: mov v0.b[6], w9
-; CHECK-NEXT: mov v0.b[7], w8
-; CHECK-NEXT: adrp x8, .LCPI0_0
-; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI0_0]
-; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT: addv b0, v0.8b
-; CHECK-NEXT: str b0, [x1]
+; CHECK-NEXT: strb w8, [x1]
; CHECK-NEXT: ret
%ld = load <8 x i1>, ptr %s
store <8 x i1> %ld, ptr %d
@@ -40,45 +20,7 @@ define void @copy_v16i1(ptr %s, ptr %d) {
; CHECK-LABEL: copy_v16i1:
; CHECK: // %bb.0:
; CHECK-NEXT: ldrh w8, [x0]
-; CHECK-NEXT: sbfx w9, w8, #0, #1
-; CHECK-NEXT: sbfx w10, w8, #1, #1
-; CHECK-NEXT: fmov s0, w9
-; CHECK-NEXT: sbfx w9, w8, #2, #1
-; CHECK-NEXT: mov v0.b[1], w10
-; CHECK-NEXT: mov v0.b[2], w9
-; CHECK-NEXT: sbfx w9, w8, #3, #1
-; CHECK-NEXT: mov v0.b[3], w9
-; CHECK-NEXT: sbfx w9, w8, #4, #1
-; CHECK-NEXT: mov v0.b[4], w9
-; CHECK-NEXT: sbfx w9, w8, #5, #1
-; CHECK-NEXT: mov v0.b[5], w9
-; CHECK-NEXT: sbfx w9, w8, #6, #1
-; CHECK-NEXT: mov v0.b[6], w9
-; CHECK-NEXT: sbfx w9, w8, #7, #1
-; CHECK-NEXT: mov v0.b[7], w9
-; CHECK-NEXT: sbfx w9, w8, #8, #1
-; CHECK-NEXT: mov v0.b[8], w9
-; CHECK-NEXT: sbfx w9, w8, #9, #1
-; CHECK-NEXT: mov v0.b[9], w9
-; CHECK-NEXT: sbfx w9, w8, #10, #1
-; CHECK-NEXT: mov v0.b[10], w9
-; CHECK-NEXT: sbfx w9, w8, #11, #1
-; CHECK-NEXT: mov v0.b[11], w9
-; CHECK-NEXT: sbfx w9, w8, #12, #1
-; CHECK-NEXT: mov v0.b[12], w9
-; CHECK-NEXT: sbfx w9, w8, #13, #1
-; CHECK-NEXT: mov v0.b[13], w9
-; CHECK-NEXT: sbfx w9, w8, #14, #1
-; CHECK-NEXT: sbfx w8, w8, #15, #1
-; CHECK-NEXT: mov v0.b[14], w9
-; CHECK-NEXT: mov v0.b[15], w8
-; CHECK-NEXT: adrp x8, .LCPI1_0
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
-; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: str h0, [x1]
+; CHECK-NEXT: strh w8, [x1]
; CHECK-NEXT: ret
%ld = load <16 x i1>, ptr %s
store <16 x i1> %ld, ptr %d
More information about the llvm-commits
mailing list