[llvm] [AArch64] Forward stores of loaded i1 vectors as integer load/store pairs (PR #210535)
Joel Walker via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 18 12:33:50 PDT 2026
https://github.com/Joel-Wwalker created https://github.com/llvm/llvm-project/pull/210535
A store of a freshly loaded i1 vector currently round-trips through an unpack to a byte vector and an `addv`-based repack — 23 instructions for an 8-lane copy that is just a byte load and store. Forward such stores as an integer load/store pair in `performSTORECombine`, before type legalization expands the value. The memory accesses are unchanged, only the value type is. Volatile accesses and loads with other uses are left alone, and non-byte-sized types such as `v4i1` are excluded.
Noticed while fixing the GlobalISel side of the same shapes in #210514, where GlobalISel now produces the 3-instruction form.
Assisted by Claude (Anthropic).
>From 5e3436ceb977361fc3a4934aa713560546bf5e44 Mon Sep 17 00:00:00 2001
From: Joel-Wwalker <theagingboy05 at gmail.com>
Date: Sat, 18 Jul 2026 15:33:12 -0400
Subject: [PATCH] [AArch64] Forward stores of loaded i1 vectors as integer
load/store pairs
A store of a freshly loaded i1 vector currently round-trips through an
unpack to a byte vector and an addv-based repack: 23 instructions for an
8-lane copy that is just a byte load and store. Forward such stores as an
integer load/store pair in performSTORECombine, before type legalization
expands the value. The memory accesses are unchanged, only the value type
is. Volatile accesses and loads with other uses are left alone, and
non-byte-sized types such as v4i1 are excluded.
Noticed while fixing the GlobalISel side of the same shapes in #210514,
where GlobalISel now produces the 3-instruction form.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 24 +++++
.../AArch64/i1-vector-store-forward.ll | 98 +++++++++++++++++++
2 files changed, 122 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index a1cf0fd66feb1..1f2cfb28a367a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27412,6 +27412,30 @@ static SDValue performSTORECombine(SDNode *N,
ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
ST->getAAInfo());
+ // A store of a freshly loaded i1 vector round-trips through an unpack to
+ // a byte vector and a repack. Forward it as an integer load/store pair
+ // instead. The memory accesses are unchanged, only the value type is.
+ if (DCI.isBeforeLegalize() && ISD::isNormalStore(N) && !ST->isVolatile() &&
+ ValueVT.isFixedLengthVector() &&
+ ValueVT.getVectorElementType() == MVT::i1 &&
+ ValueVT.getSizeInBits().isKnownMultipleOf(8) &&
+ Value.getOpcode() == ISD::LOAD && Value.hasOneUse()) {
+ LoadSDNode *LD = cast<LoadSDNode>(Value);
+ if (ISD::isNormalLoad(LD) && !LD->isVolatile() &&
+ LD->getMemoryVT() == MemVT) {
+ EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
+ SDValue NewLoad =
+ DAG.getLoad(IntVT, SDLoc(LD), LD->getChain(), LD->getBasePtr(),
+ LD->getPointerInfo(), LD->getBaseAlign(),
+ LD->getMemOperand()->getFlags(), LD->getAAInfo());
+ SDValue StoreChain = Chain.getNode() == LD ? NewLoad.getValue(1) : Chain;
+ DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLoad.getValue(1));
+ return DAG.getStore(StoreChain, DL, NewLoad, Ptr, ST->getPointerInfo(),
+ ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
+ ST->getAAInfo());
+ }
+ }
+
if (SDValue Res = combineStoreValueFPToInt(ST, DCI, DAG, Subtarget))
return Res;
diff --git a/llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll b/llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll
new file mode 100644
index 0000000000000..8617e291b1ef9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s
+
+; A store of a freshly loaded i1 vector should be forwarded as an integer
+; load/store pair instead of unpacking the bits into a byte vector and
+; repacking them with an addv.
+
+define void @copy_v8i1(ptr %s, ptr %d) {
+; CHECK-LABEL: copy_v8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrb w8, [x0]
+; CHECK-NEXT: strb w8, [x1]
+; CHECK-NEXT: ret
+ %ld = load <8 x i1>, ptr %s
+ store <8 x i1> %ld, ptr %d
+ ret void
+}
+
+define void @copy_v16i1(ptr %s, ptr %d) {
+; CHECK-LABEL: copy_v16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrh w8, [x0]
+; CHECK-NEXT: strh w8, [x1]
+; CHECK-NEXT: ret
+ %ld = load <16 x i1>, ptr %s
+ store <16 x i1> %ld, ptr %d
+ ret void
+}
+
+; Negative: the loaded value has a second use, so the vector value is
+; needed anyway.
+define <8 x i1> @copy_v8i1_multiuse(ptr %s, ptr %d) {
+; CHECK-LABEL: copy_v8i1_multiuse:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrb w9, [x0]
+; CHECK-NEXT: adrp x8, .LCPI2_0
+; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT: and w10, w9, #0x1
+; CHECK-NEXT: ubfx w11, w9, #1, #1
+; CHECK-NEXT: fmov s0, w10
+; CHECK-NEXT: ubfx w10, w9, #2, #1
+; CHECK-NEXT: mov v0.b[1], w11
+; CHECK-NEXT: mov v0.b[2], w10
+; CHECK-NEXT: ubfx w10, w9, #3, #1
+; CHECK-NEXT: mov v0.b[3], w10
+; CHECK-NEXT: ubfx w10, w9, #4, #1
+; CHECK-NEXT: mov v0.b[4], w10
+; CHECK-NEXT: ubfx w10, w9, #5, #1
+; CHECK-NEXT: mov v0.b[5], w10
+; CHECK-NEXT: ubfx w10, w9, #6, #1
+; CHECK-NEXT: lsr w9, w9, #7
+; CHECK-NEXT: mov v0.b[6], w10
+; CHECK-NEXT: mov v0.b[7], w9
+; CHECK-NEXT: shl v1.8b, v0.8b, #7
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: cmlt v1.8b, v1.8b, #0
+; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: addv b1, v1.8b
+; CHECK-NEXT: str b1, [x1]
+; CHECK-NEXT: ret
+ %ld = load <8 x i1>, ptr %s
+ store <8 x i1> %ld, ptr %d
+ ret <8 x i1> %ld
+}
+
+; Negative: volatile accesses are left alone.
+define void @copy_v8i1_volatile(ptr %s, ptr %d) {
+; CHECK-LABEL: copy_v8i1_volatile:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrb w9, [x0]
+; CHECK-NEXT: adrp x8, .LCPI3_0
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT: and w10, w9, #0x1
+; CHECK-NEXT: ubfx w11, w9, #1, #1
+; CHECK-NEXT: fmov s0, w10
+; CHECK-NEXT: ubfx w10, w9, #2, #1
+; CHECK-NEXT: mov v0.b[1], w11
+; CHECK-NEXT: mov v0.b[2], w10
+; CHECK-NEXT: ubfx w10, w9, #3, #1
+; CHECK-NEXT: mov v0.b[3], w10
+; CHECK-NEXT: ubfx w10, w9, #4, #1
+; CHECK-NEXT: mov v0.b[4], w10
+; CHECK-NEXT: ubfx w10, w9, #5, #1
+; CHECK-NEXT: mov v0.b[5], w10
+; CHECK-NEXT: ubfx w10, w9, #6, #1
+; CHECK-NEXT: lsr w9, w9, #7
+; CHECK-NEXT: mov v0.b[6], w10
+; CHECK-NEXT: mov v0.b[7], w9
+; CHECK-NEXT: shl v0.8b, v0.8b, #7
+; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: addv b0, v0.8b
+; CHECK-NEXT: str b0, [x1]
+; CHECK-NEXT: ret
+ %ld = load volatile <8 x i1>, ptr %s
+ store volatile <8 x i1> %ld, ptr %d
+ ret void
+}
More information about the llvm-commits
mailing list