[llvm] [AArch64] Forward stores of loaded i1 vectors as integer load/store pairs (PR #210535)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 18 12:34:27 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Joel Walker (Joel-Wwalker)
<details>
<summary>Changes</summary>
A store of a freshly loaded i1 vector currently round-trips through an unpack to a byte vector and an `addv`-based repack — 23 instructions for an 8-lane copy that is just a byte load and store. Forward such stores as an integer load/store pair in `performSTORECombine`, before type legalization expands the value. The memory accesses are unchanged, only the value type is. Volatile accesses and loads with other uses are left alone, and non-byte-sized types such as `v4i1` are excluded.
Noticed while fixing the GlobalISel side of the same shapes in #<!-- -->210514, where GlobalISel now produces the 3-instruction form.
Assisted by Claude (Anthropic).
---
Full diff: https://github.com/llvm/llvm-project/pull/210535.diff
2 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+24)
- (added) llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll (+98)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index a1cf0fd66feb1..1f2cfb28a367a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27412,6 +27412,30 @@ static SDValue performSTORECombine(SDNode *N,
ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
ST->getAAInfo());
+ // A store of a freshly loaded i1 vector round-trips through an unpack to
+ // a byte vector and a repack. Forward it as an integer load/store pair
+ // instead. The memory accesses are unchanged, only the value type is.
+ if (DCI.isBeforeLegalize() && ISD::isNormalStore(N) && !ST->isVolatile() &&
+ ValueVT.isFixedLengthVector() &&
+ ValueVT.getVectorElementType() == MVT::i1 &&
+ ValueVT.getSizeInBits().isKnownMultipleOf(8) &&
+ Value.getOpcode() == ISD::LOAD && Value.hasOneUse()) {
+ LoadSDNode *LD = cast<LoadSDNode>(Value);
+ if (ISD::isNormalLoad(LD) && !LD->isVolatile() &&
+ LD->getMemoryVT() == MemVT) {
+ EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
+ SDValue NewLoad =
+ DAG.getLoad(IntVT, SDLoc(LD), LD->getChain(), LD->getBasePtr(),
+ LD->getPointerInfo(), LD->getBaseAlign(),
+ LD->getMemOperand()->getFlags(), LD->getAAInfo());
+ SDValue StoreChain = Chain.getNode() == LD ? NewLoad.getValue(1) : Chain;
+ DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLoad.getValue(1));
+ return DAG.getStore(StoreChain, DL, NewLoad, Ptr, ST->getPointerInfo(),
+ ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
+ ST->getAAInfo());
+ }
+ }
+
if (SDValue Res = combineStoreValueFPToInt(ST, DCI, DAG, Subtarget))
return Res;
diff --git a/llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll b/llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll
new file mode 100644
index 0000000000000..8617e291b1ef9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/i1-vector-store-forward.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s
+
+; A store of a freshly loaded i1 vector should be forwarded as an integer
+; load/store pair instead of unpacking the bits into a byte vector and
+; repacking them with an addv.
+
+define void @copy_v8i1(ptr %s, ptr %d) {
+; CHECK-LABEL: copy_v8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrb w8, [x0]
+; CHECK-NEXT: strb w8, [x1]
+; CHECK-NEXT: ret
+ %ld = load <8 x i1>, ptr %s
+ store <8 x i1> %ld, ptr %d
+ ret void
+}
+
+define void @copy_v16i1(ptr %s, ptr %d) {
+; CHECK-LABEL: copy_v16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrh w8, [x0]
+; CHECK-NEXT: strh w8, [x1]
+; CHECK-NEXT: ret
+ %ld = load <16 x i1>, ptr %s
+ store <16 x i1> %ld, ptr %d
+ ret void
+}
+
+; Negative: the loaded value has a second use, so the vector value is
+; needed anyway.
+define <8 x i1> @copy_v8i1_multiuse(ptr %s, ptr %d) {
+; CHECK-LABEL: copy_v8i1_multiuse:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrb w9, [x0]
+; CHECK-NEXT: adrp x8, .LCPI2_0
+; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT: and w10, w9, #0x1
+; CHECK-NEXT: ubfx w11, w9, #1, #1
+; CHECK-NEXT: fmov s0, w10
+; CHECK-NEXT: ubfx w10, w9, #2, #1
+; CHECK-NEXT: mov v0.b[1], w11
+; CHECK-NEXT: mov v0.b[2], w10
+; CHECK-NEXT: ubfx w10, w9, #3, #1
+; CHECK-NEXT: mov v0.b[3], w10
+; CHECK-NEXT: ubfx w10, w9, #4, #1
+; CHECK-NEXT: mov v0.b[4], w10
+; CHECK-NEXT: ubfx w10, w9, #5, #1
+; CHECK-NEXT: mov v0.b[5], w10
+; CHECK-NEXT: ubfx w10, w9, #6, #1
+; CHECK-NEXT: lsr w9, w9, #7
+; CHECK-NEXT: mov v0.b[6], w10
+; CHECK-NEXT: mov v0.b[7], w9
+; CHECK-NEXT: shl v1.8b, v0.8b, #7
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: cmlt v1.8b, v1.8b, #0
+; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: addv b1, v1.8b
+; CHECK-NEXT: str b1, [x1]
+; CHECK-NEXT: ret
+ %ld = load <8 x i1>, ptr %s
+ store <8 x i1> %ld, ptr %d
+ ret <8 x i1> %ld
+}
+
+; Negative: volatile accesses are left alone.
+define void @copy_v8i1_volatile(ptr %s, ptr %d) {
+; CHECK-LABEL: copy_v8i1_volatile:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrb w9, [x0]
+; CHECK-NEXT: adrp x8, .LCPI3_0
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT: and w10, w9, #0x1
+; CHECK-NEXT: ubfx w11, w9, #1, #1
+; CHECK-NEXT: fmov s0, w10
+; CHECK-NEXT: ubfx w10, w9, #2, #1
+; CHECK-NEXT: mov v0.b[1], w11
+; CHECK-NEXT: mov v0.b[2], w10
+; CHECK-NEXT: ubfx w10, w9, #3, #1
+; CHECK-NEXT: mov v0.b[3], w10
+; CHECK-NEXT: ubfx w10, w9, #4, #1
+; CHECK-NEXT: mov v0.b[4], w10
+; CHECK-NEXT: ubfx w10, w9, #5, #1
+; CHECK-NEXT: mov v0.b[5], w10
+; CHECK-NEXT: ubfx w10, w9, #6, #1
+; CHECK-NEXT: lsr w9, w9, #7
+; CHECK-NEXT: mov v0.b[6], w10
+; CHECK-NEXT: mov v0.b[7], w9
+; CHECK-NEXT: shl v0.8b, v0.8b, #7
+; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: addv b0, v0.8b
+; CHECK-NEXT: str b0, [x1]
+; CHECK-NEXT: ret
+ %ld = load volatile <8 x i1>, ptr %s
+ store volatile <8 x i1> %ld, ptr %d
+ ret void
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/210535
More information about the llvm-commits
mailing list