[llvm] [AArch64][GlobalISel] Legalize <8 x i1> and <16 x i1> loads and stores (PR #210514)
Joel Walker via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 18 11:58:33 PDT 2026
https://github.com/Joel-Wwalker updated https://github.com/llvm/llvm-project/pull/210514
>From 84ccf253a51e0d1030137c2968fc553dd24a28dd Mon Sep 17 00:00:00 2001
From: Joel-Wwalker <theagingboy05 at gmail.com>
Date: Sat, 18 Jul 2026 11:23:59 -0400
Subject: [PATCH] [AArch64][GlobalISel] Legalize <8 x i1> and <16 x i1> loads
and stores
G_LOAD of <8 x i1> / <16 x i1> failed to legalize. Legalize these loads
by bitcasting to the equivalent scalar, as v4s8 already does, and fold
the inverse cast the mutation creates so the common load-then-bitcast
pattern emits a single ldrb/ldrh, matching SelectionDAG.
Stores of these types already legalized, but by expanding bit by bit,
about 17 instructions for what SelectionDAG does with one strb. Handle
them as custom before minScalarOrElt widens the elements away: store
the scalar directly when the stored value is a scalar bitcast to the
i1 vector or a single-use load of the same type (the replacement load
is built at the original load's position to preserve the memory
operation order), and fall back to the previous element-widening path
otherwise, e.g. for compare-produced values.
<4 x i1> needs i4 memory types and is left for the future, matching
the neighbouring TODO for v2i8/v2i16.
Fixes #116006.
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 100 ++++++++++++++-
.../AArch64/GISel/AArch64LegalizerInfo.h | 1 +
.../GlobalISel/legalize-load-store.mir | 81 ++++++++++++
.../test/CodeGen/AArch64/vec-i1-load-store.ll | 117 ++++++++++++++++++
4 files changed, 297 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/vec-i1-load-store.ll
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 94ca171c0b207..d3e55b5bd3fe3 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -48,6 +48,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
const LLT s32 = LLT::scalar(32);
const LLT s64 = LLT::scalar(64);
const LLT s128 = LLT::scalar(128);
+ const LLT v16s1 = LLT::fixed_vector(16, 1);
+ const LLT v8s1 = LLT::fixed_vector(8, 1);
const LLT v16s8 = LLT::fixed_vector(16, 8);
const LLT v8s8 = LLT::fixed_vector(8, 8);
const LLT v4s8 = LLT::fixed_vector(4, 8);
@@ -595,7 +597,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
},
changeTo(0, s32))
// TODO: Use BITCAST for v2i8, v2i16 after G_TRUNC gets sorted out
- .bitcastIf(typeInSet(0, {v4s8}),
+ .bitcastIf(typeInSet(0, {v4s8, v8s1, v16s1}),
[=](const LegalityQuery &Query) {
const LLT VecTy = Query.Types[0];
return std::pair(0, LLT::integer(VecTy.getSizeInBits()));
@@ -638,6 +640,12 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
{nxv4s32, p0, nxv4s32, 8},
{nxv2s64, p0, nxv2s64, 8},
})
+ // Handle i1 vector stores before the elements are widened to s8.
+ .customIf(all(typeInSet(0, {v8s1, v16s1}),
+ LegalityPredicate([=](const LegalityQuery &Query) {
+ return Query.Types[0].getSizeInBits() ==
+ Query.MMODescrs[0].MemoryTy.getSizeInBits();
+ })))
.clampScalar(0, s8, s64)
.minScalarOrElt(0, s8)
.lowerIf([=](const LegalityQuery &Query) {
@@ -1553,8 +1561,12 @@ bool AArch64LegalizerInfo::legalizeCustom(
case TargetOpcode::G_VAARG:
return legalizeVaArg(MI, MRI, MIRBuilder);
case TargetOpcode::G_LOAD:
- case TargetOpcode::G_STORE:
+ case TargetOpcode::G_STORE: {
+ LLT ValTy = MRI.getType(MI.getOperand(0).getReg());
+ if (ValTy.isVector() && ValTy.getScalarSizeInBits() == 1)
+ return legalizeI1VecStore(MI, Helper);
return legalizeLoadStore(MI, MRI, MIRBuilder, Observer);
+ }
case TargetOpcode::G_SHL:
case TargetOpcode::G_ASHR:
case TargetOpcode::G_LSHR:
@@ -1604,16 +1616,100 @@ bool AArch64LegalizerInfo::legalizeCustom(
llvm_unreachable("expected switch to return");
}
+bool AArch64LegalizerInfo::legalizeI1VecStore(MachineInstr &MI,
+ LegalizerHelper &Helper) const {
+ auto &Store = cast<GStore>(MI);
+ MachineIRBuilder &MIB = Helper.MIRBuilder;
+ MachineRegisterInfo &MRI = *MIB.getMRI();
+ Register ValReg = Store.getValueReg();
+ LLT ValTy = MRI.getType(ValReg);
+ MachineFunction &MF = MIB.getMF();
+
+ // If the stored value is a scalar bitcast to an i1 vector, store the
+ // scalar directly instead of expanding the vector value bit by bit.
+ MachineInstr *Def = MRI.getVRegDef(ValReg);
+ if (Def && Def->getOpcode() == TargetOpcode::G_BITCAST) {
+ Register SrcReg = Def->getOperand(1).getReg();
+ LLT SrcTy = MRI.getType(SrcReg);
+ if (SrcTy.isScalar() &&
+ SrcTy.getSizeInBits() ==
+ Store.getMMO().getMemoryType().getSizeInBits()) {
+ MachineMemOperand *NewMMO =
+ MF.getMachineMemOperand(&Store.getMMO(), /*Offset=*/0, SrcTy);
+ MIB.setInstrAndDebugLoc(MI);
+ MIB.buildStore(SrcReg, Store.getPointerReg(), *NewMMO);
+ MI.eraseFromParent();
+ return true;
+ }
+ }
+
+ // Similarly, an i1 vector copied from memory to memory can be copied as
+ // the equivalent integer. The replacement load is built at the original
+ // load's position to keep the memory operation order.
+ if (Def && Def->getOpcode() == TargetOpcode::G_LOAD &&
+ MRI.hasOneNonDBGUse(ValReg)) {
+ auto *Load = cast<GLoad>(Def);
+ if (Load->getMMO().getMemoryType() == ValTy &&
+ Store.getMMO().getMemoryType() == ValTy) {
+ LLT IntTy = LLT::integer(ValTy.getSizeInBits());
+ MachineMemOperand *LoadMMO =
+ MF.getMachineMemOperand(&Load->getMMO(), /*Offset=*/0, IntTy);
+ MachineMemOperand *StoreMMO =
+ MF.getMachineMemOperand(&Store.getMMO(), /*Offset=*/0, IntTy);
+ MIB.setInstrAndDebugLoc(*Load);
+ auto NewLoad = MIB.buildLoad(IntTy, Load->getPointerReg(), *LoadMMO);
+ MIB.setInstrAndDebugLoc(MI);
+ MIB.buildStore(NewLoad, Store.getPointerReg(), *StoreMMO);
+ MI.eraseFromParent();
+ return true;
+ }
+ }
+
+ // Otherwise widen the elements to s8, as the generic path would have.
+ return Helper.widenScalar(MI, /*TypeIdx=*/0, ValTy.changeElementSize(8)) ==
+ LegalizerHelper::Legalized;
+}
+
bool AArch64LegalizerInfo::legalizeBitcast(MachineInstr &MI,
LegalizerHelper &Helper) const {
assert(MI.getOpcode() == TargetOpcode::G_BITCAST && "Unexpected opcode");
auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
+ MachineIRBuilder &MIB = Helper.MIRBuilder;
+ MachineRegisterInfo &MRI = *MIB.getMRI();
+
// We're trying to handle casts from i1 vectors to scalars but reloading from
// stack.
if (!DstTy.isScalar() || !SrcTy.isVector() ||
SrcTy.getElementType() != LLT::scalar(1))
return false;
+ MachineInstr *SrcMI = MRI.getVRegDef(SrcReg);
+
+ // Fold a cast of a cast from the destination type, such as the inverse
+ // cast created when an i1 vector G_LOAD is legalized by bitcasting to a
+ // scalar.
+ if (SrcMI && SrcMI->getOpcode() == TargetOpcode::G_BITCAST &&
+ MRI.getType(SrcMI->getOperand(1).getReg()) == DstTy) {
+ MIB.setInstrAndDebugLoc(MI);
+ MIB.buildCopy(DstReg, SrcMI->getOperand(1).getReg());
+ MI.eraseFromParent();
+ return true;
+ }
+
+ // Fold a cast of an i1 vector G_LOAD into a scalar load of the same
+ // memory, instead of expanding the vector value bit by bit.
+ if (SrcMI && SrcMI->getOpcode() == TargetOpcode::G_LOAD &&
+ MRI.hasOneNonDBGUse(SrcReg)) {
+ auto *Load = cast<GLoad>(SrcMI);
+ MachineFunction &MF = MIB.getMF();
+ MachineMemOperand *NewMMO =
+ MF.getMachineMemOperand(&Load->getMMO(), /*Offset=*/0, DstTy);
+ MIB.setInstrAndDebugLoc(MI);
+ MIB.buildLoad(DstReg, Load->getPointerReg(), *NewMMO);
+ MI.eraseFromParent();
+ return true;
+ }
+
Helper.createStackStoreLoad(DstReg, SrcReg);
MI.eraseFromParent();
return true;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
index 535240a67a2cc..fcd09b69d7ec1 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
@@ -67,6 +67,7 @@ class AArch64LegalizerInfo : public LegalizerInfo {
bool legalizeDynStackAlloc(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizePrefetch(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizeBitcast(MachineInstr &MI, LegalizerHelper &Helper) const;
+ bool legalizeI1VecStore(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizeConcatVectors(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &MIRBuilder) const;
bool legalizeFptrunc(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir
index ab4ecc5241a26..000d3641d29a3 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir
@@ -736,3 +736,84 @@ body: |
%val:_(<6 x p0>) = G_LOAD %ptr(p0) :: (load (<6 x p0>))
G_STORE %val(<6 x p0>), %ptr(p0) :: (store (<6 x p0>))
RET_ReallyLR
+...
+---
+name: load_v8s1
+body: |
+ bb.0:
+ liveins: $x0
+ ; CHECK-LABEL: name: load_v8s1
+ ; CHECK: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+ ; CHECK-NEXT: %bc:_(i8) = G_LOAD %ptr(p0) :: (load (i8))
+ ; CHECK-NEXT: %ext:_(i64) = G_ANYEXT %bc(i8)
+ ; CHECK-NEXT: $x0 = COPY %ext(i64)
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ %ptr:_(p0) = COPY $x0
+ %val:_(<8 x i1>) = G_LOAD %ptr(p0) :: (load (<8 x i1>))
+ %bc:_(i8) = G_BITCAST %val(<8 x i1>)
+ %ext:_(i64) = G_ANYEXT %bc(i8)
+ $x0 = COPY %ext(i64)
+ RET_ReallyLR implicit $x0
+...
+---
+name: load_v16s1
+body: |
+ bb.0:
+ liveins: $x0
+ ; CHECK-LABEL: name: load_v16s1
+ ; CHECK: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+ ; CHECK-NEXT: %bc:_(i16) = G_LOAD %ptr(p0) :: (load (i16))
+ ; CHECK-NEXT: %ext:_(i64) = G_ANYEXT %bc(i16)
+ ; CHECK-NEXT: $x0 = COPY %ext(i64)
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ %ptr:_(p0) = COPY $x0
+ %val:_(<16 x i1>) = G_LOAD %ptr(p0) :: (load (<16 x i1>))
+ %bc:_(i16) = G_BITCAST %val(<16 x i1>)
+ %ext:_(i64) = G_ANYEXT %bc(i16)
+ $x0 = COPY %ext(i64)
+ RET_ReallyLR implicit $x0
+...
+---
+name: store_v8s1
+body: |
+ bb.0:
+ liveins: $x0, $w1
+ ; CHECK-LABEL: name: store_v8s1
+ ; CHECK: liveins: $x0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+ ; CHECK-NEXT: %in:_(i32) = COPY $w1
+ ; CHECK-NEXT: %tr:_(i8) = G_TRUNC %in(i32)
+ ; CHECK-NEXT: G_STORE %tr(i8), %ptr(p0) :: (store (i8))
+ ; CHECK-NEXT: RET_ReallyLR
+ %ptr:_(p0) = COPY $x0
+ %in:_(i32) = COPY $w1
+ %tr:_(i8) = G_TRUNC %in(i32)
+ %bc:_(<8 x i1>) = G_BITCAST %tr(i8)
+ G_STORE %bc(<8 x i1>), %ptr(p0) :: (store (<8 x i1>))
+ RET_ReallyLR
+...
+---
+name: store_v16s1
+body: |
+ bb.0:
+ liveins: $x0, $w1
+ ; CHECK-LABEL: name: store_v16s1
+ ; CHECK: liveins: $x0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+ ; CHECK-NEXT: %in:_(i32) = COPY $w1
+ ; CHECK-NEXT: %tr:_(i16) = G_TRUNC %in(i32)
+ ; CHECK-NEXT: G_STORE %tr(i16), %ptr(p0) :: (store (i16))
+ ; CHECK-NEXT: RET_ReallyLR
+ %ptr:_(p0) = COPY $x0
+ %in:_(i32) = COPY $w1
+ %tr:_(i16) = G_TRUNC %in(i32)
+ %bc:_(<16 x i1>) = G_BITCAST %tr(i16)
+ G_STORE %bc(<16 x i1>), %ptr(p0) :: (store (<16 x i1>))
+ RET_ReallyLR
+...
diff --git a/llvm/test/CodeGen/AArch64/vec-i1-load-store.ll b/llvm/test/CodeGen/AArch64/vec-i1-load-store.ll
new file mode 100644
index 0000000000000..c086c49ab316f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vec-i1-load-store.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64 -global-isel < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; Loads of <8 x i1> / <16 x i1> used to fail to legalize in GlobalISel, and
+; the corresponding stores were expanded bit by bit. See issue #116006.
+
+define i8 @load_v8i1(ptr %d) {
+; CHECK-LABEL: load_v8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrb w0, [x0]
+; CHECK-NEXT: ret
+ %ld = load <8 x i1>, ptr %d
+ %b = bitcast <8 x i1> %ld to i8
+ ret i8 %b
+}
+
+define i16 @load_v16i1(ptr %d) {
+; CHECK-LABEL: load_v16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrh w0, [x0]
+; CHECK-NEXT: ret
+ %ld = load <16 x i1>, ptr %d
+ %b = bitcast <16 x i1> %ld to i16
+ ret i16 %b
+}
+
+define void @store_v8i1(ptr %d, i8 %v) {
+; CHECK-LABEL: store_v8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: strb w1, [x0]
+; CHECK-NEXT: ret
+ %b = bitcast i8 %v to <8 x i1>
+ store <8 x i1> %b, ptr %d
+ ret void
+}
+
+define void @store_v16i1(ptr %d, i16 %v) {
+; CHECK-LABEL: store_v16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: strh w1, [x0]
+; CHECK-NEXT: ret
+ %b = bitcast i16 %v to <16 x i1>
+ store <16 x i1> %b, ptr %d
+ ret void
+}
+
+define void @copy_v8i1(ptr %s, ptr %d) {
+; CHECK-SD-LABEL: copy_v8i1:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: ldrb w8, [x0]
+; CHECK-SD-NEXT: sbfx w9, w8, #0, #1
+; CHECK-SD-NEXT: sbfx w10, w8, #1, #1
+; CHECK-SD-NEXT: fmov s0, w9
+; CHECK-SD-NEXT: sbfx w9, w8, #2, #1
+; CHECK-SD-NEXT: mov v0.b[1], w10
+; CHECK-SD-NEXT: mov v0.b[2], w9
+; CHECK-SD-NEXT: sbfx w9, w8, #3, #1
+; CHECK-SD-NEXT: mov v0.b[3], w9
+; CHECK-SD-NEXT: sbfx w9, w8, #4, #1
+; CHECK-SD-NEXT: mov v0.b[4], w9
+; CHECK-SD-NEXT: sbfx w9, w8, #5, #1
+; CHECK-SD-NEXT: mov v0.b[5], w9
+; CHECK-SD-NEXT: sbfx w9, w8, #6, #1
+; CHECK-SD-NEXT: sbfx w8, w8, #7, #1
+; CHECK-SD-NEXT: mov v0.b[6], w9
+; CHECK-SD-NEXT: mov v0.b[7], w8
+; CHECK-SD-NEXT: adrp x8, .LCPI4_0
+; CHECK-SD-NEXT: ldr d1, [x8, :lo12:.LCPI4_0]
+; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: addv b0, v0.8b
+; CHECK-SD-NEXT: str b0, [x1]
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: copy_v8i1:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldrb w8, [x0]
+; CHECK-GI-NEXT: strb w8, [x1]
+; CHECK-GI-NEXT: ret
+ %ld = load <8 x i1>, ptr %s
+ store <8 x i1> %ld, ptr %d
+ ret void
+}
+
+define i1 @extract_v8i1(ptr %d) {
+; CHECK-SD-LABEL: extract_v8i1:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: ldrb w8, [x0]
+; CHECK-SD-NEXT: ubfx w0, w8, #3, #1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: extract_v8i1:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldrb w8, [x0]
+; CHECK-GI-NEXT: fmov s0, w8
+; CHECK-GI-NEXT: lsr w9, w8, #1
+; CHECK-GI-NEXT: mov v0.b[1], w9
+; CHECK-GI-NEXT: lsr w9, w8, #2
+; CHECK-GI-NEXT: mov v0.b[2], w9
+; CHECK-GI-NEXT: lsr w9, w8, #3
+; CHECK-GI-NEXT: mov v0.b[3], w9
+; CHECK-GI-NEXT: lsr w9, w8, #4
+; CHECK-GI-NEXT: mov v0.b[4], w9
+; CHECK-GI-NEXT: lsr w9, w8, #5
+; CHECK-GI-NEXT: mov v0.b[5], w9
+; CHECK-GI-NEXT: lsr w9, w8, #6
+; CHECK-GI-NEXT: lsr w8, w8, #7
+; CHECK-GI-NEXT: mov v0.b[6], w9
+; CHECK-GI-NEXT: mov v0.b[7], w8
+; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: umov w8, v0.h[3]
+; CHECK-GI-NEXT: and w0, w8, #0x1
+; CHECK-GI-NEXT: ret
+ %ld = load <8 x i1>, ptr %d
+ %e = extractelement <8 x i1> %ld, i32 3
+ ret i1 %e
+}
More information about the llvm-commits
mailing list