[llvm] [AArch64][GlobalISel] Legalize <8 x i1> and <16 x i1> loads and stores (PR #210514)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 18 08:24:47 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-globalisel
Author: Joel Walker (Joel-Wwalker)
<details>
<summary>Changes</summary>
Fixes #<!-- -->116006.
`G_LOAD` of `<8 x i1>` / `<16 x i1>` failed to legalize on AArch64. Legalize these loads by bitcasting to the equivalent scalar (as `v4s8` already does), and fold the inverse cast the mutation creates, so the common load-then-bitcast pattern emits a single `ldrb`/`ldrh`, matching SelectionDAG.
Stores of these types already legalized, but by expanding bit by bit — about 17 instructions for what SelectionDAG does with one `strb`. Handle them as custom before `minScalarOrElt` widens the elements away: store the scalar directly when the stored value is a scalar bitcast to the i1 vector or a single-use load of the same type (the replacement load stays at the original load's position to preserve memory ordering), and fall back to the previous element-widening path otherwise (e.g. for compare-produced values).
`<4 x i1>` needs i4 memory types and is left for the future, matching the neighbouring TODO for v2i8/v2i16.
Assisted by Claude (Anthropic).
---
Full diff: https://github.com/llvm/llvm-project/pull/210514.diff
4 Files Affected:
- (modified) llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp (+98-2)
- (modified) llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h (+1)
- (modified) llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir (+81)
- (added) llvm/test/CodeGen/AArch64/vec-i1-load-store.ll (+83)
``````````diff
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 94ca171c0b207..d3e55b5bd3fe3 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -48,6 +48,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
const LLT s32 = LLT::scalar(32);
const LLT s64 = LLT::scalar(64);
const LLT s128 = LLT::scalar(128);
+ const LLT v16s1 = LLT::fixed_vector(16, 1);
+ const LLT v8s1 = LLT::fixed_vector(8, 1);
const LLT v16s8 = LLT::fixed_vector(16, 8);
const LLT v8s8 = LLT::fixed_vector(8, 8);
const LLT v4s8 = LLT::fixed_vector(4, 8);
@@ -595,7 +597,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
},
changeTo(0, s32))
// TODO: Use BITCAST for v2i8, v2i16 after G_TRUNC gets sorted out
- .bitcastIf(typeInSet(0, {v4s8}),
+ .bitcastIf(typeInSet(0, {v4s8, v8s1, v16s1}),
[=](const LegalityQuery &Query) {
const LLT VecTy = Query.Types[0];
return std::pair(0, LLT::integer(VecTy.getSizeInBits()));
@@ -638,6 +640,12 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
{nxv4s32, p0, nxv4s32, 8},
{nxv2s64, p0, nxv2s64, 8},
})
+ // Handle i1 vector stores before the elements are widened to s8.
+ .customIf(all(typeInSet(0, {v8s1, v16s1}),
+ LegalityPredicate([=](const LegalityQuery &Query) {
+ return Query.Types[0].getSizeInBits() ==
+ Query.MMODescrs[0].MemoryTy.getSizeInBits();
+ })))
.clampScalar(0, s8, s64)
.minScalarOrElt(0, s8)
.lowerIf([=](const LegalityQuery &Query) {
@@ -1553,8 +1561,12 @@ bool AArch64LegalizerInfo::legalizeCustom(
case TargetOpcode::G_VAARG:
return legalizeVaArg(MI, MRI, MIRBuilder);
case TargetOpcode::G_LOAD:
- case TargetOpcode::G_STORE:
+ case TargetOpcode::G_STORE: {
+ LLT ValTy = MRI.getType(MI.getOperand(0).getReg());
+ if (ValTy.isVector() && ValTy.getScalarSizeInBits() == 1)
+ return legalizeI1VecStore(MI, Helper);
return legalizeLoadStore(MI, MRI, MIRBuilder, Observer);
+ }
case TargetOpcode::G_SHL:
case TargetOpcode::G_ASHR:
case TargetOpcode::G_LSHR:
@@ -1604,16 +1616,100 @@ bool AArch64LegalizerInfo::legalizeCustom(
llvm_unreachable("expected switch to return");
}
+bool AArch64LegalizerInfo::legalizeI1VecStore(MachineInstr &MI,
+ LegalizerHelper &Helper) const {
+ auto &Store = cast<GStore>(MI);
+ MachineIRBuilder &MIB = Helper.MIRBuilder;
+ MachineRegisterInfo &MRI = *MIB.getMRI();
+ Register ValReg = Store.getValueReg();
+ LLT ValTy = MRI.getType(ValReg);
+ MachineFunction &MF = MIB.getMF();
+
+ // If the stored value is a scalar bitcast to an i1 vector, store the
+ // scalar directly instead of expanding the vector value bit by bit.
+ MachineInstr *Def = MRI.getVRegDef(ValReg);
+ if (Def && Def->getOpcode() == TargetOpcode::G_BITCAST) {
+ Register SrcReg = Def->getOperand(1).getReg();
+ LLT SrcTy = MRI.getType(SrcReg);
+ if (SrcTy.isScalar() &&
+ SrcTy.getSizeInBits() ==
+ Store.getMMO().getMemoryType().getSizeInBits()) {
+ MachineMemOperand *NewMMO =
+ MF.getMachineMemOperand(&Store.getMMO(), /*Offset=*/0, SrcTy);
+ MIB.setInstrAndDebugLoc(MI);
+ MIB.buildStore(SrcReg, Store.getPointerReg(), *NewMMO);
+ MI.eraseFromParent();
+ return true;
+ }
+ }
+
+ // Similarly, an i1 vector copied from memory to memory can be copied as
+ // the equivalent integer. The replacement load is built at the original
+ // load's position to keep the memory operation order.
+ if (Def && Def->getOpcode() == TargetOpcode::G_LOAD &&
+ MRI.hasOneNonDBGUse(ValReg)) {
+ auto *Load = cast<GLoad>(Def);
+ if (Load->getMMO().getMemoryType() == ValTy &&
+ Store.getMMO().getMemoryType() == ValTy) {
+ LLT IntTy = LLT::integer(ValTy.getSizeInBits());
+ MachineMemOperand *LoadMMO =
+ MF.getMachineMemOperand(&Load->getMMO(), /*Offset=*/0, IntTy);
+ MachineMemOperand *StoreMMO =
+ MF.getMachineMemOperand(&Store.getMMO(), /*Offset=*/0, IntTy);
+ MIB.setInstrAndDebugLoc(*Load);
+ auto NewLoad = MIB.buildLoad(IntTy, Load->getPointerReg(), *LoadMMO);
+ MIB.setInstrAndDebugLoc(MI);
+ MIB.buildStore(NewLoad, Store.getPointerReg(), *StoreMMO);
+ MI.eraseFromParent();
+ return true;
+ }
+ }
+
+ // Otherwise widen the elements to s8, as the generic path would have.
+ return Helper.widenScalar(MI, /*TypeIdx=*/0, ValTy.changeElementSize(8)) ==
+ LegalizerHelper::Legalized;
+}
+
bool AArch64LegalizerInfo::legalizeBitcast(MachineInstr &MI,
LegalizerHelper &Helper) const {
assert(MI.getOpcode() == TargetOpcode::G_BITCAST && "Unexpected opcode");
auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
+ MachineIRBuilder &MIB = Helper.MIRBuilder;
+ MachineRegisterInfo &MRI = *MIB.getMRI();
+
// We're trying to handle casts from i1 vectors to scalars but reloading from
// stack.
if (!DstTy.isScalar() || !SrcTy.isVector() ||
SrcTy.getElementType() != LLT::scalar(1))
return false;
+ MachineInstr *SrcMI = MRI.getVRegDef(SrcReg);
+
+ // Fold a cast of a cast from the destination type, such as the inverse
+ // cast created when an i1 vector G_LOAD is legalized by bitcasting to a
+ // scalar.
+ if (SrcMI && SrcMI->getOpcode() == TargetOpcode::G_BITCAST &&
+ MRI.getType(SrcMI->getOperand(1).getReg()) == DstTy) {
+ MIB.setInstrAndDebugLoc(MI);
+ MIB.buildCopy(DstReg, SrcMI->getOperand(1).getReg());
+ MI.eraseFromParent();
+ return true;
+ }
+
+ // Fold a cast of an i1 vector G_LOAD into a scalar load of the same
+ // memory, instead of expanding the vector value bit by bit.
+ if (SrcMI && SrcMI->getOpcode() == TargetOpcode::G_LOAD &&
+ MRI.hasOneNonDBGUse(SrcReg)) {
+ auto *Load = cast<GLoad>(SrcMI);
+ MachineFunction &MF = MIB.getMF();
+ MachineMemOperand *NewMMO =
+ MF.getMachineMemOperand(&Load->getMMO(), /*Offset=*/0, DstTy);
+ MIB.setInstrAndDebugLoc(MI);
+ MIB.buildLoad(DstReg, Load->getPointerReg(), *NewMMO);
+ MI.eraseFromParent();
+ return true;
+ }
+
Helper.createStackStoreLoad(DstReg, SrcReg);
MI.eraseFromParent();
return true;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
index 535240a67a2cc..fcd09b69d7ec1 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
@@ -67,6 +67,7 @@ class AArch64LegalizerInfo : public LegalizerInfo {
bool legalizeDynStackAlloc(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizePrefetch(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizeBitcast(MachineInstr &MI, LegalizerHelper &Helper) const;
+ bool legalizeI1VecStore(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizeConcatVectors(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &MIRBuilder) const;
bool legalizeFptrunc(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir
index ab4ecc5241a26..000d3641d29a3 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir
@@ -736,3 +736,84 @@ body: |
%val:_(<6 x p0>) = G_LOAD %ptr(p0) :: (load (<6 x p0>))
G_STORE %val(<6 x p0>), %ptr(p0) :: (store (<6 x p0>))
RET_ReallyLR
+...
+---
+name: load_v8s1
+body: |
+ bb.0:
+ liveins: $x0
+ ; CHECK-LABEL: name: load_v8s1
+ ; CHECK: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+ ; CHECK-NEXT: %bc:_(i8) = G_LOAD %ptr(p0) :: (load (i8))
+ ; CHECK-NEXT: %ext:_(i64) = G_ANYEXT %bc(i8)
+ ; CHECK-NEXT: $x0 = COPY %ext(i64)
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ %ptr:_(p0) = COPY $x0
+ %val:_(<8 x i1>) = G_LOAD %ptr(p0) :: (load (<8 x i1>))
+ %bc:_(i8) = G_BITCAST %val(<8 x i1>)
+ %ext:_(i64) = G_ANYEXT %bc(i8)
+ $x0 = COPY %ext(i64)
+ RET_ReallyLR implicit $x0
+...
+---
+name: load_v16s1
+body: |
+ bb.0:
+ liveins: $x0
+ ; CHECK-LABEL: name: load_v16s1
+ ; CHECK: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+ ; CHECK-NEXT: %bc:_(i16) = G_LOAD %ptr(p0) :: (load (i16))
+ ; CHECK-NEXT: %ext:_(i64) = G_ANYEXT %bc(i16)
+ ; CHECK-NEXT: $x0 = COPY %ext(i64)
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ %ptr:_(p0) = COPY $x0
+ %val:_(<16 x i1>) = G_LOAD %ptr(p0) :: (load (<16 x i1>))
+ %bc:_(i16) = G_BITCAST %val(<16 x i1>)
+ %ext:_(i64) = G_ANYEXT %bc(i16)
+ $x0 = COPY %ext(i64)
+ RET_ReallyLR implicit $x0
+...
+---
+name: store_v8s1
+body: |
+ bb.0:
+ liveins: $x0, $w1
+ ; CHECK-LABEL: name: store_v8s1
+ ; CHECK: liveins: $x0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+ ; CHECK-NEXT: %in:_(i32) = COPY $w1
+ ; CHECK-NEXT: %tr:_(i8) = G_TRUNC %in(i32)
+ ; CHECK-NEXT: G_STORE %tr(i8), %ptr(p0) :: (store (i8))
+ ; CHECK-NEXT: RET_ReallyLR
+ %ptr:_(p0) = COPY $x0
+ %in:_(i32) = COPY $w1
+ %tr:_(i8) = G_TRUNC %in(i32)
+ %bc:_(<8 x i1>) = G_BITCAST %tr(i8)
+ G_STORE %bc(<8 x i1>), %ptr(p0) :: (store (<8 x i1>))
+ RET_ReallyLR
+...
+---
+name: store_v16s1
+body: |
+ bb.0:
+ liveins: $x0, $w1
+ ; CHECK-LABEL: name: store_v16s1
+ ; CHECK: liveins: $x0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+ ; CHECK-NEXT: %in:_(i32) = COPY $w1
+ ; CHECK-NEXT: %tr:_(i16) = G_TRUNC %in(i32)
+ ; CHECK-NEXT: G_STORE %tr(i16), %ptr(p0) :: (store (i16))
+ ; CHECK-NEXT: RET_ReallyLR
+ %ptr:_(p0) = COPY $x0
+ %in:_(i32) = COPY $w1
+ %tr:_(i16) = G_TRUNC %in(i32)
+ %bc:_(<16 x i1>) = G_BITCAST %tr(i16)
+ G_STORE %bc(<16 x i1>), %ptr(p0) :: (store (<16 x i1>))
+ RET_ReallyLR
+...
diff --git a/llvm/test/CodeGen/AArch64/vec-i1-load-store.ll b/llvm/test/CodeGen/AArch64/vec-i1-load-store.ll
new file mode 100644
index 0000000000000..edad26d37d326
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vec-i1-load-store.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64 -global-isel < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; Loads of <8 x i1> / <16 x i1> used to fail to legalize in GlobalISel, and
+; the corresponding stores were expanded bit by bit. See issue #116006.
+
+define i8 @load_v8i1(ptr %d) {
+; CHECK-LABEL: load_v8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrb w0, [x0]
+; CHECK-NEXT: ret
+ %ld = load <8 x i1>, ptr %d
+ %b = bitcast <8 x i1> %ld to i8
+ ret i8 %b
+}
+
+define i16 @load_v16i1(ptr %d) {
+; CHECK-LABEL: load_v16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrh w0, [x0]
+; CHECK-NEXT: ret
+ %ld = load <16 x i1>, ptr %d
+ %b = bitcast <16 x i1> %ld to i16
+ ret i16 %b
+}
+
+define void @store_v8i1(ptr %d, i8 %v) {
+; CHECK-LABEL: store_v8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: strb w1, [x0]
+; CHECK-NEXT: ret
+ %b = bitcast i8 %v to <8 x i1>
+ store <8 x i1> %b, ptr %d
+ ret void
+}
+
+define void @store_v16i1(ptr %d, i16 %v) {
+; CHECK-LABEL: store_v16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: strh w1, [x0]
+; CHECK-NEXT: ret
+ %b = bitcast i16 %v to <16 x i1>
+ store <16 x i1> %b, ptr %d
+ ret void
+}
+
+define void @copy_v8i1(ptr %s, ptr %d) {
+; CHECK-SD-LABEL: copy_v8i1:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: ldrb w8, [x0]
+; CHECK-SD-NEXT: sbfx w9, w8, #0, #1
+; CHECK-SD-NEXT: sbfx w10, w8, #1, #1
+; CHECK-SD-NEXT: fmov s0, w9
+; CHECK-SD-NEXT: sbfx w9, w8, #2, #1
+; CHECK-SD-NEXT: mov v0.b[1], w10
+; CHECK-SD-NEXT: mov v0.b[2], w9
+; CHECK-SD-NEXT: sbfx w9, w8, #3, #1
+; CHECK-SD-NEXT: mov v0.b[3], w9
+; CHECK-SD-NEXT: sbfx w9, w8, #4, #1
+; CHECK-SD-NEXT: mov v0.b[4], w9
+; CHECK-SD-NEXT: sbfx w9, w8, #5, #1
+; CHECK-SD-NEXT: mov v0.b[5], w9
+; CHECK-SD-NEXT: sbfx w9, w8, #6, #1
+; CHECK-SD-NEXT: sbfx w8, w8, #7, #1
+; CHECK-SD-NEXT: mov v0.b[6], w9
+; CHECK-SD-NEXT: mov v0.b[7], w8
+; CHECK-SD-NEXT: adrp x8, .LCPI4_0
+; CHECK-SD-NEXT: ldr d1, [x8, :lo12:.LCPI4_0]
+; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: addv b0, v0.8b
+; CHECK-SD-NEXT: str b0, [x1]
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: copy_v8i1:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldrb w8, [x0]
+; CHECK-GI-NEXT: strb w8, [x1]
+; CHECK-GI-NEXT: ret
+ %ld = load <8 x i1>, ptr %s
+ store <8 x i1> %ld, ptr %d
+ ret void
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/210514
More information about the llvm-commits
mailing list