[llvm] [AArch64][GlobalISel] Legalize <8 x i1> and <16 x i1> loads and stores (PR #210514)
Joel Walker via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 06:01:00 PDT 2026
https://github.com/Joel-Wwalker updated https://github.com/llvm/llvm-project/pull/210514
>From 223110bf3159dc00b991097b5d9faa70c635e94d Mon Sep 17 00:00:00 2001
From: Joel-Wwalker <theagingboy05 at gmail.com>
Date: Tue, 4 Aug 2026 09:32:37 -0400
Subject: [PATCH 1/2] [AArch64][GlobalISel] Pack i1 vector to scalar bitcasts
in vector registers
Casting an i1 vector to a scalar currently round-trips the stack: the
mask is stored bit by bit and reloaded as an integer, about 50
instructions and a stack slot for a 16-lane compare result.
Legalize these casts the way SelectionDAG lowers them instead:
sign-extend the lanes to all-ones or all-zeros, mask lane i down to the
weight 1 << i, and add-reduce, which selects to cmlt/and/addv. The pack
applies to 8 and 16 lanes, where the reduction is efficient; other lane
counts keep the stack round-trip.
Also fold casts of a single-use i1 vector load into a scalar load of
the same memory, so a loaded mask is not unpacked just to be packed
again.
Split out of #210514, which adds i1 vector load/store rules on top of
these changes.
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 43 +-
.../AArch64/GlobalISel/legalize-bitcast.mir | 66 +--
.../AArch64/vec-combine-compare-to-bitmask.ll | 521 +++---------------
3 files changed, 132 insertions(+), 498 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 94ca171c0b207..a17cb89e64d81 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1608,13 +1608,54 @@ bool AArch64LegalizerInfo::legalizeBitcast(MachineInstr &MI,
LegalizerHelper &Helper) const {
assert(MI.getOpcode() == TargetOpcode::G_BITCAST && "Unexpected opcode");
auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
+ MachineIRBuilder &MIB = Helper.MIRBuilder;
+ MachineRegisterInfo &MRI = *MIB.getMRI();
+
// We're trying to handle casts from i1 vectors to scalars but reloading from
// stack.
if (!DstTy.isScalar() || !SrcTy.isVector() ||
SrcTy.getElementType() != LLT::scalar(1))
return false;
- Helper.createStackStoreLoad(DstReg, SrcReg);
+ MachineInstr *SrcMI = MRI.getVRegDef(SrcReg);
+
+ // Fold a cast of an i1 vector G_LOAD into a scalar load of the same
+ // memory, instead of expanding the vector value bit by bit.
+ if (SrcMI && SrcMI->getOpcode() == TargetOpcode::G_LOAD &&
+ MRI.hasOneNonDBGUse(SrcReg)) {
+ auto *Load = cast<GLoad>(SrcMI);
+ MachineFunction &MF = MIB.getMF();
+ MachineMemOperand *NewMMO =
+ MF.getMachineMemOperand(&Load->getMMO(), /*Offset=*/0, DstTy);
+ MIB.setInstrAndDebugLoc(MI);
+ MIB.buildLoad(DstReg, Load->getPointerReg(), *NewMMO);
+ MI.eraseFromParent();
+ return true;
+ }
+
+ // Pack the mask in vector registers, as SelectionDAG does: sign-extend
+ // each lane to all-ones or all-zeros, mask lane i down to the value
+ // 1 << i, and sum the lanes. The lane count always equals the destination
+ // bit width here, so the weights fit the widened element type. Only do
+ // this for lane counts with an efficient reduction; other sizes keep the
+ // stack round-trip below.
+ unsigned NumElts = SrcTy.getNumElements();
+ if (NumElts != 8 && NumElts != 16) {
+ Helper.createStackStoreLoad(DstReg, SrcReg);
+ MI.eraseFromParent();
+ return true;
+ }
+ MIB.setInstrAndDebugLoc(MI);
+ LLT WideVecTy = LLT::fixed_vector(NumElts, DstTy);
+ auto Sext = MIB.buildSExt(WideVecTy, SrcReg);
+ SmallVector<Register> Weights;
+ for (unsigned I = 0; I != NumElts; ++I)
+ Weights.push_back(
+ MIB.buildConstant(DstTy, APInt::getOneBitSet(DstTy.getSizeInBits(), I))
+ .getReg(0));
+ auto Mask = MIB.buildBuildVector(WideVecTy, Weights);
+ auto Masked = MIB.buildAnd(WideVecTy, Sext, Mask);
+ MIB.buildInstr(TargetOpcode::G_VECREDUCE_ADD, {DstReg}, {Masked});
MI.eraseFromParent();
return true;
}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
index f08db556d14d8..5ef6e457ad6ec 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
@@ -55,60 +55,18 @@ body: |
bb.1:
; CHECK-LABEL: name: boolean_vector_to_scalar
; CHECK: [[DEF:%[0-9]+]]:_(<8 x i8>) = G_IMPLICIT_DEF
- ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
- ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C]](i64)
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC]](i8)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ANYEXT]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C]](i64)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
- ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C3]](i64)
- ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC1]](i8)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ANYEXT1]], [[C1]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i64)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
- ; CHECK-NEXT: [[EVEC2:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C4]](i64)
- ; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC2]](i8)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ANYEXT2]], [[C1]]
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C4]](i64)
- ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
- ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 3
- ; CHECK-NEXT: [[EVEC3:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C5]](i64)
- ; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC3]](i8)
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ANYEXT3]], [[C1]]
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C5]](i64)
- ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[OR2]], [[SHL3]]
- ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
- ; CHECK-NEXT: [[EVEC4:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C6]](i64)
- ; CHECK-NEXT: [[ANYEXT4:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC4]](i8)
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[ANYEXT4]], [[C1]]
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND4]], [[C6]](i64)
- ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[OR3]], [[SHL4]]
- ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i64) = G_CONSTANT i64 5
- ; CHECK-NEXT: [[EVEC5:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C7]](i64)
- ; CHECK-NEXT: [[ANYEXT5:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC5]](i8)
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[ANYEXT5]], [[C1]]
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i64)
- ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[OR4]], [[SHL5]]
- ; CHECK-NEXT: [[C8:%[0-9]+]]:_(i64) = G_CONSTANT i64 6
- ; CHECK-NEXT: [[EVEC6:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C8]](i64)
- ; CHECK-NEXT: [[ANYEXT6:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC6]](i8)
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[ANYEXT6]], [[C1]]
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND6]], [[C8]](i64)
- ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[OR5]], [[SHL6]]
- ; CHECK-NEXT: [[C9:%[0-9]+]]:_(i64) = G_CONSTANT i64 7
- ; CHECK-NEXT: [[EVEC7:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C9]](i64)
- ; CHECK-NEXT: [[ANYEXT7:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC7]](i8)
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[ANYEXT7]], [[C1]]
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C9]](i64)
- ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[OR6]], [[SHL7]]
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[OR7]](i32)
- ; CHECK-NEXT: G_STORE [[TRUNC]](i8), [[FRAME_INDEX]](p0) :: (store (i8) into %stack.0)
- ; CHECK-NEXT: %bc:_(i8) = G_LOAD [[FRAME_INDEX]](p0) :: (load (i8) from %stack.0)
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(<8 x i8>) = G_SEXT_INREG [[DEF]], 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i8) = G_CONSTANT i8 1
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i8) = G_CONSTANT i8 2
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i8) = G_CONSTANT i8 4
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i8) = G_CONSTANT i8 8
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i8) = G_CONSTANT i8 16
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i8) = G_CONSTANT i8 32
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i8) = G_CONSTANT i8 64
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i8) = G_CONSTANT i8 -128
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i8>) = G_BUILD_VECTOR [[C]](i8), [[C1]](i8), [[C2]](i8), [[C3]](i8), [[C4]](i8), [[C5]](i8), [[C6]](i8), [[C7]](i8)
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<8 x i8>) = G_AND [[SEXT_INREG]], [[BUILD_VECTOR]]
+ ; CHECK-NEXT: %bc:_(i8) = G_VECREDUCE_ADD [[AND]](<8 x i8>)
; CHECK-NEXT: %ext:_(i32) = G_ANYEXT %bc(i8)
; CHECK-NEXT: $w0 = COPY %ext(i32)
; CHECK-NEXT: RET_ReallyLR implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
index f9ecae443d399..a50bb39c0159f 100644
--- a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
@@ -25,58 +25,22 @@ define i16 @convert_to_bitmask16(<16 x i8> %vec) {
;
; CHECK-GI-LABEL: convert_to_bitmask16:
; CHECK-GI: ; %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: cmtst.16b v0, v0, v0
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
-; CHECK-GI-NEXT: and w8, w8, #0x1
-; CHECK-GI-NEXT: bfi w9, w8, #1, #31
-; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[8]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[9]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[10]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #8
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[11]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #9
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[12]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #10
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[13]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #11
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[14]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #12
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[15]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #13
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #14
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #15
-; CHECK-GI-NEXT: strh w8, [sp, #14]
-; CHECK-GI-NEXT: and w0, w8, #0xffff
-; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: adrp x8, lCPI0_1 at PAGE
+; CHECK-GI-NEXT: adrp x9, lCPI0_0 at PAGE
+; CHECK-GI-NEXT: ldr q2, [x8, lCPI0_1 at PAGEOFF]
+; CHECK-GI-NEXT: ldr q3, [x9, lCPI0_0 at PAGEOFF]
+; CHECK-GI-NEXT: ushll.8h v1, v0, #0
+; CHECK-GI-NEXT: ushll2.8h v0, v0, #0
+; CHECK-GI-NEXT: shl.8h v1, v1, #15
+; CHECK-GI-NEXT: shl.8h v0, v0, #15
+; CHECK-GI-NEXT: cmlt.8h v1, v1, #0
+; CHECK-GI-NEXT: cmlt.8h v0, v0, #0
+; CHECK-GI-NEXT: and.16b v1, v1, v2
+; CHECK-GI-NEXT: and.16b v0, v0, v3
+; CHECK-GI-NEXT: add.8h v0, v1, v0
+; CHECK-GI-NEXT: addv.8h h0, v0
+; CHECK-GI-NEXT: fmov w0, s0
; CHECK-GI-NEXT: ret
; Actual conversion
@@ -100,35 +64,14 @@ define i16 @convert_to_bitmask8(<8 x i16> %vec) {
;
; CHECK-GI-LABEL: convert_to_bitmask8:
; CHECK-GI: ; %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: cmtst.8h v0, v0, v0
+; CHECK-GI-NEXT: adrp x8, lCPI1_0 at PAGE
+; CHECK-GI-NEXT: ldr d1, [x8, lCPI1_0 at PAGEOFF]
; CHECK-GI-NEXT: xtn.8b v0, v0
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
-; CHECK-GI-NEXT: and w8, w8, #0x1
-; CHECK-GI-NEXT: bfi w9, w8, #1, #31
-; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
-; CHECK-GI-NEXT: strb w8, [sp, #15]
-; CHECK-GI-NEXT: and w0, w8, #0xff
-; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: and.8b v0, v0, v1
+; CHECK-GI-NEXT: addv.8b b0, v0
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: uxtb w0, w8
; CHECK-GI-NEXT: ret
@@ -208,36 +151,16 @@ define i8 @clang_builtins_undef_concat_convert_to_bitmask4(<4 x i32> %vec) {
;
; CHECK-GI-LABEL: clang_builtins_undef_concat_convert_to_bitmask4:
; CHECK-GI: ; %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: cmtst.4s v0, v0, v0
+; CHECK-GI-NEXT: adrp x8, lCPI4_0 at PAGE
+; CHECK-GI-NEXT: ldr d1, [x8, lCPI4_0 at PAGEOFF]
; CHECK-GI-NEXT: xtn.4h v0, v0
; CHECK-GI-NEXT: uzp1.8b v0, v0, v0
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
-; CHECK-GI-NEXT: and w8, w8, #0x1
-; CHECK-GI-NEXT: bfi w9, w8, #1, #31
-; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
-; CHECK-GI-NEXT: strb w8, [sp, #15]
-; CHECK-GI-NEXT: and w0, w8, #0xff
-; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: shl.8b v0, v0, #7
+; CHECK-GI-NEXT: cmlt.8b v0, v0, #0
+; CHECK-GI-NEXT: and.8b v0, v0, v1
+; CHECK-GI-NEXT: addv.8b b0, v0
+; CHECK-GI-NEXT: fmov w0, s0
; CHECK-GI-NEXT: ret
@@ -529,57 +452,21 @@ define i16 @convert_to_bitmask_without_knowing_type(<16 x i1> %vec) {
;
; CHECK-GI-LABEL: convert_to_bitmask_without_knowing_type:
; CHECK-GI: ; %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
-; CHECK-GI-NEXT: and w8, w8, #0x1
-; CHECK-GI-NEXT: bfi w9, w8, #1, #31
-; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[8]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[9]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[10]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #8
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[11]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #9
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[12]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #10
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[13]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #11
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[14]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #12
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[15]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #13
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #14
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #15
-; CHECK-GI-NEXT: strh w8, [sp, #14]
-; CHECK-GI-NEXT: and w0, w8, #0xffff
-; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: ushll.8h v1, v0, #0
+; CHECK-GI-NEXT: ushll2.8h v0, v0, #0
+; CHECK-GI-NEXT: adrp x8, lCPI10_1 at PAGE
+; CHECK-GI-NEXT: adrp x9, lCPI10_0 at PAGE
+; CHECK-GI-NEXT: ldr q2, [x8, lCPI10_1 at PAGEOFF]
+; CHECK-GI-NEXT: ldr q3, [x9, lCPI10_0 at PAGEOFF]
+; CHECK-GI-NEXT: shl.8h v1, v1, #15
+; CHECK-GI-NEXT: shl.8h v0, v0, #15
+; CHECK-GI-NEXT: cmlt.8h v1, v1, #0
+; CHECK-GI-NEXT: cmlt.8h v0, v0, #0
+; CHECK-GI-NEXT: and.16b v1, v1, v2
+; CHECK-GI-NEXT: and.16b v0, v0, v3
+; CHECK-GI-NEXT: add.8h v0, v1, v0
+; CHECK-GI-NEXT: addv.8h h0, v0
+; CHECK-GI-NEXT: fmov w0, s0
; CHECK-GI-NEXT: ret
%bitmask = bitcast <16 x i1> %vec to i16
@@ -644,50 +531,17 @@ define i4 @convert_to_bitmask_4xi8(<4 x i8> %vec) {
}
define i8 @convert_to_bitmask_8xi2(<8 x i2> %vec) {
-; CHECK-SD-LABEL: convert_to_bitmask_8xi2:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: movi.8b v1, #3
-; CHECK-SD-NEXT: adrp x8, lCPI13_0 at PAGE
-; CHECK-SD-NEXT: and.8b v0, v0, v1
-; CHECK-SD-NEXT: ldr d1, [x8, lCPI13_0 at PAGEOFF]
-; CHECK-SD-NEXT: cmeq.8b v0, v0, #0
-; CHECK-SD-NEXT: bic.8b v0, v1, v0
-; CHECK-SD-NEXT: addv.8b b0, v0
-; CHECK-SD-NEXT: fmov w0, s0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: convert_to_bitmask_8xi2:
-; CHECK-GI: ; %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: movi.8b v1, #3
-; CHECK-GI-NEXT: cmtst.8b v0, v0, v1
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
-; CHECK-GI-NEXT: and w8, w8, #0x1
-; CHECK-GI-NEXT: bfi w9, w8, #1, #31
-; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
-; CHECK-GI-NEXT: strb w8, [sp, #15]
-; CHECK-GI-NEXT: and w0, w8, #0xff
-; CHECK-GI-NEXT: add sp, sp, #16
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: convert_to_bitmask_8xi2:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: movi.8b v1, #3
+; CHECK-NEXT: adrp x8, lCPI13_0 at PAGE
+; CHECK-NEXT: and.8b v0, v0, v1
+; CHECK-NEXT: ldr d1, [x8, lCPI13_0 at PAGEOFF]
+; CHECK-NEXT: cmeq.8b v0, v0, #0
+; CHECK-NEXT: bic.8b v0, v1, v0
+; CHECK-NEXT: addv.8b b0, v0
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
%cmp_result = icmp ne <8 x i2> %vec, zeroinitializer
%bitmask = bitcast <8 x i1> %cmp_result to i8
@@ -756,37 +610,15 @@ define i8 @convert_large_vector(<8 x i32> %vec) {
;
; CHECK-GI-LABEL: convert_large_vector:
; CHECK-GI: ; %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: cmtst.4s v0, v0, v0
; CHECK-GI-NEXT: cmtst.4s v1, v1, v1
+; CHECK-GI-NEXT: adrp x8, lCPI15_0 at PAGE
; CHECK-GI-NEXT: uzp1.8h v0, v0, v1
+; CHECK-GI-NEXT: ldr d1, [x8, lCPI15_0 at PAGEOFF]
; CHECK-GI-NEXT: xtn.8b v0, v0
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
-; CHECK-GI-NEXT: and w8, w8, #0x1
-; CHECK-GI-NEXT: bfi w9, w8, #1, #31
-; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
-; CHECK-GI-NEXT: strb w8, [sp, #15]
-; CHECK-GI-NEXT: and w0, w8, #0xff
-; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: and.8b v0, v0, v1
+; CHECK-GI-NEXT: addv.8b b0, v0
+; CHECK-GI-NEXT: fmov w0, s0
; CHECK-GI-NEXT: ret
@@ -859,37 +691,17 @@ define i8 @no_direct_convert_for_bad_concat(<4 x i32> %vec) {
;
; CHECK-GI-LABEL: no_direct_convert_for_bad_concat:
; CHECK-GI: ; %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: cmtst.4s v0, v0, v0
+; CHECK-GI-NEXT: adrp x8, lCPI17_0 at PAGE
+; CHECK-GI-NEXT: ldr d1, [x8, lCPI17_0 at PAGEOFF]
; CHECK-GI-NEXT: xtn.4h v0, v0
; CHECK-GI-NEXT: uzp1.8b v0, v0, v0
; CHECK-GI-NEXT: mov.s v0[1], v0[0]
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
-; CHECK-GI-NEXT: and w8, w8, #0x1
-; CHECK-GI-NEXT: bfi w9, w8, #1, #31
-; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
-; CHECK-GI-NEXT: strb w8, [sp, #15]
-; CHECK-GI-NEXT: and w0, w8, #0xff
-; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: shl.8b v0, v0, #7
+; CHECK-GI-NEXT: cmlt.8b v0, v0, #0
+; CHECK-GI-NEXT: and.8b v0, v0, v1
+; CHECK-GI-NEXT: addv.8b b0, v0
+; CHECK-GI-NEXT: fmov w0, s0
; CHECK-GI-NEXT: ret
%cmp_result = icmp ne <4 x i32> %vec, zeroinitializer
@@ -1003,197 +815,20 @@ define <2 x i8> @vector_to_vector_cast(<16 x i1> %arg) nounwind {
;
; CHECK-GI-LABEL: vector_to_vector_cast:
; CHECK-GI: ; %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: umov.b w8, v0[1]
; CHECK-GI-NEXT: mov d1, v0[1]
-; CHECK-GI-NEXT: umov.b w10, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w13, v0[0]
-; CHECK-GI-NEXT: umov.b w14, v0[2]
-; CHECK-GI-NEXT: umov.b w15, v0[3]
-; CHECK-GI-NEXT: umov.b w11, v0[2]
-; CHECK-GI-NEXT: umov.b w16, v0[4]
-; CHECK-GI-NEXT: umov.b w17, v0[5]
-; CHECK-GI-NEXT: umov.b w12, v0[3]
-; CHECK-GI-NEXT: and w8, w8, #0x1
-; CHECK-GI-NEXT: and w10, w10, #0x1
-; CHECK-GI-NEXT: umov.b w0, v1[1]
-; CHECK-GI-NEXT: bfi w9, w8, #1, #31
-; CHECK-GI-NEXT: bfi w13, w10, #1, #31
-; CHECK-GI-NEXT: and w14, w14, #0x1
-; CHECK-GI-NEXT: umov.b w8, v1[0]
-; CHECK-GI-NEXT: umov.b w10, v1[2]
-; CHECK-GI-NEXT: and w15, w15, #0x1
-; CHECK-GI-NEXT: orr w13, w13, w14, lsl #2
-; CHECK-GI-NEXT: umov.b w14, v1[3]
-; CHECK-GI-NEXT: and w11, w11, #0x1
-; CHECK-GI-NEXT: and w0, w0, #0x1
-; CHECK-GI-NEXT: and w16, w16, #0x1
-; CHECK-GI-NEXT: orr w9, w9, w11, lsl #2
-; CHECK-GI-NEXT: orr w13, w13, w15, lsl #3
-; CHECK-GI-NEXT: umov.b w15, v1[4]
-; CHECK-GI-NEXT: umov.b w11, v0[6]
-; CHECK-GI-NEXT: bfi w8, w0, #1, #31
-; CHECK-GI-NEXT: and w10, w10, #0x1
-; CHECK-GI-NEXT: and w17, w17, #0x1
-; CHECK-GI-NEXT: orr w13, w13, w16, lsl #4
-; CHECK-GI-NEXT: and w14, w14, #0x1
-; CHECK-GI-NEXT: umov.b w0, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w10, lsl #2
-; CHECK-GI-NEXT: umov.b w10, v1[5]
-; CHECK-GI-NEXT: umov.b w16, v1[6]
-; CHECK-GI-NEXT: orr w13, w13, w17, lsl #5
-; CHECK-GI-NEXT: umov.b w17, v0[4]
-; CHECK-GI-NEXT: and w15, w15, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w14, lsl #3
-; CHECK-GI-NEXT: and w12, w12, #0x1
-; CHECK-GI-NEXT: and w11, w11, #0x1
-; CHECK-GI-NEXT: umov.b w14, v1[7]
-; CHECK-GI-NEXT: orr w9, w9, w12, lsl #3
-; CHECK-GI-NEXT: orr w11, w13, w11, lsl #6
-; CHECK-GI-NEXT: orr w8, w8, w15, lsl #4
-; CHECK-GI-NEXT: umov.b w15, v0[5]
-; CHECK-GI-NEXT: and w10, w10, #0x1
-; CHECK-GI-NEXT: and w0, w0, #0x1
-; CHECK-GI-NEXT: and w12, w17, #0x1
-; CHECK-GI-NEXT: umov.b w13, v0[1]
-; CHECK-GI-NEXT: orr w8, w8, w10, lsl #5
-; CHECK-GI-NEXT: and w16, w16, #0x1
-; CHECK-GI-NEXT: orr w9, w9, w12, lsl #4
-; CHECK-GI-NEXT: umov.b w10, v0[0]
-; CHECK-GI-NEXT: orr w11, w11, w0, lsl #7
-; CHECK-GI-NEXT: and w14, w14, #0x1
-; CHECK-GI-NEXT: and w12, w15, #0x1
-; CHECK-GI-NEXT: umov.b w15, v0[2]
-; CHECK-GI-NEXT: orr w8, w8, w16, lsl #6
-; CHECK-GI-NEXT: orr w9, w9, w12, lsl #5
-; CHECK-GI-NEXT: umov.b w12, v0[6]
-; CHECK-GI-NEXT: strb w11, [sp, #8]
-; CHECK-GI-NEXT: and w11, w13, #0x1
-; CHECK-GI-NEXT: umov.b w13, v0[3]
-; CHECK-GI-NEXT: orr w8, w8, w14, lsl #7
-; CHECK-GI-NEXT: umov.b w14, v0[7]
-; CHECK-GI-NEXT: ldr b0, [sp, #8]
-; CHECK-GI-NEXT: bfi w10, w11, #1, #31
-; CHECK-GI-NEXT: and w11, w15, #0x1
-; CHECK-GI-NEXT: strb w8, [sp, #9]
-; CHECK-GI-NEXT: umov.b w15, v0[4]
-; CHECK-GI-NEXT: and w8, w12, #0x1
-; CHECK-GI-NEXT: orr w10, w10, w11, lsl #2
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #6
-; CHECK-GI-NEXT: and w9, w13, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[1]
-; CHECK-GI-NEXT: orr w9, w10, w9, lsl #3
-; CHECK-GI-NEXT: umov.b w10, v0[5]
-; CHECK-GI-NEXT: umov.b w12, v0[0]
-; CHECK-GI-NEXT: and w13, w14, #0x1
-; CHECK-GI-NEXT: umov.b w16, v0[2]
-; CHECK-GI-NEXT: umov.b w17, v0[3]
-; CHECK-GI-NEXT: and w14, w15, #0x1
-; CHECK-GI-NEXT: umov.b w15, v0[2]
-; CHECK-GI-NEXT: orr w8, w8, w13, lsl #7
-; CHECK-GI-NEXT: orr w9, w9, w14, lsl #4
-; CHECK-GI-NEXT: umov.b w13, v0[6]
-; CHECK-GI-NEXT: and w11, w11, #0x1
-; CHECK-GI-NEXT: umov.b w14, v0[3]
-; CHECK-GI-NEXT: strb w8, [sp, #10]
-; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: bfi w12, w11, #1, #31
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #5
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: and w9, w15, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: umov.b w15, v0[1]
-; CHECK-GI-NEXT: orr w9, w12, w9, lsl #2
-; CHECK-GI-NEXT: umov.b w12, v0[5]
-; CHECK-GI-NEXT: and w13, w13, #0x1
-; CHECK-GI-NEXT: and w14, w14, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w13, lsl #6
-; CHECK-GI-NEXT: umov.b w13, v0[0]
-; CHECK-GI-NEXT: orr w9, w9, w14, lsl #3
-; CHECK-GI-NEXT: and w10, w10, #0x1
-; CHECK-GI-NEXT: umov.b w14, v0[6]
-; CHECK-GI-NEXT: and w11, w11, #0x1
-; CHECK-GI-NEXT: and w15, w15, #0x1
-; CHECK-GI-NEXT: umov.b w0, v0[3]
-; CHECK-GI-NEXT: orr w9, w9, w10, lsl #4
-; CHECK-GI-NEXT: and w10, w12, #0x1
-; CHECK-GI-NEXT: umov.b w12, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w11, lsl #7
-; CHECK-GI-NEXT: bfi w13, w15, #1, #31
-; CHECK-GI-NEXT: and w11, w16, #0x1
-; CHECK-GI-NEXT: orr w9, w9, w10, lsl #5
-; CHECK-GI-NEXT: and w10, w14, #0x1
-; CHECK-GI-NEXT: umov.b w14, v0[4]
-; CHECK-GI-NEXT: strb w8, [sp, #11]
-; CHECK-GI-NEXT: umov.b w15, v0[1]
-; CHECK-GI-NEXT: umov.b w16, v0[3]
-; CHECK-GI-NEXT: orr w8, w9, w10, lsl #6
-; CHECK-GI-NEXT: orr w9, w13, w11, lsl #2
-; CHECK-GI-NEXT: and w10, w12, #0x1
-; CHECK-GI-NEXT: and w11, w17, #0x1
-; CHECK-GI-NEXT: umov.b w12, v0[5]
-; CHECK-GI-NEXT: umov.b w17, v0[0]
-; CHECK-GI-NEXT: orr w8, w8, w10, lsl #7
-; CHECK-GI-NEXT: orr w9, w9, w11, lsl #3
-; CHECK-GI-NEXT: umov.b w10, v0[1]
-; CHECK-GI-NEXT: and w11, w14, #0x1
-; CHECK-GI-NEXT: umov.b w14, v0[0]
-; CHECK-GI-NEXT: and w15, w15, #0x1
-; CHECK-GI-NEXT: orr w9, w9, w11, lsl #4
-; CHECK-GI-NEXT: umov.b w11, v0[2]
-; CHECK-GI-NEXT: umov.b w13, v0[6]
-; CHECK-GI-NEXT: and w12, w12, #0x1
-; CHECK-GI-NEXT: bfi w17, w15, #1, #31
-; CHECK-GI-NEXT: umov.b w15, v0[5]
-; CHECK-GI-NEXT: orr w9, w9, w12, lsl #5
-; CHECK-GI-NEXT: and w10, w10, #0x1
-; CHECK-GI-NEXT: umov.b w12, v0[2]
-; CHECK-GI-NEXT: bfi w14, w10, #1, #31
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: ldr b1, [sp, #9]
-; CHECK-GI-NEXT: and w11, w11, #0x1
-; CHECK-GI-NEXT: and w13, w13, #0x1
-; CHECK-GI-NEXT: strb w8, [sp, #12]
-; CHECK-GI-NEXT: orr w11, w14, w11, lsl #2
-; CHECK-GI-NEXT: and w14, w16, #0x1
-; CHECK-GI-NEXT: umov.b w16, v0[4]
-; CHECK-GI-NEXT: and w12, w12, #0x1
-; CHECK-GI-NEXT: and w15, w15, #0x1
-; CHECK-GI-NEXT: orr w9, w9, w13, lsl #6
-; CHECK-GI-NEXT: orr w11, w11, w14, lsl #3
-; CHECK-GI-NEXT: orr w12, w17, w12, lsl #2
-; CHECK-GI-NEXT: and w10, w10, #0x1
-; CHECK-GI-NEXT: and w17, w0, #0x1
-; CHECK-GI-NEXT: umov.b w0, v0[5]
-; CHECK-GI-NEXT: umov.b w14, v0[6]
-; CHECK-GI-NEXT: orr w10, w11, w10, lsl #4
-; CHECK-GI-NEXT: orr w12, w12, w17, lsl #3
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: and w16, w16, #0x1
-; CHECK-GI-NEXT: umov.b w17, v0[6]
-; CHECK-GI-NEXT: orr w10, w10, w15, lsl #5
-; CHECK-GI-NEXT: umov.b w15, v0[7]
-; CHECK-GI-NEXT: orr w12, w12, w16, lsl #4
-; CHECK-GI-NEXT: and w16, w0, #0x1
-; CHECK-GI-NEXT: umov.b w0, v0[7]
-; CHECK-GI-NEXT: and w14, w14, #0x1
-; CHECK-GI-NEXT: orr w12, w12, w16, lsl #5
-; CHECK-GI-NEXT: orr w10, w10, w14, lsl #6
-; CHECK-GI-NEXT: and w11, w11, #0x1
-; CHECK-GI-NEXT: and w13, w17, #0x1
-; CHECK-GI-NEXT: orr w9, w9, w11, lsl #7
-; CHECK-GI-NEXT: mov.s v0[1], v1[0]
-; CHECK-GI-NEXT: orr w11, w12, w13, lsl #6
-; CHECK-GI-NEXT: and w12, w15, #0x1
+; CHECK-GI-NEXT: adrp x8, lCPI20_0 at PAGE
+; CHECK-GI-NEXT: shl.8b v0, v0, #7
+; CHECK-GI-NEXT: ldr d2, [x8, lCPI20_0 at PAGEOFF]
+; CHECK-GI-NEXT: cmlt.8b v0, v0, #0
+; CHECK-GI-NEXT: shl.8b v1, v1, #7
+; CHECK-GI-NEXT: and.8b v0, v0, v2
+; CHECK-GI-NEXT: cmlt.8b v1, v1, #0
+; CHECK-GI-NEXT: addv.8b b0, v0
+; CHECK-GI-NEXT: and.8b v1, v1, v2
+; CHECK-GI-NEXT: addv.8b b1, v1
+; CHECK-GI-NEXT: fmov w8, s1
+; CHECK-GI-NEXT: mov.s v0[1], w8
; CHECK-GI-NEXT: ; kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT: orr w8, w10, w12, lsl #7
-; CHECK-GI-NEXT: and w10, w0, #0x1
-; CHECK-GI-NEXT: strb w9, [sp, #13]
-; CHECK-GI-NEXT: orr w9, w11, w10, lsl #7
-; CHECK-GI-NEXT: strb w8, [sp, #14]
-; CHECK-GI-NEXT: strb w9, [sp, #15]
-; CHECK-GI-NEXT: add sp, sp, #16
; CHECK-GI-NEXT: ret
%bc = bitcast <16 x i1> %arg to <2 x i8>
ret <2 x i8> %bc
>From 0b4ee2d456792274e5cda9cd87903d301c06d043 Mon Sep 17 00:00:00 2001
From: Joel-Wwalker <theagingboy05 at gmail.com>
Date: Wed, 5 Aug 2026 06:22:58 -0400
Subject: [PATCH 2/2] [AArch64][GlobalISel] Legalize <8 x i1> and <16 x i1>
loads and stores
G_LOAD of <8 x i1> / <16 x i1> failed to legalize, and stores of these
types expanded bit by bit, about 17 instructions for what SelectionDAG
does with one strb.
Legalize loads by bitcasting to the equivalent scalar, as v4s8 already
does, and fold the inverse cast this creates so the common
load-then-cast pattern stays a single scalar load. Stores are handled
as custom before minScalarOrElt widens the elements away: always store
the equivalent integer; the bitcast this introduces folds with the
value's definition or is packed in vector registers by bitcast
legalization (#213946).
A compare fed store of <8 x i1> now matches SelectionDAG exactly, and
the i1 vector copy becomes an integer load/store pair.
Fixes #116006.
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 48 ++++++-
.../AArch64/GISel/AArch64LegalizerInfo.h | 1 +
.../GlobalISel/legalize-load-store.mir | 83 +++++++++++
.../legalize-store-vector-bools.mir | 69 ++--------
.../test/CodeGen/AArch64/vec-i1-load-store.ll | 129 ++++++++++++++++++
5 files changed, 274 insertions(+), 56 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/vec-i1-load-store.ll
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index a17cb89e64d81..1da96291b47ae 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -48,6 +48,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
const LLT s32 = LLT::scalar(32);
const LLT s64 = LLT::scalar(64);
const LLT s128 = LLT::scalar(128);
+ const LLT v16s1 = LLT::fixed_vector(16, 1);
+ const LLT v8s1 = LLT::fixed_vector(8, 1);
const LLT v16s8 = LLT::fixed_vector(16, 8);
const LLT v8s8 = LLT::fixed_vector(8, 8);
const LLT v4s8 = LLT::fixed_vector(4, 8);
@@ -595,7 +597,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
},
changeTo(0, s32))
// TODO: Use BITCAST for v2i8, v2i16 after G_TRUNC gets sorted out
- .bitcastIf(typeInSet(0, {v4s8}),
+ .bitcastIf(typeInSet(0, {v4s8, v8s1, v16s1}),
[=](const LegalityQuery &Query) {
const LLT VecTy = Query.Types[0];
return std::pair(0, LLT::integer(VecTy.getSizeInBits()));
@@ -638,6 +640,12 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
{nxv4s32, p0, nxv4s32, 8},
{nxv2s64, p0, nxv2s64, 8},
})
+ // Handle i1 vector stores before the elements are widened to s8.
+ .customIf(all(typeInSet(0, {v8s1, v16s1}),
+ LegalityPredicate([=](const LegalityQuery &Query) {
+ return Query.Types[0].getSizeInBits() ==
+ Query.MMODescrs[0].MemoryTy.getSizeInBits();
+ })))
.clampScalar(0, s8, s64)
.minScalarOrElt(0, s8)
.lowerIf([=](const LegalityQuery &Query) {
@@ -1553,8 +1561,12 @@ bool AArch64LegalizerInfo::legalizeCustom(
case TargetOpcode::G_VAARG:
return legalizeVaArg(MI, MRI, MIRBuilder);
case TargetOpcode::G_LOAD:
- case TargetOpcode::G_STORE:
+ case TargetOpcode::G_STORE: {
+ LLT ValTy = MRI.getType(MI.getOperand(0).getReg());
+ if (ValTy.isVector() && ValTy.getScalarSizeInBits() == 1)
+ return legalizeI1VecStore(MI, Helper);
return legalizeLoadStore(MI, MRI, MIRBuilder, Observer);
+ }
case TargetOpcode::G_SHL:
case TargetOpcode::G_ASHR:
case TargetOpcode::G_LSHR:
@@ -1604,6 +1616,27 @@ bool AArch64LegalizerInfo::legalizeCustom(
llvm_unreachable("expected switch to return");
}
+bool AArch64LegalizerInfo::legalizeI1VecStore(MachineInstr &MI,
+ LegalizerHelper &Helper) const {
+ auto &Store = cast<GStore>(MI);
+ MachineIRBuilder &MIB = Helper.MIRBuilder;
+ MachineRegisterInfo &MRI = *MIB.getMRI();
+ Register ValReg = Store.getValueReg();
+ LLT ValTy = MRI.getType(ValReg);
+ MachineFunction &MF = MIB.getMF();
+
+ // Store the equivalent integer instead. The bitcast either folds with the
+ // value's definition or is packed in vector registers by legalizeBitcast.
+ LLT IntTy = LLT::integer(ValTy.getSizeInBits());
+ MachineMemOperand *NewMMO =
+ MF.getMachineMemOperand(&Store.getMMO(), /*Offset=*/0, IntTy);
+ MIB.setInstrAndDebugLoc(MI);
+ auto Cast = MIB.buildBitcast(IntTy, ValReg);
+ MIB.buildStore(Cast, Store.getPointerReg(), *NewMMO);
+ MI.eraseFromParent();
+ return true;
+}
+
bool AArch64LegalizerInfo::legalizeBitcast(MachineInstr &MI,
LegalizerHelper &Helper) const {
assert(MI.getOpcode() == TargetOpcode::G_BITCAST && "Unexpected opcode");
@@ -1619,6 +1652,17 @@ bool AArch64LegalizerInfo::legalizeBitcast(MachineInstr &MI,
MachineInstr *SrcMI = MRI.getVRegDef(SrcReg);
+ // Fold a cast of a cast from the destination type, such as the inverse
+ // cast created when an i1 vector G_LOAD is legalized by bitcasting to a
+ // scalar.
+ if (SrcMI && SrcMI->getOpcode() == TargetOpcode::G_BITCAST &&
+ MRI.getType(SrcMI->getOperand(1).getReg()) == DstTy) {
+ MIB.setInstrAndDebugLoc(MI);
+ MIB.buildCopy(DstReg, SrcMI->getOperand(1).getReg());
+ MI.eraseFromParent();
+ return true;
+ }
+
// Fold a cast of an i1 vector G_LOAD into a scalar load of the same
// memory, instead of expanding the vector value bit by bit.
if (SrcMI && SrcMI->getOpcode() == TargetOpcode::G_LOAD &&
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
index 535240a67a2cc..fcd09b69d7ec1 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
@@ -67,6 +67,7 @@ class AArch64LegalizerInfo : public LegalizerInfo {
bool legalizeDynStackAlloc(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizePrefetch(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizeBitcast(MachineInstr &MI, LegalizerHelper &Helper) const;
+ bool legalizeI1VecStore(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizeConcatVectors(MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &MIRBuilder) const;
bool legalizeFptrunc(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir
index ab4ecc5241a26..74acbeb84a362 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir
@@ -736,3 +736,86 @@ body: |
%val:_(<6 x p0>) = G_LOAD %ptr(p0) :: (load (<6 x p0>))
G_STORE %val(<6 x p0>), %ptr(p0) :: (store (<6 x p0>))
RET_ReallyLR
+...
+---
+name: load_v8s1
+body: |
+ bb.0:
+ liveins: $x0
+ ; CHECK-LABEL: name: load_v8s1
+ ; CHECK: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+ ; CHECK-NEXT: %bc:_(i8) = G_LOAD %ptr(p0) :: (load (i8))
+ ; CHECK-NEXT: %ext:_(i64) = G_ANYEXT %bc(i8)
+ ; CHECK-NEXT: $x0 = COPY %ext(i64)
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ %ptr:_(p0) = COPY $x0
+ %val:_(<8 x i1>) = G_LOAD %ptr(p0) :: (load (<8 x i1>))
+ %bc:_(i8) = G_BITCAST %val(<8 x i1>)
+ %ext:_(i64) = G_ANYEXT %bc(i8)
+ $x0 = COPY %ext(i64)
+ RET_ReallyLR implicit $x0
+...
+---
+name: load_v16s1
+body: |
+ bb.0:
+ liveins: $x0
+ ; CHECK-LABEL: name: load_v16s1
+ ; CHECK: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+ ; CHECK-NEXT: %bc:_(i16) = G_LOAD %ptr(p0) :: (load (i16))
+ ; CHECK-NEXT: %ext:_(i64) = G_ANYEXT %bc(i16)
+ ; CHECK-NEXT: $x0 = COPY %ext(i64)
+ ; CHECK-NEXT: RET_ReallyLR implicit $x0
+ %ptr:_(p0) = COPY $x0
+ %val:_(<16 x i1>) = G_LOAD %ptr(p0) :: (load (<16 x i1>))
+ %bc:_(i16) = G_BITCAST %val(<16 x i1>)
+ %ext:_(i64) = G_ANYEXT %bc(i16)
+ $x0 = COPY %ext(i64)
+ RET_ReallyLR implicit $x0
+...
+---
+name: store_v8s1
+body: |
+ bb.0:
+ liveins: $x0, $w1
+ ; CHECK-LABEL: name: store_v8s1
+ ; CHECK: liveins: $x0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+ ; CHECK-NEXT: %in:_(i32) = COPY $w1
+ ; CHECK-NEXT: %tr:_(i8) = G_TRUNC %in(i32)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i8) = COPY %tr(i8)
+ ; CHECK-NEXT: G_STORE [[COPY]](i8), %ptr(p0) :: (store (i8))
+ ; CHECK-NEXT: RET_ReallyLR
+ %ptr:_(p0) = COPY $x0
+ %in:_(i32) = COPY $w1
+ %tr:_(i8) = G_TRUNC %in(i32)
+ %bc:_(<8 x i1>) = G_BITCAST %tr(i8)
+ G_STORE %bc(<8 x i1>), %ptr(p0) :: (store (<8 x i1>))
+ RET_ReallyLR
+...
+---
+name: store_v16s1
+body: |
+ bb.0:
+ liveins: $x0, $w1
+ ; CHECK-LABEL: name: store_v16s1
+ ; CHECK: liveins: $x0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+ ; CHECK-NEXT: %in:_(i32) = COPY $w1
+ ; CHECK-NEXT: %tr:_(i16) = G_TRUNC %in(i32)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i16) = COPY %tr(i16)
+ ; CHECK-NEXT: G_STORE [[COPY]](i16), %ptr(p0) :: (store (i16))
+ ; CHECK-NEXT: RET_ReallyLR
+ %ptr:_(p0) = COPY $x0
+ %in:_(i32) = COPY $w1
+ %tr:_(i16) = G_TRUNC %in(i32)
+ %bc:_(<16 x i1>) = G_BITCAST %tr(i16)
+ G_STORE %bc(<16 x i1>), %ptr(p0) :: (store (<16 x i1>))
+ RET_ReallyLR
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-store-vector-bools.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-store-vector-bools.mir
index 84e4ca21a5575..caf87f5159626 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-store-vector-bools.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-store-vector-bools.mir
@@ -19,60 +19,21 @@ body: |
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(<4 x s32>) = G_ICMP intpred(slt), [[COPY1]](<4 x s32>), [[BUILD_VECTOR1]]
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[ICMP]](<4 x s32>)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[ICMP1]](<4 x s32>)
- ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[TRUNC]](<4 x s16>), [[TRUNC1]](<4 x s16>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(<8 x s8>) = G_TRUNC [[CONCAT_VECTORS]](<8 x s16>)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
- ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[TRUNC2]](<8 x s8>), [[C1]](i64)
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC]](s8)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ANYEXT]], [[C2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C1]](i64)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL]]
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
- ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[TRUNC2]](<8 x s8>), [[C4]](i64)
- ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC1]](s8)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ANYEXT1]], [[C2]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i64)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
- ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
- ; CHECK-NEXT: [[EVEC2:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[TRUNC2]](<8 x s8>), [[C5]](i64)
- ; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC2]](s8)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ANYEXT2]], [[C2]]
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C5]](i64)
- ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
- ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i64) = G_CONSTANT i64 3
- ; CHECK-NEXT: [[EVEC3:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[TRUNC2]](<8 x s8>), [[C6]](i64)
- ; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC3]](s8)
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ANYEXT3]], [[C2]]
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C6]](i64)
- ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[OR2]], [[SHL3]]
- ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
- ; CHECK-NEXT: [[EVEC4:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[TRUNC2]](<8 x s8>), [[C7]](i64)
- ; CHECK-NEXT: [[ANYEXT4:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC4]](s8)
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[ANYEXT4]], [[C2]]
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND4]], [[C7]](i64)
- ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[OR3]], [[SHL4]]
- ; CHECK-NEXT: [[C8:%[0-9]+]]:_(i64) = G_CONSTANT i64 5
- ; CHECK-NEXT: [[EVEC5:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[TRUNC2]](<8 x s8>), [[C8]](i64)
- ; CHECK-NEXT: [[ANYEXT5:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC5]](s8)
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[ANYEXT5]], [[C2]]
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C8]](i64)
- ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[OR4]], [[SHL5]]
- ; CHECK-NEXT: [[C9:%[0-9]+]]:_(i64) = G_CONSTANT i64 6
- ; CHECK-NEXT: [[EVEC6:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[TRUNC2]](<8 x s8>), [[C9]](i64)
- ; CHECK-NEXT: [[ANYEXT6:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC6]](s8)
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[ANYEXT6]], [[C2]]
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND6]], [[C9]](i64)
- ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[OR5]], [[SHL6]]
- ; CHECK-NEXT: [[C10:%[0-9]+]]:_(i64) = G_CONSTANT i64 7
- ; CHECK-NEXT: [[EVEC7:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[TRUNC2]](<8 x s8>), [[C10]](i64)
- ; CHECK-NEXT: [[ANYEXT7:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC7]](s8)
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[ANYEXT7]], [[C2]]
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C10]](i64)
- ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[OR6]], [[SHL7]]
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(i8) = G_TRUNC [[OR7]](i32)
- ; CHECK-NEXT: G_STORE [[TRUNC3]](i8), %ptr(p0) :: (store (i8))
+ ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x i16>) = G_CONCAT_VECTORS [[TRUNC]](<4 x s16>), [[TRUNC1]](<4 x s16>)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(<8 x i8>) = G_TRUNC [[CONCAT_VECTORS]](<8 x i16>)
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(<8 x i8>) = G_SEXT_INREG [[TRUNC2]], 1
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i8) = G_CONSTANT i8 1
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i8) = G_CONSTANT i8 2
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i8) = G_CONSTANT i8 4
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i8) = G_CONSTANT i8 8
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i8) = G_CONSTANT i8 16
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i8) = G_CONSTANT i8 32
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i8) = G_CONSTANT i8 64
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:_(i8) = G_CONSTANT i8 -128
+ ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x i8>) = G_BUILD_VECTOR [[C1]](i8), [[C2]](i8), [[C3]](i8), [[C4]](i8), [[C5]](i8), [[C6]](i8), [[C7]](i8), [[C8]](i8)
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<8 x i8>) = G_AND [[SEXT_INREG]], [[BUILD_VECTOR2]]
+ ; CHECK-NEXT: [[VECREDUCE_ADD:%[0-9]+]]:_(i8) = G_VECREDUCE_ADD [[AND]](<8 x i8>)
+ ; CHECK-NEXT: G_STORE [[VECREDUCE_ADD]](i8), %ptr(p0) :: (store (i8))
; CHECK-NEXT: RET_ReallyLR
%1:_(<4 x s32>) = COPY $q0
%2:_(<4 x s32>) = COPY $q1
diff --git a/llvm/test/CodeGen/AArch64/vec-i1-load-store.ll b/llvm/test/CodeGen/AArch64/vec-i1-load-store.ll
new file mode 100644
index 0000000000000..a2f1092d95273
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vec-i1-load-store.ll
@@ -0,0 +1,129 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64 -global-isel < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+define i8 @load_v8i1(ptr %d) {
+; CHECK-LABEL: load_v8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrb w0, [x0]
+; CHECK-NEXT: ret
+ %ld = load <8 x i1>, ptr %d
+ %b = bitcast <8 x i1> %ld to i8
+ ret i8 %b
+}
+
+define i16 @load_v16i1(ptr %d) {
+; CHECK-LABEL: load_v16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldrh w0, [x0]
+; CHECK-NEXT: ret
+ %ld = load <16 x i1>, ptr %d
+ %b = bitcast <16 x i1> %ld to i16
+ ret i16 %b
+}
+
+define void @store_v8i1(ptr %d, i8 %v) {
+; CHECK-LABEL: store_v8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: strb w1, [x0]
+; CHECK-NEXT: ret
+ %b = bitcast i8 %v to <8 x i1>
+ store <8 x i1> %b, ptr %d
+ ret void
+}
+
+define void @store_v16i1(ptr %d, i16 %v) {
+; CHECK-LABEL: store_v16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: strh w1, [x0]
+; CHECK-NEXT: ret
+ %b = bitcast i16 %v to <16 x i1>
+ store <16 x i1> %b, ptr %d
+ ret void
+}
+
+define void @copy_v8i1(ptr %s, ptr %d) {
+; CHECK-SD-LABEL: copy_v8i1:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: ldrb w8, [x0]
+; CHECK-SD-NEXT: sbfx w9, w8, #0, #1
+; CHECK-SD-NEXT: sbfx w10, w8, #1, #1
+; CHECK-SD-NEXT: fmov s0, w9
+; CHECK-SD-NEXT: sbfx w9, w8, #2, #1
+; CHECK-SD-NEXT: mov v0.b[1], w10
+; CHECK-SD-NEXT: mov v0.b[2], w9
+; CHECK-SD-NEXT: sbfx w9, w8, #3, #1
+; CHECK-SD-NEXT: mov v0.b[3], w9
+; CHECK-SD-NEXT: sbfx w9, w8, #4, #1
+; CHECK-SD-NEXT: mov v0.b[4], w9
+; CHECK-SD-NEXT: sbfx w9, w8, #5, #1
+; CHECK-SD-NEXT: mov v0.b[5], w9
+; CHECK-SD-NEXT: sbfx w9, w8, #6, #1
+; CHECK-SD-NEXT: sbfx w8, w8, #7, #1
+; CHECK-SD-NEXT: mov v0.b[6], w9
+; CHECK-SD-NEXT: mov v0.b[7], w8
+; CHECK-SD-NEXT: adrp x8, .LCPI4_0
+; CHECK-SD-NEXT: ldr d1, [x8, :lo12:.LCPI4_0]
+; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: addv b0, v0.8b
+; CHECK-SD-NEXT: str b0, [x1]
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: copy_v8i1:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldrb w8, [x0]
+; CHECK-GI-NEXT: strb w8, [x1]
+; CHECK-GI-NEXT: ret
+ %ld = load <8 x i1>, ptr %s
+ store <8 x i1> %ld, ptr %d
+ ret void
+}
+
+define i1 @extract_v8i1(ptr %d) {
+; CHECK-SD-LABEL: extract_v8i1:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: ldrb w8, [x0]
+; CHECK-SD-NEXT: ubfx w0, w8, #3, #1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: extract_v8i1:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: ldrb w8, [x0]
+; CHECK-GI-NEXT: fmov s0, w8
+; CHECK-GI-NEXT: lsr w9, w8, #1
+; CHECK-GI-NEXT: mov v0.b[1], w9
+; CHECK-GI-NEXT: lsr w9, w8, #2
+; CHECK-GI-NEXT: mov v0.b[2], w9
+; CHECK-GI-NEXT: lsr w9, w8, #3
+; CHECK-GI-NEXT: mov v0.b[3], w9
+; CHECK-GI-NEXT: lsr w9, w8, #4
+; CHECK-GI-NEXT: mov v0.b[4], w9
+; CHECK-GI-NEXT: lsr w9, w8, #5
+; CHECK-GI-NEXT: mov v0.b[5], w9
+; CHECK-GI-NEXT: lsr w9, w8, #6
+; CHECK-GI-NEXT: lsr w8, w8, #7
+; CHECK-GI-NEXT: mov v0.b[6], w9
+; CHECK-GI-NEXT: mov v0.b[7], w8
+; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: umov w8, v0.h[3]
+; CHECK-GI-NEXT: and w0, w8, #0x1
+; CHECK-GI-NEXT: ret
+ %ld = load <8 x i1>, ptr %d
+ %e = extractelement <8 x i1> %ld, i32 3
+ ret i1 %e
+}
+
+define void @store_v8i1_icmp(ptr %d, <8 x i8> %x, <8 x i8> %y) {
+; CHECK-LABEL: store_v8i1_icmp:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x8, .LCPI6_0
+; CHECK-NEXT: cmeq v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI6_0]
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: addv b0, v0.8b
+; CHECK-NEXT: str b0, [x0]
+; CHECK-NEXT: ret
+ %c = icmp eq <8 x i8> %x, %y
+ store <8 x i1> %c, ptr %d
+ ret void
+}
More information about the llvm-commits
mailing list