[llvm] [AArch64][GlobalISel] Pack i1 vector to scalar bitcasts in vector registers (PR #213946)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 07:27:01 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-globalisel
Author: Joel Walker (Joel-Wwalker)
<details>
<summary>Changes</summary>
Casting an i1 vector to a scalar currently round-trips the stack: the mask is stored bit by bit and reloaded as an integer, about 50 instructions and a stack slot for a 16-lane compare result.
Legalize these casts the way SelectionDAG lowers them instead: sign-extend the lanes to all-ones or all-zeros, mask lane i down to the weight 1 << i, and add-reduce, which selects to cmlt/and/addv. The pack applies to 8 and 16 lanes, where the reduction is efficient; other lane counts keep the stack round-trip.
Also fold casts whose source is another bitcast from the destination type, or a single-use i1 vector load of the same memory, so those become a plain copy or scalar load instead of being packed. The load fold already lets load-then-cast of `<8 x i1>` compile to a single ldrb; the bare vector load still needs the rules in #<!-- -->210514.
Split out of #<!-- -->210514 per review there; that PR adds the i1 vector load/store rules on top of this one.
Assisted by Claude (Anthropic).
---
Patch is 31.54 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/213946.diff
3 Files Affected:
- (modified) llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp (+53-1)
- (modified) llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir (+12-54)
- (modified) llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll (+78-443)
``````````diff
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 94ca171c0b207..e25f2e493a865 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1608,13 +1608,65 @@ bool AArch64LegalizerInfo::legalizeBitcast(MachineInstr &MI,
LegalizerHelper &Helper) const {
assert(MI.getOpcode() == TargetOpcode::G_BITCAST && "Unexpected opcode");
auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
+ MachineIRBuilder &MIB = Helper.MIRBuilder;
+ MachineRegisterInfo &MRI = *MIB.getMRI();
+
// We're trying to handle casts from i1 vectors to scalars but reloading from
// stack.
if (!DstTy.isScalar() || !SrcTy.isVector() ||
SrcTy.getElementType() != LLT::scalar(1))
return false;
- Helper.createStackStoreLoad(DstReg, SrcReg);
+ MachineInstr *SrcMI = MRI.getVRegDef(SrcReg);
+
+ // Fold a cast of a cast from the destination type, such as the inverse
+ // cast created when an i1 vector G_LOAD is legalized by bitcasting to a
+ // scalar.
+ if (SrcMI && SrcMI->getOpcode() == TargetOpcode::G_BITCAST &&
+ MRI.getType(SrcMI->getOperand(1).getReg()) == DstTy) {
+ MIB.setInstrAndDebugLoc(MI);
+ MIB.buildCopy(DstReg, SrcMI->getOperand(1).getReg());
+ MI.eraseFromParent();
+ return true;
+ }
+
+ // Fold a cast of an i1 vector G_LOAD into a scalar load of the same
+ // memory, instead of expanding the vector value bit by bit.
+ if (SrcMI && SrcMI->getOpcode() == TargetOpcode::G_LOAD &&
+ MRI.hasOneNonDBGUse(SrcReg)) {
+ auto *Load = cast<GLoad>(SrcMI);
+ MachineFunction &MF = MIB.getMF();
+ MachineMemOperand *NewMMO =
+ MF.getMachineMemOperand(&Load->getMMO(), /*Offset=*/0, DstTy);
+ MIB.setInstrAndDebugLoc(MI);
+ MIB.buildLoad(DstReg, Load->getPointerReg(), *NewMMO);
+ MI.eraseFromParent();
+ return true;
+ }
+
+ // Pack the mask in vector registers, as SelectionDAG does: sign-extend
+ // each lane to all-ones or all-zeros, mask lane i down to the value
+ // 1 << i, and sum the lanes. The lane count always equals the destination
+ // bit width here, so the weights fit the widened element type. Only do
+ // this for lane counts with an efficient reduction; other sizes keep the
+ // stack round-trip below.
+ unsigned NumElts = SrcTy.getNumElements();
+ if (NumElts != 8 && NumElts != 16) {
+ Helper.createStackStoreLoad(DstReg, SrcReg);
+ MI.eraseFromParent();
+ return true;
+ }
+ MIB.setInstrAndDebugLoc(MI);
+ LLT WideVecTy = LLT::fixed_vector(NumElts, DstTy);
+ auto Sext = MIB.buildSExt(WideVecTy, SrcReg);
+ SmallVector<Register> Weights;
+ for (unsigned I = 0; I != NumElts; ++I)
+ Weights.push_back(
+ MIB.buildConstant(DstTy, APInt::getOneBitSet(DstTy.getSizeInBits(), I))
+ .getReg(0));
+ auto Mask = MIB.buildBuildVector(WideVecTy, Weights);
+ auto Masked = MIB.buildAnd(WideVecTy, Sext, Mask);
+ MIB.buildInstr(TargetOpcode::G_VECREDUCE_ADD, {DstReg}, {Masked});
MI.eraseFromParent();
return true;
}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
index f08db556d14d8..5ef6e457ad6ec 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
@@ -55,60 +55,18 @@ body: |
bb.1:
; CHECK-LABEL: name: boolean_vector_to_scalar
; CHECK: [[DEF:%[0-9]+]]:_(<8 x i8>) = G_IMPLICIT_DEF
- ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
- ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C]](i64)
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC]](i8)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ANYEXT]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C]](i64)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
- ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C3]](i64)
- ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC1]](i8)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ANYEXT1]], [[C1]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i64)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
- ; CHECK-NEXT: [[EVEC2:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C4]](i64)
- ; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC2]](i8)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ANYEXT2]], [[C1]]
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C4]](i64)
- ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
- ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 3
- ; CHECK-NEXT: [[EVEC3:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C5]](i64)
- ; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC3]](i8)
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ANYEXT3]], [[C1]]
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C5]](i64)
- ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[OR2]], [[SHL3]]
- ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
- ; CHECK-NEXT: [[EVEC4:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C6]](i64)
- ; CHECK-NEXT: [[ANYEXT4:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC4]](i8)
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[ANYEXT4]], [[C1]]
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND4]], [[C6]](i64)
- ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[OR3]], [[SHL4]]
- ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i64) = G_CONSTANT i64 5
- ; CHECK-NEXT: [[EVEC5:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C7]](i64)
- ; CHECK-NEXT: [[ANYEXT5:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC5]](i8)
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[ANYEXT5]], [[C1]]
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i64)
- ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[OR4]], [[SHL5]]
- ; CHECK-NEXT: [[C8:%[0-9]+]]:_(i64) = G_CONSTANT i64 6
- ; CHECK-NEXT: [[EVEC6:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C8]](i64)
- ; CHECK-NEXT: [[ANYEXT6:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC6]](i8)
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[ANYEXT6]], [[C1]]
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND6]], [[C8]](i64)
- ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[OR5]], [[SHL6]]
- ; CHECK-NEXT: [[C9:%[0-9]+]]:_(i64) = G_CONSTANT i64 7
- ; CHECK-NEXT: [[EVEC7:%[0-9]+]]:_(i8) = G_EXTRACT_VECTOR_ELT [[DEF]](<8 x i8>), [[C9]](i64)
- ; CHECK-NEXT: [[ANYEXT7:%[0-9]+]]:_(i32) = G_ANYEXT [[EVEC7]](i8)
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[ANYEXT7]], [[C1]]
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C9]](i64)
- ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[OR6]], [[SHL7]]
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[OR7]](i32)
- ; CHECK-NEXT: G_STORE [[TRUNC]](i8), [[FRAME_INDEX]](p0) :: (store (i8) into %stack.0)
- ; CHECK-NEXT: %bc:_(i8) = G_LOAD [[FRAME_INDEX]](p0) :: (load (i8) from %stack.0)
+ ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(<8 x i8>) = G_SEXT_INREG [[DEF]], 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i8) = G_CONSTANT i8 1
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i8) = G_CONSTANT i8 2
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i8) = G_CONSTANT i8 4
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i8) = G_CONSTANT i8 8
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i8) = G_CONSTANT i8 16
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i8) = G_CONSTANT i8 32
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i8) = G_CONSTANT i8 64
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i8) = G_CONSTANT i8 -128
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i8>) = G_BUILD_VECTOR [[C]](i8), [[C1]](i8), [[C2]](i8), [[C3]](i8), [[C4]](i8), [[C5]](i8), [[C6]](i8), [[C7]](i8)
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<8 x i8>) = G_AND [[SEXT_INREG]], [[BUILD_VECTOR]]
+ ; CHECK-NEXT: %bc:_(i8) = G_VECREDUCE_ADD [[AND]](<8 x i8>)
; CHECK-NEXT: %ext:_(i32) = G_ANYEXT %bc(i8)
; CHECK-NEXT: $w0 = COPY %ext(i32)
; CHECK-NEXT: RET_ReallyLR implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
index f9ecae443d399..a50bb39c0159f 100644
--- a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
@@ -25,58 +25,22 @@ define i16 @convert_to_bitmask16(<16 x i8> %vec) {
;
; CHECK-GI-LABEL: convert_to_bitmask16:
; CHECK-GI: ; %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: cmtst.16b v0, v0, v0
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
-; CHECK-GI-NEXT: and w8, w8, #0x1
-; CHECK-GI-NEXT: bfi w9, w8, #1, #31
-; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[8]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[9]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[10]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #8
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[11]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #9
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[12]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #10
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[13]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #11
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[14]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #12
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[15]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #13
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #14
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #15
-; CHECK-GI-NEXT: strh w8, [sp, #14]
-; CHECK-GI-NEXT: and w0, w8, #0xffff
-; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: adrp x8, lCPI0_1 at PAGE
+; CHECK-GI-NEXT: adrp x9, lCPI0_0 at PAGE
+; CHECK-GI-NEXT: ldr q2, [x8, lCPI0_1 at PAGEOFF]
+; CHECK-GI-NEXT: ldr q3, [x9, lCPI0_0 at PAGEOFF]
+; CHECK-GI-NEXT: ushll.8h v1, v0, #0
+; CHECK-GI-NEXT: ushll2.8h v0, v0, #0
+; CHECK-GI-NEXT: shl.8h v1, v1, #15
+; CHECK-GI-NEXT: shl.8h v0, v0, #15
+; CHECK-GI-NEXT: cmlt.8h v1, v1, #0
+; CHECK-GI-NEXT: cmlt.8h v0, v0, #0
+; CHECK-GI-NEXT: and.16b v1, v1, v2
+; CHECK-GI-NEXT: and.16b v0, v0, v3
+; CHECK-GI-NEXT: add.8h v0, v1, v0
+; CHECK-GI-NEXT: addv.8h h0, v0
+; CHECK-GI-NEXT: fmov w0, s0
; CHECK-GI-NEXT: ret
; Actual conversion
@@ -100,35 +64,14 @@ define i16 @convert_to_bitmask8(<8 x i16> %vec) {
;
; CHECK-GI-LABEL: convert_to_bitmask8:
; CHECK-GI: ; %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: cmtst.8h v0, v0, v0
+; CHECK-GI-NEXT: adrp x8, lCPI1_0 at PAGE
+; CHECK-GI-NEXT: ldr d1, [x8, lCPI1_0 at PAGEOFF]
; CHECK-GI-NEXT: xtn.8b v0, v0
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
-; CHECK-GI-NEXT: and w8, w8, #0x1
-; CHECK-GI-NEXT: bfi w9, w8, #1, #31
-; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
-; CHECK-GI-NEXT: strb w8, [sp, #15]
-; CHECK-GI-NEXT: and w0, w8, #0xff
-; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: and.8b v0, v0, v1
+; CHECK-GI-NEXT: addv.8b b0, v0
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: uxtb w0, w8
; CHECK-GI-NEXT: ret
@@ -208,36 +151,16 @@ define i8 @clang_builtins_undef_concat_convert_to_bitmask4(<4 x i32> %vec) {
;
; CHECK-GI-LABEL: clang_builtins_undef_concat_convert_to_bitmask4:
; CHECK-GI: ; %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
; CHECK-GI-NEXT: cmtst.4s v0, v0, v0
+; CHECK-GI-NEXT: adrp x8, lCPI4_0 at PAGE
+; CHECK-GI-NEXT: ldr d1, [x8, lCPI4_0 at PAGEOFF]
; CHECK-GI-NEXT: xtn.4h v0, v0
; CHECK-GI-NEXT: uzp1.8b v0, v0, v0
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
-; CHECK-GI-NEXT: and w8, w8, #0x1
-; CHECK-GI-NEXT: bfi w9, w8, #1, #31
-; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
-; CHECK-GI-NEXT: strb w8, [sp, #15]
-; CHECK-GI-NEXT: and w0, w8, #0xff
-; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: shl.8b v0, v0, #7
+; CHECK-GI-NEXT: cmlt.8b v0, v0, #0
+; CHECK-GI-NEXT: and.8b v0, v0, v1
+; CHECK-GI-NEXT: addv.8b b0, v0
+; CHECK-GI-NEXT: fmov w0, s0
; CHECK-GI-NEXT: ret
@@ -529,57 +452,21 @@ define i16 @convert_to_bitmask_without_knowing_type(<16 x i1> %vec) {
;
; CHECK-GI-LABEL: convert_to_bitmask_without_knowing_type:
; CHECK-GI: ; %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
-; CHECK-GI-NEXT: and w8, w8, #0x1
-; CHECK-GI-NEXT: bfi w9, w8, #1, #31
-; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[8]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[9]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[10]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #8
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[11]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #9
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[12]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #10
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[13]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #11
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[14]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #12
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[15]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #13
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #14
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #15
-; CHECK-GI-NEXT: strh w8, [sp, #14]
-; CHECK-GI-NEXT: and w0, w8, #0xffff
-; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: ushll.8h v1, v0, #0
+; CHECK-GI-NEXT: ushll2.8h v0, v0, #0
+; CHECK-GI-NEXT: adrp x8, lCPI10_1 at PAGE
+; CHECK-GI-NEXT: adrp x9, lCPI10_0 at PAGE
+; CHECK-GI-NEXT: ldr q2, [x8, lCPI10_1 at PAGEOFF]
+; CHECK-GI-NEXT: ldr q3, [x9, lCPI10_0 at PAGEOFF]
+; CHECK-GI-NEXT: shl.8h v1, v1, #15
+; CHECK-GI-NEXT: shl.8h v0, v0, #15
+; CHECK-GI-NEXT: cmlt.8h v1, v1, #0
+; CHECK-GI-NEXT: cmlt.8h v0, v0, #0
+; CHECK-GI-NEXT: and.16b v1, v1, v2
+; CHECK-GI-NEXT: and.16b v0, v0, v3
+; CHECK-GI-NEXT: add.8h v0, v1, v0
+; CHECK-GI-NEXT: addv.8h h0, v0
+; CHECK-GI-NEXT: fmov w0, s0
; CHECK-GI-NEXT: ret
%bitmask = bitcast <16 x i1> %vec to i16
@@ -644,50 +531,17 @@ define i4 @convert_to_bitmask_4xi8(<4 x i8> %vec) {
}
define i8 @convert_to_bitmask_8xi2(<8 x i2> %vec) {
-; CHECK-SD-LABEL: convert_to_bitmask_8xi2:
-; CHECK-SD: ; %bb.0:
-; CHECK-SD-NEXT: movi.8b v1, #3
-; CHECK-SD-NEXT: adrp x8, lCPI13_0 at PAGE
-; CHECK-SD-NEXT: and.8b v0, v0, v1
-; CHECK-SD-NEXT: ldr d1, [x8, lCPI13_0 at PAGEOFF]
-; CHECK-SD-NEXT: cmeq.8b v0, v0, #0
-; CHECK-SD-NEXT: bic.8b v0, v1, v0
-; CHECK-SD-NEXT: addv.8b b0, v0
-; CHECK-SD-NEXT: fmov w0, s0
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: convert_to_bitmask_8xi2:
-; CHECK-GI: ; %bb.0:
-; CHECK-GI-NEXT: sub sp, sp, #16
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT: movi.8b v1, #3
-; CHECK-GI-NEXT: cmtst.8b v0, v0, v1
-; CHECK-GI-NEXT: umov.b w8, v0[1]
-; CHECK-GI-NEXT: umov.b w9, v0[0]
-; CHECK-GI-NEXT: umov.b w10, v0[2]
-; CHECK-GI-NEXT: umov.b w11, v0[3]
-; CHECK-GI-NEXT: and w8, w8, #0x1
-; CHECK-GI-NEXT: bfi w9, w8, #1, #31
-; CHECK-GI-NEXT: and w8, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[4]
-; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[5]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: umov.b w10, v0[6]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: umov.b w11, v0[7]
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
-; CHECK-GI-NEXT: and w9, w10, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
-; CHECK-GI-NEXT: and w9, w11, #0x1
-; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
-; CHECK-GI-NEXT: strb w8, [sp, #15]
-; CHECK-GI-NEXT: and w0, w8, #0xff
-; CHECK-GI-NEXT: add sp, sp, #16
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: convert_to_bitmask_8xi2:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: movi.8b v1, #3
+; CHECK-NEXT: adrp x8, lCPI13_0 at PAGE
+; CHECK-NEXT: and.8b v0, v0, v1
+; CHECK-NEXT: ldr d1, [x8, lCPI13_0 at PAGEOFF]
+; CHECK-NEXT: cmeq.8b v0, v0, #0
+; CHECK-NEXT: bic.8b v0, v1, v0
+; CHECK-NEXT: addv.8b b0, v0
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
%cmp_result = icmp ne <8 x i2> %vec, zeroinitializer
%bitmask = bitcast <8 x i1> %cmp_result to i8
@@ -756,37 +610,15 @@ define i8 @convert_large_vector(<8 x i32> %vec) {
;
; CHECK-GI-LABEL: convert_large_vector:
; CHECK-GI: ; %bb.0:
-; CHECK-GI-NE...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/213946
More information about the llvm-commits
mailing list