[llvm] [AArch64][GlobalISel] Fix nonterminating legalization for <8 x s4> vectors. (PR #192747)
Amara Emerson via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 17 14:49:01 PDT 2026
https://github.com/aemerson created https://github.com/llvm/llvm-project/pull/192747
G_CONCAT_VECTORS with <16 x s4> sources hits the bitcast legalization
path, which round-trips through scalar types (e.g. s32) and regenerates
<8 x s4> vectors via G_UNMERGE_VALUES and G_BUILD_VECTOR. The
G_BUILD_VECTOR is then widened to <8 x s8> (via .minScalarOrElt(0, s8)),
producing G_ANYEXT/G_TRUNC artifact pairs. The artifact combiner folds
these pairs away, restoring the original <8 x s4> types, which feeds
back into G_CONCAT_VECTORS again.
This change:
* Adds .minScalarOrElt(1, s8) to the G_ICMP rules to ensure operand
vector elements are at least s8. This causes <16 x s4> operands to be widened
to <16 x s8>, and the result type follows via minScalarEltSameAs.
* Add custom legalization for G_CONCAT_VECTORS when element size < 8.
The custom handler widens source operands via G_ANYEXT (e.g.
<8 x s4> -> <8 x s8>), concats the widened vectors (producing a
legal type like <16 x s8>), and truncs the result back to the
original type.
The code is bad but we can fix that later, some fallbacks are fixed with
this.
rdar://153760145
>From 1456899c61d3920f01ed5df458267f53a09ed0a8 Mon Sep 17 00:00:00 2001
From: Amara Emerson <amara at apple.com>
Date: Fri, 17 Apr 2026 11:15:25 -0700
Subject: [PATCH] [AArch64][GlobalISel] Fix nonterminating legalization for <8
x s4> vectors.
G_CONCAT_VECTORS with <16 x s4> sources hits the bitcast legalization
path, which round-trips through scalar types (e.g. s32) and regenerates
<8 x s4> vectors via G_UNMERGE_VALUES and G_BUILD_VECTOR. The
G_BUILD_VECTOR is then widened to <8 x s8> (via .minScalarOrElt(0, s8)),
producing G_ANYEXT/G_TRUNC artifact pairs. The artifact combiner folds
these pairs away, restoring the original <8 x s4> types, which feeds
back into G_CONCAT_VECTORS again.
This change:
* Adds .minScalarOrElt(1, s8) to the G_ICMP rules to ensure operand
vector elements are at least s8. This causes <16 x s4> operands to be widened
to <16 x s8>, and the result type follows via minScalarEltSameAs.
* Add custom legalization for G_CONCAT_VECTORS when element size < 8.
The custom handler widens source operands via G_ANYEXT (e.g.
<8 x s4> -> <8 x s8>), concats the widened vectors (producing a
legal type like <16 x s8>), and truncs the result back to the
original type.
The code is bad but we can fix that later, some fallbacks are fixed with
this.
rdar://153760145
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 35 ++++
.../AArch64/GISel/AArch64LegalizerInfo.h | 2 +
llvm/test/CodeGen/AArch64/arm64-zip.ll | 177 +++++++++++++----
llvm/test/CodeGen/AArch64/dup.ll | 1 -
.../AArch64/vec-combine-compare-to-bitmask.ll | 186 ++++++++++++++----
5 files changed, 317 insertions(+), 84 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 5b07480677e47..5f240867ccdfe 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -698,6 +698,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
getActionDefinitionsBuilder(G_ICMP)
.legalFor({{s32, s32}, {s32, s64}, {s32, p0}})
.widenScalarOrEltToNextPow2(1)
+ .minScalarOrElt(1, s8)
.clampScalar(1, s32, s64)
.clampScalar(0, s32, s32)
.scalarizeIf(scalarOrEltWiderThan(1, 64), 1)
@@ -1261,6 +1262,10 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
getActionDefinitionsBuilder(G_CONCAT_VECTORS)
.legalFor({{v16s8, v8s8}, {v8s16, v4s16}, {v4s32, v2s32}})
+ .customIf([=](const LegalityQuery &Query) {
+ return Query.Types[0].isFixedVector() &&
+ Query.Types[0].getScalarSizeInBits() < 8;
+ })
.bitcastIf(
[=](const LegalityQuery &Query) {
return Query.Types[0].isFixedVector() &&
@@ -1501,6 +1506,8 @@ bool AArch64LegalizerInfo::legalizeCustom(
return legalizeICMP(MI, MRI, MIRBuilder);
case TargetOpcode::G_BITCAST:
return legalizeBitcast(MI, Helper);
+ case TargetOpcode::G_CONCAT_VECTORS:
+ return legalizeConcatVectors(MI, MRI, MIRBuilder);
case TargetOpcode::G_FPTRUNC:
// In order to lower f16 to f64 properly, we need to use f32 as an
// intermediary
@@ -2567,6 +2574,34 @@ bool AArch64LegalizerInfo::legalizePrefetch(MachineInstr &MI,
return true;
}
+bool AArch64LegalizerInfo::legalizeConcatVectors(
+ MachineInstr &MI, MachineRegisterInfo &MRI,
+ MachineIRBuilder &MIRBuilder) const {
+ // Widen sub-byte element vectors to byte-sized elements before concatenating.
+ // This is analogous to SDAG's integer type promotion for sub-byte types.
+ auto &Concat = cast<GConcatVectors>(MI);
+ Register DstReg = Concat.getReg(0);
+ LLT DstTy = MRI.getType(DstReg);
+ assert(DstTy.getScalarSizeInBits() < 8 && "Expected dst ty to be < 8b");
+
+ unsigned WideEltSize =
+ std::max(8u, (unsigned)PowerOf2Ceil(DstTy.getScalarSizeInBits()));
+ LLT SrcTy = MRI.getType(Concat.getSourceReg(0));
+ LLT WideSrcTy = SrcTy.changeElementSize(WideEltSize);
+ LLT WideDstTy = DstTy.changeElementSize(WideEltSize);
+
+ SmallVector<Register> WideSrcs;
+ for (unsigned I = 0; I < Concat.getNumSources(); ++I) {
+ auto Wide = MIRBuilder.buildAnyExt(WideSrcTy, Concat.getSourceReg(I));
+ WideSrcs.push_back(Wide.getReg(0));
+ }
+
+ auto WideConcat = MIRBuilder.buildConcatVectors(WideDstTy, WideSrcs);
+ MIRBuilder.buildTrunc(DstReg, WideConcat);
+ MI.eraseFromParent();
+ return true;
+}
+
bool AArch64LegalizerInfo::legalizeFptrunc(MachineInstr &MI,
MachineIRBuilder &MIRBuilder,
MachineRegisterInfo &MRI) const {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
index 12b6a6fa395a8..535240a67a2cc 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
@@ -67,6 +67,8 @@ class AArch64LegalizerInfo : public LegalizerInfo {
bool legalizeDynStackAlloc(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizePrefetch(MachineInstr &MI, LegalizerHelper &Helper) const;
bool legalizeBitcast(MachineInstr &MI, LegalizerHelper &Helper) const;
+ bool legalizeConcatVectors(MachineInstr &MI, MachineRegisterInfo &MRI,
+ MachineIRBuilder &MIRBuilder) const;
bool legalizeFptrunc(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
MachineRegisterInfo &MRI) const;
const AArch64Subtarget *ST;
diff --git a/llvm/test/CodeGen/AArch64/arm64-zip.ll b/llvm/test/CodeGen/AArch64/arm64-zip.ll
index 44411a1032dca..cb3d2569a38d5 100644
--- a/llvm/test/CodeGen/AArch64/arm64-zip.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-zip.ll
@@ -2,10 +2,6 @@
; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-; CHECK-GI: warning: Instruction selection used fallback path for shuffle_zip1
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for shuffle_zip2
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for shuffle_zip3
-
define <8 x i8> @vzipi8(ptr %A, ptr %B) nounwind {
; CHECK-LABEL: vzipi8:
; CHECK: // %bb.0:
@@ -379,18 +375,52 @@ define <16 x i8> @combine_v8i16_8firstundef(<8 x i8> %0, <8 x i8> %1) {
}
define <4 x float> @shuffle_zip1(<4 x float> %arg) {
-; CHECK-LABEL: shuffle_zip1:
-; CHECK: // %bb.0: // %bb
-; CHECK-NEXT: fcmgt.4s v0, v0, #0.0
-; CHECK-NEXT: uzp1.8h v1, v0, v0
-; CHECK-NEXT: xtn.4h v0, v0
-; CHECK-NEXT: xtn.4h v1, v1
-; CHECK-NEXT: zip2.4h v0, v0, v1
-; CHECK-NEXT: fmov.4s v1, #1.00000000
-; CHECK-NEXT: zip1.4h v0, v0, v0
-; CHECK-NEXT: sshll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: shuffle_zip1:
+; CHECK-SD: // %bb.0: // %bb
+; CHECK-SD-NEXT: fcmgt.4s v0, v0, #0.0
+; CHECK-SD-NEXT: uzp1.8h v1, v0, v0
+; CHECK-SD-NEXT: xtn.4h v0, v0
+; CHECK-SD-NEXT: xtn.4h v1, v1
+; CHECK-SD-NEXT: zip2.4h v0, v0, v1
+; CHECK-SD-NEXT: fmov.4s v1, #1.00000000
+; CHECK-SD-NEXT: zip1.4h v0, v0, v0
+; CHECK-SD-NEXT: sshll.4s v0, v0, #0
+; CHECK-SD-NEXT: and.16b v0, v0, v1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: shuffle_zip1:
+; CHECK-GI: // %bb.0: // %bb
+; CHECK-GI-NEXT: fcmgt.4s v0, v0, #0.0
+; CHECK-GI-NEXT: fmov.4s v2, #1.00000000
+; CHECK-GI-NEXT: mov.s w8, v0[1]
+; CHECK-GI-NEXT: mov.s w9, v0[2]
+; CHECK-GI-NEXT: mov.s w10, v0[3]
+; CHECK-GI-NEXT: mov.b v0[1], w8
+; CHECK-GI-NEXT: adrp x8, .LCPI27_1
+; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI27_1]
+; CHECK-GI-NEXT: adrp x8, .LCPI27_0
+; CHECK-GI-NEXT: mov.b v0[2], w9
+; CHECK-GI-NEXT: mov.b v0[3], w10
+; CHECK-GI-NEXT: mov.d v0[1], v0[0]
+; CHECK-GI-NEXT: tbl.16b v0, { v0 }, v1
+; CHECK-GI-NEXT: mov.b v1[0], v0[0]
+; CHECK-GI-NEXT: mov.b v1[1], v0[1]
+; CHECK-GI-NEXT: mov.d v1[1], v0[0]
+; CHECK-GI-NEXT: ldr d0, [x8, :lo12:.LCPI27_0]
+; CHECK-GI-NEXT: tbl.16b v0, { v1 }, v0
+; CHECK-GI-NEXT: umov.b w8, v0[0]
+; CHECK-GI-NEXT: umov.b w9, v0[1]
+; CHECK-GI-NEXT: fmov s1, w8
+; CHECK-GI-NEXT: umov.b w8, v0[2]
+; CHECK-GI-NEXT: mov.s v1[1], w9
+; CHECK-GI-NEXT: umov.b w9, v0[3]
+; CHECK-GI-NEXT: mov.s v1[2], w8
+; CHECK-GI-NEXT: mov.s v1[3], w9
+; CHECK-GI-NEXT: shl.4s v0, v1, #31
+; CHECK-GI-NEXT: movi.2d v1, #0000000000000000
+; CHECK-GI-NEXT: cmlt.4s v0, v0, #0
+; CHECK-GI-NEXT: bsl.16b v0, v2, v1
+; CHECK-GI-NEXT: ret
bb:
%inst = fcmp olt <4 x float> zeroinitializer, %arg
%inst1 = shufflevector <4 x i1> %inst, <4 x i1> zeroinitializer, <2 x i32> <i32 2, i32 0>
@@ -400,18 +430,52 @@ bb:
}
define <4 x i32> @shuffle_zip2(<4 x i32> %arg) {
-; CHECK-LABEL: shuffle_zip2:
-; CHECK: // %bb.0: // %bb
-; CHECK-NEXT: cmtst.4s v0, v0, v0
-; CHECK-NEXT: uzp1.8h v1, v0, v0
-; CHECK-NEXT: xtn.4h v0, v0
-; CHECK-NEXT: xtn.4h v1, v1
-; CHECK-NEXT: zip2.4h v0, v0, v1
-; CHECK-NEXT: movi.4s v1, #1
-; CHECK-NEXT: zip1.4h v0, v0, v0
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: shuffle_zip2:
+; CHECK-SD: // %bb.0: // %bb
+; CHECK-SD-NEXT: cmtst.4s v0, v0, v0
+; CHECK-SD-NEXT: uzp1.8h v1, v0, v0
+; CHECK-SD-NEXT: xtn.4h v0, v0
+; CHECK-SD-NEXT: xtn.4h v1, v1
+; CHECK-SD-NEXT: zip2.4h v0, v0, v1
+; CHECK-SD-NEXT: movi.4s v1, #1
+; CHECK-SD-NEXT: zip1.4h v0, v0, v0
+; CHECK-SD-NEXT: ushll.4s v0, v0, #0
+; CHECK-SD-NEXT: and.16b v0, v0, v1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: shuffle_zip2:
+; CHECK-GI: // %bb.0: // %bb
+; CHECK-GI-NEXT: movi.2d v1, #0000000000000000
+; CHECK-GI-NEXT: cmhi.4s v0, v0, v1
+; CHECK-GI-NEXT: mov.s w8, v0[1]
+; CHECK-GI-NEXT: mov.s w9, v0[2]
+; CHECK-GI-NEXT: mov.s w10, v0[3]
+; CHECK-GI-NEXT: mov.b v0[1], w8
+; CHECK-GI-NEXT: adrp x8, .LCPI28_1
+; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI28_1]
+; CHECK-GI-NEXT: adrp x8, .LCPI28_0
+; CHECK-GI-NEXT: mov.b v0[2], w9
+; CHECK-GI-NEXT: mov.b v0[3], w10
+; CHECK-GI-NEXT: mov.d v0[1], v0[0]
+; CHECK-GI-NEXT: tbl.16b v0, { v0 }, v1
+; CHECK-GI-NEXT: mov.b v1[0], v0[0]
+; CHECK-GI-NEXT: mov.b v1[1], v0[1]
+; CHECK-GI-NEXT: mov.d v1[1], v0[0]
+; CHECK-GI-NEXT: ldr d0, [x8, :lo12:.LCPI28_0]
+; CHECK-GI-NEXT: tbl.16b v0, { v1 }, v0
+; CHECK-GI-NEXT: umov.b w8, v0[0]
+; CHECK-GI-NEXT: umov.b w9, v0[1]
+; CHECK-GI-NEXT: fmov s1, w8
+; CHECK-GI-NEXT: umov.b w8, v0[2]
+; CHECK-GI-NEXT: mov.s v1[1], w9
+; CHECK-GI-NEXT: umov.b w9, v0[3]
+; CHECK-GI-NEXT: mov.s v1[2], w8
+; CHECK-GI-NEXT: mov.s v1[3], w9
+; CHECK-GI-NEXT: shl.4s v0, v1, #31
+; CHECK-GI-NEXT: movi.4s v1, #1
+; CHECK-GI-NEXT: cmlt.4s v0, v0, #0
+; CHECK-GI-NEXT: and.16b v0, v1, v0
+; CHECK-GI-NEXT: ret
bb:
%inst = icmp ult <4 x i32> zeroinitializer, %arg
%inst1 = shufflevector <4 x i1> %inst, <4 x i1> zeroinitializer, <2 x i32> <i32 2, i32 0>
@@ -421,18 +485,51 @@ bb:
}
define <4 x i32> @shuffle_zip3(<4 x i32> %arg) {
-; CHECK-LABEL: shuffle_zip3:
-; CHECK: // %bb.0: // %bb
-; CHECK-NEXT: cmgt.4s v0, v0, #0
-; CHECK-NEXT: uzp1.8h v1, v0, v0
-; CHECK-NEXT: xtn.4h v0, v0
-; CHECK-NEXT: xtn.4h v1, v1
-; CHECK-NEXT: zip2.4h v0, v0, v1
-; CHECK-NEXT: movi.4s v1, #1
-; CHECK-NEXT: zip1.4h v0, v0, v0
-; CHECK-NEXT: ushll.4s v0, v0, #0
-; CHECK-NEXT: and.16b v0, v0, v1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: shuffle_zip3:
+; CHECK-SD: // %bb.0: // %bb
+; CHECK-SD-NEXT: cmgt.4s v0, v0, #0
+; CHECK-SD-NEXT: uzp1.8h v1, v0, v0
+; CHECK-SD-NEXT: xtn.4h v0, v0
+; CHECK-SD-NEXT: xtn.4h v1, v1
+; CHECK-SD-NEXT: zip2.4h v0, v0, v1
+; CHECK-SD-NEXT: movi.4s v1, #1
+; CHECK-SD-NEXT: zip1.4h v0, v0, v0
+; CHECK-SD-NEXT: ushll.4s v0, v0, #0
+; CHECK-SD-NEXT: and.16b v0, v0, v1
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: shuffle_zip3:
+; CHECK-GI: // %bb.0: // %bb
+; CHECK-GI-NEXT: cmgt.4s v0, v0, #0
+; CHECK-GI-NEXT: mov.s w8, v0[1]
+; CHECK-GI-NEXT: mov.s w9, v0[2]
+; CHECK-GI-NEXT: mov.s w10, v0[3]
+; CHECK-GI-NEXT: mov.b v0[1], w8
+; CHECK-GI-NEXT: adrp x8, .LCPI29_1
+; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI29_1]
+; CHECK-GI-NEXT: adrp x8, .LCPI29_0
+; CHECK-GI-NEXT: mov.b v0[2], w9
+; CHECK-GI-NEXT: mov.b v0[3], w10
+; CHECK-GI-NEXT: mov.d v0[1], v0[0]
+; CHECK-GI-NEXT: tbl.16b v0, { v0 }, v1
+; CHECK-GI-NEXT: mov.b v1[0], v0[0]
+; CHECK-GI-NEXT: mov.b v1[1], v0[1]
+; CHECK-GI-NEXT: mov.d v1[1], v0[0]
+; CHECK-GI-NEXT: ldr d0, [x8, :lo12:.LCPI29_0]
+; CHECK-GI-NEXT: tbl.16b v0, { v1 }, v0
+; CHECK-GI-NEXT: umov.b w8, v0[0]
+; CHECK-GI-NEXT: umov.b w9, v0[1]
+; CHECK-GI-NEXT: fmov s1, w8
+; CHECK-GI-NEXT: umov.b w8, v0[2]
+; CHECK-GI-NEXT: mov.s v1[1], w9
+; CHECK-GI-NEXT: umov.b w9, v0[3]
+; CHECK-GI-NEXT: mov.s v1[2], w8
+; CHECK-GI-NEXT: mov.s v1[3], w9
+; CHECK-GI-NEXT: shl.4s v0, v1, #31
+; CHECK-GI-NEXT: movi.4s v1, #1
+; CHECK-GI-NEXT: cmlt.4s v0, v0, #0
+; CHECK-GI-NEXT: and.16b v0, v1, v0
+; CHECK-GI-NEXT: ret
bb:
%inst = icmp slt <4 x i32> zeroinitializer, %arg
%inst1 = shufflevector <4 x i1> %inst, <4 x i1> zeroinitializer, <2 x i32> <i32 2, i32 0>
diff --git a/llvm/test/CodeGen/AArch64/dup.ll b/llvm/test/CodeGen/AArch64/dup.ll
index 16748cf16cb19..8b8d4392461c8 100644
--- a/llvm/test/CodeGen/AArch64/dup.ll
+++ b/llvm/test/CodeGen/AArch64/dup.ll
@@ -2,7 +2,6 @@
; RUN: llc -mtriple=aarch64-none-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple=aarch64-none-none-eabi -verify-machineinstrs -global-isel -global-isel-abort=2 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-; CHECK-GI: warning: Instruction selection used fallback path for v8i4_to_v16i8
define <2 x i8> @dup_v2i8(i8 %a) {
; CHECK-SD-LABEL: dup_v2i8:
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
index af8cda4b4c1ca..a9a6570c9ebf5 100644
--- a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
@@ -3,10 +3,7 @@
; RUN: llc -mtriple=aarch64-apple-darwin -mattr=+neon -aarch64-enable-collect-loh=false -global-isel -global-isel-abort=2 -verify-machineinstrs < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
; CHECK-GI: warning: Instruction selection used fallback path for convert_to_bitmask2
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for clang_builtins_undef_concat_convert_to_bitmask4
; CHECK-GI-NEXT: warning: Instruction selection used fallback path for convert_to_bitmask_2xi32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for convert_to_bitmask_8xi2
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for no_direct_convert_for_bad_concat
; Basic tests from input vector to bitmask
; IR generated from clang for:
@@ -199,15 +196,49 @@ define i8 @convert_to_bitmask2(<2 x i64> %vec) {
; Clang's __builtin_convertvector adds an undef vector concat for vectors with <8 elements.
define i8 @clang_builtins_undef_concat_convert_to_bitmask4(<4 x i32> %vec) {
-; CHECK-LABEL: clang_builtins_undef_concat_convert_to_bitmask4:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: adrp x8, lCPI4_0 at PAGE
-; CHECK-NEXT: cmeq.4s v0, v0, #0
-; CHECK-NEXT: ldr q1, [x8, lCPI4_0 at PAGEOFF]
-; CHECK-NEXT: bic.16b v0, v1, v0
-; CHECK-NEXT: addv.4s s0, v0
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: clang_builtins_undef_concat_convert_to_bitmask4:
+; CHECK-SD: ; %bb.0:
+; CHECK-SD-NEXT: adrp x8, lCPI4_0 at PAGE
+; CHECK-SD-NEXT: cmeq.4s v0, v0, #0
+; CHECK-SD-NEXT: ldr q1, [x8, lCPI4_0 at PAGEOFF]
+; CHECK-SD-NEXT: bic.16b v0, v1, v0
+; CHECK-SD-NEXT: addv.4s s0, v0
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: clang_builtins_undef_concat_convert_to_bitmask4:
+; CHECK-GI: ; %bb.0:
+; CHECK-GI-NEXT: sub sp, sp, #16
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: cmtst.4s v0, v0, v0
+; CHECK-GI-NEXT: xtn.4h v0, v0
+; CHECK-GI-NEXT: uzp1.8b v0, v0, v0
+; CHECK-GI-NEXT: umov.b w8, v0[1]
+; CHECK-GI-NEXT: umov.b w9, v0[0]
+; CHECK-GI-NEXT: umov.b w10, v0[2]
+; CHECK-GI-NEXT: umov.b w11, v0[3]
+; CHECK-GI-NEXT: and w8, w8, #0x1
+; CHECK-GI-NEXT: bfi w9, w8, #1, #31
+; CHECK-GI-NEXT: and w8, w10, #0x1
+; CHECK-GI-NEXT: umov.b w10, v0[4]
+; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov.b w11, v0[5]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov.b w10, v0[6]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov.b w11, v0[7]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
+; CHECK-GI-NEXT: strb w8, [sp, #15]
+; CHECK-GI-NEXT: and w0, w8, #0xff
+; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: ret
%cmp_result = icmp ne <4 x i32> %vec, zeroinitializer
@@ -616,17 +647,50 @@ define i4 @convert_to_bitmask_4xi8(<4 x i8> %vec) {
}
define i8 @convert_to_bitmask_8xi2(<8 x i2> %vec) {
-; CHECK-LABEL: convert_to_bitmask_8xi2:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: movi.8b v1, #3
-; CHECK-NEXT: adrp x8, lCPI13_0 at PAGE
-; CHECK-NEXT: and.8b v0, v0, v1
-; CHECK-NEXT: ldr d1, [x8, lCPI13_0 at PAGEOFF]
-; CHECK-NEXT: cmeq.8b v0, v0, #0
-; CHECK-NEXT: bic.8b v0, v1, v0
-; CHECK-NEXT: addv.8b b0, v0
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: convert_to_bitmask_8xi2:
+; CHECK-SD: ; %bb.0:
+; CHECK-SD-NEXT: movi.8b v1, #3
+; CHECK-SD-NEXT: adrp x8, lCPI13_0 at PAGE
+; CHECK-SD-NEXT: and.8b v0, v0, v1
+; CHECK-SD-NEXT: ldr d1, [x8, lCPI13_0 at PAGEOFF]
+; CHECK-SD-NEXT: cmeq.8b v0, v0, #0
+; CHECK-SD-NEXT: bic.8b v0, v1, v0
+; CHECK-SD-NEXT: addv.8b b0, v0
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: convert_to_bitmask_8xi2:
+; CHECK-GI: ; %bb.0:
+; CHECK-GI-NEXT: sub sp, sp, #16
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: movi.8b v1, #3
+; CHECK-GI-NEXT: cmtst.8b v0, v0, v1
+; CHECK-GI-NEXT: umov.b w8, v0[1]
+; CHECK-GI-NEXT: umov.b w9, v0[0]
+; CHECK-GI-NEXT: umov.b w10, v0[2]
+; CHECK-GI-NEXT: umov.b w11, v0[3]
+; CHECK-GI-NEXT: and w8, w8, #0x1
+; CHECK-GI-NEXT: bfi w9, w8, #1, #31
+; CHECK-GI-NEXT: and w8, w10, #0x1
+; CHECK-GI-NEXT: umov.b w10, v0[4]
+; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov.b w11, v0[5]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov.b w10, v0[6]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov.b w11, v0[7]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
+; CHECK-GI-NEXT: strb w8, [sp, #15]
+; CHECK-GI-NEXT: and w0, w8, #0xff
+; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: ret
%cmp_result = icmp ne <8 x i2> %vec, zeroinitializer
%bitmask = bitcast <8 x i1> %cmp_result to i8
@@ -775,26 +839,62 @@ define i4 @convert_legalized_illegal_element_size(<4 x i22> %vec) {
; This may still be converted as a v8i8 after the vector concat (but not as v4iX).
define i8 @no_direct_convert_for_bad_concat(<4 x i32> %vec) {
-; CHECK-LABEL: no_direct_convert_for_bad_concat:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: cmtst.4s v0, v0, v0
-; CHECK-NEXT: adrp x8, lCPI17_0 at PAGE
-; CHECK-NEXT: xtn.4h v0, v0
-; CHECK-NEXT: umov.h w9, v0[0]
-; CHECK-NEXT: mov.b v1[4], w9
-; CHECK-NEXT: umov.h w9, v0[1]
-; CHECK-NEXT: mov.b v1[5], w9
-; CHECK-NEXT: umov.h w9, v0[2]
-; CHECK-NEXT: mov.b v1[6], w9
-; CHECK-NEXT: umov.h w9, v0[3]
-; CHECK-NEXT: mov.b v1[7], w9
-; CHECK-NEXT: shl.8b v0, v1, #7
-; CHECK-NEXT: ldr d1, [x8, lCPI17_0 at PAGEOFF]
-; CHECK-NEXT: cmlt.8b v0, v0, #0
-; CHECK-NEXT: and.8b v0, v0, v1
-; CHECK-NEXT: addv.8b b0, v0
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: no_direct_convert_for_bad_concat:
+; CHECK-SD: ; %bb.0:
+; CHECK-SD-NEXT: cmtst.4s v0, v0, v0
+; CHECK-SD-NEXT: adrp x8, lCPI17_0 at PAGE
+; CHECK-SD-NEXT: xtn.4h v0, v0
+; CHECK-SD-NEXT: umov.h w9, v0[0]
+; CHECK-SD-NEXT: mov.b v1[4], w9
+; CHECK-SD-NEXT: umov.h w9, v0[1]
+; CHECK-SD-NEXT: mov.b v1[5], w9
+; CHECK-SD-NEXT: umov.h w9, v0[2]
+; CHECK-SD-NEXT: mov.b v1[6], w9
+; CHECK-SD-NEXT: umov.h w9, v0[3]
+; CHECK-SD-NEXT: mov.b v1[7], w9
+; CHECK-SD-NEXT: shl.8b v0, v1, #7
+; CHECK-SD-NEXT: ldr d1, [x8, lCPI17_0 at PAGEOFF]
+; CHECK-SD-NEXT: cmlt.8b v0, v0, #0
+; CHECK-SD-NEXT: and.8b v0, v0, v1
+; CHECK-SD-NEXT: addv.8b b0, v0
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: no_direct_convert_for_bad_concat:
+; CHECK-GI: ; %bb.0:
+; CHECK-GI-NEXT: sub sp, sp, #16
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT: cmtst.4s v0, v0, v0
+; CHECK-GI-NEXT: xtn.4h v0, v0
+; CHECK-GI-NEXT: uzp1.8b v0, v0, v0
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: mov.s v0[1], w8
+; CHECK-GI-NEXT: umov.b w8, v0[1]
+; CHECK-GI-NEXT: umov.b w9, v0[0]
+; CHECK-GI-NEXT: umov.b w10, v0[2]
+; CHECK-GI-NEXT: umov.b w11, v0[3]
+; CHECK-GI-NEXT: and w8, w8, #0x1
+; CHECK-GI-NEXT: bfi w9, w8, #1, #31
+; CHECK-GI-NEXT: and w8, w10, #0x1
+; CHECK-GI-NEXT: umov.b w10, v0[4]
+; CHECK-GI-NEXT: orr w8, w9, w8, lsl #2
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov.b w11, v0[5]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #3
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: umov.b w10, v0[6]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #4
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: umov.b w11, v0[7]
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #5
+; CHECK-GI-NEXT: and w9, w10, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #6
+; CHECK-GI-NEXT: and w9, w11, #0x1
+; CHECK-GI-NEXT: orr w8, w8, w9, lsl #7
+; CHECK-GI-NEXT: strb w8, [sp, #15]
+; CHECK-GI-NEXT: and w0, w8, #0xff
+; CHECK-GI-NEXT: add sp, sp, #16
+; CHECK-GI-NEXT: ret
%cmp_result = icmp ne <4 x i32> %vec, zeroinitializer
%vector_pad = shufflevector <4 x i1> poison, <4 x i1> %cmp_result, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 4, i32 5, i32 6, i32 7>
More information about the llvm-commits
mailing list