[llvm] [GlobalISel][AArch64] Lower G_BITCAST vectors with non-integer element ratio (PR #215154)
Kacper Doga via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 03:23:46 PDT 2026
https://github.com/varev-dev updated https://github.com/llvm/llvm-project/pull/215154
>From 23b01d476647e55d9e3becb031d48a4242d60418 Mon Sep 17 00:00:00 2001
From: Kacper Doga <doga.kacper at gmail.com>
Date: Sun, 9 Aug 2026 22:52:31 +0200
Subject: [PATCH 1/3] [GlobalISel][AArch64] Lower G_BITCAST vectors with
non-integer element ratio
`lowerBitcast` only handled vector-to-vector casts where one element
count divides the other. When neither does, the cast fell through to
code that aassumed divisibility.
Split such cast through the largest element type dividing both sides, so
that both halves have a divisibile element count and take the existing
path:
```
%1:_(<2 x i24>) = G_BITCAST %0:_(<3 x i16>)
=>
%2:_(<6 x i8>) = G_BITCAST %0:_(<3 x i16>)
%1:_(<2 x i24>) = G_BITCAST %2:_(<6 x i8>)
```
When that type is not a whole number of bytes go through a scalar of the
full width instead. Otherwise both halves end up unmerging to scalars
that are not byte multiples, and `widenScalarUnmergeValues` builds its
unmerge/remerge ladder in a sub-byte GCD type that the artifact combiner
inverts exactly, so the legalization never reaches a fixed point.
---
Two AArch64 rules are needed to reach the lowering:
- `G_BITCAST` - moreElementsToNextPow2(0) matched first for an odd
destination element count and moreElementsVector then gave up because
the source cannot be padded to match (`SrcNumElts * MoreElts % DstNumElts
!= 0`). Lower those ahead of it.
- `G_{MERGE,UNMERGE}_VALUES` - `i24 = G_MERGE_VALUES i8, i8, i8` had no
legal action. Its little type is already legal, so the rules ask to
widen the big type and widenScalarMergeValues only supports widening the
little one. Lower non-pow2 big types below s64. **Note:** The bound
matters because above i64 the lowered shifts are narrowed back into a
merge of the same type, and legalization does not terminate.
---
Tests `legalize-s{add,sub}sat.mir` are regenerated. Narrowing an 88-bit
`G_SADDO` reassembles its result through `i24 = G_MERGE_VALUES i8, i8,
i8` - those previously stayed unlegalized and were folded away by the
artifact combiner against the neighbouring unmerges. They are now
lowered to shifts before the combiner sees them. The final assembly is
unchanged.
Fixes #199333
---
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 42 +++++-
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 15 ++
.../AArch64/GlobalISel/legalize-bitcast.mir | 134 +++++++++++++++++-
.../AArch64/GlobalISel/legalize-saddsat.mir | 54 +++++--
.../AArch64/GlobalISel/legalize-ssubsat.mir | 54 +++++--
5 files changed, 280 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index a6c5a267c87db..608a75b38f3c2 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -3743,10 +3743,48 @@ LegalizerHelper::lowerBitcast(MachineInstr &MI) {
SmallVector<Register, 8> SrcRegs;
if (DstTy.isVector()) {
+ LLT DstEltTy = DstTy.getElementType();
int NumDstElt = DstTy.getNumElements();
int NumSrcElt = SrcTy.getNumElements();
- LLT DstEltTy = DstTy.getElementType();
+ if (NumSrcElt % NumDstElt != 0 && NumDstElt % NumSrcElt != 0) {
+ // Split non-integer element ratio bitcast
+ //
+ // %1:_(<3 x s16>) = G_BITCAST %0:_(<2 x s24>)
+ //
+ // =>
+ //
+ // %2:_(<6 x s8>) = G_BITCAST %0:_(<2 x s24>)
+ // %1:_(<3 x s16>) = G_BITCAST %2:_(<6 x s8>)
+ int SrcEltSize = SrcEltTy.getSizeInBits();
+ int DstEltSize = DstEltTy.getSizeInBits();
+ int PieceSize = std::gcd(SrcEltSize, DstEltSize);
+
+ if (PieceSize % 8 != 0) {
+ // Split bitcast whose pieces are not whole bytes through a scalar
+ //
+ // %1:_(<3 x s8>) = G_BITCAST %0:_(<2 x s12>)
+ //
+ // =>
+ //
+ // %2:_(s24) = G_BITCAST %0:_(<2 x s12>)
+ // %1:_(<3 x s8>) = G_BITCAST %2:_(s24)
+ LLT ScalarTy = LLT::integer(SrcTy.getSizeInBits());
+ Register ScalarReg = MIRBuilder.buildBitcast(ScalarTy, Src).getReg(0);
+ MIRBuilder.buildBitcast(Dst, ScalarReg);
+ MI.eraseFromParent();
+ return Legalized;
+ }
+ LLT PieceTy = LLT::integer(PieceSize);
+ int NumPieces = SrcTy.getSizeInBits() / PieceSize;
+ LLT PiecesVecTy = LLT::fixed_vector(NumPieces, PieceTy);
+ Register PiecesReg =
+ MIRBuilder.buildBitcast(PiecesVecTy, Src).getReg(0);
+ MIRBuilder.buildBitcast(Dst, PiecesReg);
+ MI.eraseFromParent();
+ return Legalized;
+ }
+
LLT DstCastTy = DstEltTy; // Intermediate bitcast result type
LLT SrcPartTy = SrcEltTy; // Original unmerge result type.
@@ -3760,7 +3798,7 @@ LegalizerHelper::lowerBitcast(MachineInstr &MI) {
// %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0
// %3:_(<2 x s8>) = G_BITCAST %2
// %4:_(<2 x s8>) = G_BITCAST %3
- // %1:_(<4 x s16>) = G_CONCAT_VECTORS %3, %4
+ // %1:_(<4 x s8>) = G_CONCAT_VECTORS %3, %4
DstCastTy = DstTy.changeVectorElementCount(
ElementCount::getFixed(NumDstElt / NumSrcElt));
SrcPartTy = SrcEltTy;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 2263712120de8..cc2bce010c4de 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1123,6 +1123,15 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
.lowerIf([=](const LegalityQuery &Query) {
return Query.Types[0].isVector() != Query.Types[1].isVector();
})
+ // moreElementsToNextPow2 cannot pad the source to match, so lower
+ .lowerIf([=](const LegalityQuery &Query) {
+ LLT DstTy = Query.Types[0];
+ LLT SrcTy = Query.Types[1];
+ if (!DstTy.isFixedVector() || !SrcTy.isFixedVector())
+ return false;
+ unsigned MoreElts = 1u << Log2_32_Ceil(DstTy.getNumElements());
+ return SrcTy.getNumElements() * MoreElts % DstTy.getNumElements() != 0;
+ })
.moreElementsToNextPow2(0)
.clampNumElements(0, v8s8, v16s8)
.clampNumElements(0, v4s16, v8s16)
@@ -1175,6 +1184,12 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
getActionDefinitionsBuilder(Op)
.widenScalarToNextPow2(LitTyIdx, 8)
+ // Above s64 lowered shifts narrow back to a merge and never terminate
+ .lowerIf([=](const LegalityQuery &Q) {
+ const LLT BigTy = Q.Types[BigTyIdx];
+ return BigTy.isScalar() && !isPowerOf2_32(BigTy.getSizeInBits()) &&
+ BigTy.getSizeInBits() < 64;
+ })
.widenScalarToNextPow2(BigTyIdx, 32)
.clampScalar(LitTyIdx, s8, s64)
.clampScalar(BigTyIdx, s32, s128)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
index b418d9e224b46..7a6726d4ad679 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
@@ -46,7 +46,6 @@ body: |
%3:_(p0) = G_CONSTANT i64 0
G_STORE %1(<16 x i32>), %3(p0) :: (store (<16 x i32>))
G_BR %bb.2
-
...
---
name: boolean_vector_to_scalar
@@ -118,3 +117,136 @@ body: |
$w0 = COPY %ext(i32)
RET_ReallyLR implicit $w0
...
+---
+name: v3i16_to_v2i24
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $d0
+ ; CHECK-LABEL: name: v3i16_to_v2i24
+ ; CHECK: liveins: $d0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %vec:_(<4 x i16>) = COPY $d0
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<8 x i8>) = G_BITCAST %vec(<4 x i16>)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i8), [[UV1:%[0-9]+]]:_(i8), [[UV2:%[0-9]+]]:_(i8), [[UV3:%[0-9]+]]:_(i8), [[UV4:%[0-9]+]]:_(i8), [[UV5:%[0-9]+]]:_(i8), [[UV6:%[0-9]+]]:_(i8), [[UV7:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[BITCAST]](<8 x i8>)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UV1]](i8)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT]], [[C]](i64)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[UV]](i8)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT1]], [[SHL]]
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[UV2]](i8)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT2]], [[C1]](i64)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[UV4]](i8)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i64)
+ ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[UV3]](i8)
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL2]]
+ ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[UV5]](i8)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C1]](i64)
+ ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[OR2]], [[SHL3]]
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[OR1]](i32), [[OR3]](i32)
+ ; CHECK-NEXT: $d0 = COPY [[BUILD_VECTOR]](<2 x i32>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $d0
+ %vec:_(<4 x i16>) = COPY $d0
+ %e0:_(i16), %e1:_(i16), %e2:_(i16), %e3:_(i16) = G_UNMERGE_VALUES %vec(<4 x i16>)
+ %src:_(<3 x i16>) = G_BUILD_VECTOR %e0(i16), %e1(i16), %e2(i16)
+ %bc:_(<2 x i24>) = G_BITCAST %src(<3 x i16>)
+ %ext:_(<2 x i32>) = G_ANYEXT %bc(<2 x i24>)
+ $d0 = COPY %ext(<2 x i32>)
+ RET_ReallyLR implicit $d0
+...
+---
+name: v2i24_to_v3i16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $d0
+ ; CHECK-LABEL: name: v2i24_to_v3i16
+ ; CHECK: liveins: $d0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %vec:_(<2 x i32>) = COPY $d0
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES %vec(<2 x i32>)
+ ; CHECK-NEXT: [[UV2:%[0-9]+]]:_(i8), [[UV3:%[0-9]+]]:_(i8), [[UV4:%[0-9]+]]:_(i8), [[UV5:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[UV]](i32)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:_(i8), [[UV7:%[0-9]+]]:_(i8), [[UV8:%[0-9]+]]:_(i8), [[UV9:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[UV1]](i32)
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(i8) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i8>) = G_BUILD_VECTOR [[UV2]](i8), [[UV3]](i8), [[UV4]](i8), [[UV6]](i8), [[UV7]](i8), [[UV8]](i8), [[DEF]](i8), [[DEF]](i8)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<4 x i16>) = G_BITCAST [[BUILD_VECTOR]](<8 x i8>)
+ ; CHECK-NEXT: $d0 = COPY [[BITCAST]](<4 x i16>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $d0
+ %vec:_(<2 x i32>) = COPY $d0
+ %src:_(<2 x i24>) = G_TRUNC %vec(<2 x i32>)
+ %bc:_(<3 x i16>) = G_BITCAST %src(<2 x i24>)
+ %e0:_(i16), %e1:_(i16), %e2:_(i16) = G_UNMERGE_VALUES %bc(<3 x i16>)
+ %undef:_(i16) = G_IMPLICIT_DEF
+ %out:_(<4 x i16>) = G_BUILD_VECTOR %e0(i16), %e1(i16), %e2(i16), %undef(i16)
+ $d0 = COPY %out(<4 x i16>)
+ RET_ReallyLR implicit $d0
+...
+---
+name: v2i12_to_v3i8
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0
+ ; CHECK-LABEL: name: v2i12_to_v3i8
+ ; CHECK: liveins: $w0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w:_(i32) = COPY $w0
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i8), [[UV1:%[0-9]+]]:_(i8), [[UV2:%[0-9]+]]:_(i8), [[UV3:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES %w(i32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UV1]](i8)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT]], [[C]](i64)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[UV]](i8)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT1]], [[SHL]]
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[UV2]](i8)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT2]], [[C1]](i64)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; CHECK-NEXT: $w0 = COPY [[OR1]](i32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w:_(i32) = COPY $w0
+ %t:_(i24) = G_TRUNC %w(i32)
+ %src:_(<2 x i12>) = G_BITCAST %t(i24)
+ %bc:_(<3 x i8>) = G_BITCAST %src(<2 x i12>)
+ %r:_(i24) = G_BITCAST %bc(<3 x i8>)
+ %e:_(i32) = G_ANYEXT %r(i24)
+ $w0 = COPY %e(i32)
+ RET_ReallyLR implicit $w0
+...
+---
+name: v4i3_to_v3i4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0
+ ; CHECK-LABEL: name: v4i3_to_v3i4
+ ; CHECK: liveins: $w0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %w:_(i32) = COPY $w0
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i8), [[UV1:%[0-9]+]]:_(i8), [[UV2:%[0-9]+]]:_(i8), [[UV3:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES %w(i32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UV]](i8)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[ZEXT]], [[C]](i64)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 15
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C]](i64)
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV]](i8)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ANYEXT]], [[C1]]
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL]]
+ ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV1]](i8)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ANYEXT1]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C2]](i64)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; CHECK-NEXT: $w0 = COPY [[OR1]](i32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %w:_(i32) = COPY $w0
+ %t:_(i12) = G_TRUNC %w(i32)
+ %src:_(<4 x i3>) = G_BITCAST %t(i12)
+ %bc:_(<3 x i4>) = G_BITCAST %src(<4 x i3>)
+ %r:_(i12) = G_BITCAST %bc(<3 x i4>)
+ %e:_(i32) = G_ANYEXT %r(i12)
+ $w0 = COPY %e(i32)
+ RET_ReallyLR implicit $w0
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-saddsat.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-saddsat.mir
index c31ce19084b0d..80677bf03797b 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-saddsat.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-saddsat.mir
@@ -218,16 +218,54 @@ body: |
; CHECK-NEXT: [[UADDE:%[0-9]+]]:_(i32), [[UADDE1:%[0-9]+]]:_(i32) = G_UADDE [[SEXT_INREG]], [[SEXT_INREG1]], [[UADDO1]]
; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(i32) = G_SEXT_INREG [[UADDE]], 24
; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i32) = G_ICMP intpred(ne), [[UADDE]](i32), [[SEXT_INREG2]]
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:_(i8), [[UV5:%[0-9]+]]:_(i8), [[UV6:%[0-9]+]]:_(i8), [[UV7:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[UADDE]](i32)
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:_(i8), [[UV5:%[0-9]+]]:_(i8), [[UV6:%[0-9]+]]:_(i8), [[UV7:%[0-9]+]]:_(i8), [[UV8:%[0-9]+]]:_(i8), [[UV9:%[0-9]+]]:_(i8), [[UV10:%[0-9]+]]:_(i8), [[UV11:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[UADDO]](i64)
+ ; CHECK-NEXT: [[UV12:%[0-9]+]]:_(i8), [[UV13:%[0-9]+]]:_(i8), [[UV14:%[0-9]+]]:_(i8), [[UV15:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[UADDE]](i32)
; CHECK-NEXT: [[DEF:%[0-9]+]]:_(i8) = G_IMPLICIT_DEF
- ; CHECK-NEXT: [[MV:%[0-9]+]]:_(i32) = G_MERGE_VALUES [[UV4]](i8), [[UV5]](i8), [[UV6]](i8), [[DEF]](i8)
- ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(i32) = G_MERGE_VALUES [[DEF]](i8), [[DEF]](i8), [[DEF]](i8), [[DEF]](i8)
- ; CHECK-NEXT: [[MV2:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[MV]](i32), [[MV1]](i32)
- ; CHECK-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(i64) = G_SEXT_INREG [[MV2]], 24
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 23
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(i64) = G_ASHR [[SEXT_INREG3]], [[C1]](i64)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UV5]](i8)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT]], [[C1]](i64)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[UV4]](i8)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT1]], [[SHL]]
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[UV6]](i8)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT2]], [[C2]](i64)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[UV8]](i8)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C1]](i64)
+ ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[UV7]](i8)
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL2]]
+ ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[UV9]](i8)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C2]](i64)
+ ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[OR2]], [[SHL3]]
+ ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(i32) = G_ZEXT [[UV11]](i8)
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT6]], [[C1]](i64)
+ ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(i32) = G_ZEXT [[UV10]](i8)
+ ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT7]], [[SHL4]]
+ ; CHECK-NEXT: [[ZEXT8:%[0-9]+]]:_(i32) = G_ZEXT [[UV12]](i8)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT8]], [[C2]](i64)
+ ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[OR4]], [[SHL5]]
+ ; CHECK-NEXT: [[ZEXT9:%[0-9]+]]:_(i32) = G_ZEXT [[UV14]](i8)
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ZEXT9]], [[C1]](i64)
+ ; CHECK-NEXT: [[ZEXT10:%[0-9]+]]:_(i32) = G_ZEXT [[UV13]](i8)
+ ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[ZEXT10]], [[SHL6]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C2]](i64)
+ ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[OR6]], [[SHL7]]
+ ; CHECK-NEXT: [[UV16:%[0-9]+]]:_(i8), [[UV17:%[0-9]+]]:_(i8), [[UV18:%[0-9]+]]:_(i8), [[UV19:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[OR1]](i32)
+ ; CHECK-NEXT: [[UV20:%[0-9]+]]:_(i8), [[UV21:%[0-9]+]]:_(i8), [[UV22:%[0-9]+]]:_(i8), [[UV23:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[OR3]](i32)
+ ; CHECK-NEXT: [[UV24:%[0-9]+]]:_(i8), [[UV25:%[0-9]+]]:_(i8), [[UV26:%[0-9]+]]:_(i8), [[UV27:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[OR5]](i32)
+ ; CHECK-NEXT: [[UV28:%[0-9]+]]:_(i8), [[UV29:%[0-9]+]]:_(i8), [[UV30:%[0-9]+]]:_(i8), [[UV31:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[OR7]](i32)
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(i32) = G_MERGE_VALUES [[UV16]](i8), [[UV17]](i8), [[UV18]](i8), [[UV20]](i8)
+ ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(i32) = G_MERGE_VALUES [[UV21]](i8), [[UV22]](i8), [[UV24]](i8), [[UV25]](i8)
+ ; CHECK-NEXT: [[MV2:%[0-9]+]]:_(i32) = G_MERGE_VALUES [[UV26]](i8), [[UV28]](i8), [[UV29]](i8), [[UV30]](i8)
+ ; CHECK-NEXT: [[MV3:%[0-9]+]]:_(i32) = G_MERGE_VALUES [[DEF]](i8), [[DEF]](i8), [[DEF]](i8), [[DEF]](i8)
+ ; CHECK-NEXT: [[MV4:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[MV2]](i32), [[MV3]](i32)
+ ; CHECK-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(i64) = G_SEXT_INREG [[MV4]], 24
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 23
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(i64) = G_ASHR [[SEXT_INREG3]], [[C4]](i64)
; CHECK-NEXT: [[UADDO2:%[0-9]+]]:_(i64), [[UADDO3:%[0-9]+]]:_(i32) = G_UADDO [[ASHR]], [[C]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i64) = G_SELECT [[ICMP]](i32), [[UADDO2]], [[UADDO]]
+ ; CHECK-NEXT: [[MV5:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[MV]](i32), [[MV1]](i32)
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i64) = G_SELECT [[ICMP]](i32), [[UADDO2]], [[MV5]]
; CHECK-NEXT: $x0 = COPY [[SELECT]](i64)
; CHECK-NEXT: RET_ReallyLR implicit $x0
%copy_1:_(i128) = COPY $q0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ssubsat.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ssubsat.mir
index 4799956694d81..0ae9272daaf26 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ssubsat.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ssubsat.mir
@@ -218,16 +218,54 @@ body: |
; CHECK-NEXT: [[USUBE:%[0-9]+]]:_(i32), [[USUBE1:%[0-9]+]]:_(i32) = G_USUBE [[SEXT_INREG]], [[SEXT_INREG1]], [[USUBO1]]
; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(i32) = G_SEXT_INREG [[USUBE]], 24
; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i32) = G_ICMP intpred(ne), [[USUBE]](i32), [[SEXT_INREG2]]
- ; CHECK-NEXT: [[UV4:%[0-9]+]]:_(i8), [[UV5:%[0-9]+]]:_(i8), [[UV6:%[0-9]+]]:_(i8), [[UV7:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[USUBE]](i32)
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:_(i8), [[UV5:%[0-9]+]]:_(i8), [[UV6:%[0-9]+]]:_(i8), [[UV7:%[0-9]+]]:_(i8), [[UV8:%[0-9]+]]:_(i8), [[UV9:%[0-9]+]]:_(i8), [[UV10:%[0-9]+]]:_(i8), [[UV11:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[USUBO]](i64)
+ ; CHECK-NEXT: [[UV12:%[0-9]+]]:_(i8), [[UV13:%[0-9]+]]:_(i8), [[UV14:%[0-9]+]]:_(i8), [[UV15:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[USUBE]](i32)
; CHECK-NEXT: [[DEF:%[0-9]+]]:_(i8) = G_IMPLICIT_DEF
- ; CHECK-NEXT: [[MV:%[0-9]+]]:_(i32) = G_MERGE_VALUES [[UV4]](i8), [[UV5]](i8), [[UV6]](i8), [[DEF]](i8)
- ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(i32) = G_MERGE_VALUES [[DEF]](i8), [[DEF]](i8), [[DEF]](i8), [[DEF]](i8)
- ; CHECK-NEXT: [[MV2:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[MV]](i32), [[MV1]](i32)
- ; CHECK-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(i64) = G_SEXT_INREG [[MV2]], 24
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 23
- ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(i64) = G_ASHR [[SEXT_INREG3]], [[C1]](i64)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UV5]](i8)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT]], [[C1]](i64)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[UV4]](i8)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT1]], [[SHL]]
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[UV6]](i8)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT2]], [[C2]](i64)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[UV8]](i8)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C1]](i64)
+ ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[UV7]](i8)
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL2]]
+ ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[UV9]](i8)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C2]](i64)
+ ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[OR2]], [[SHL3]]
+ ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(i32) = G_ZEXT [[UV11]](i8)
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT6]], [[C1]](i64)
+ ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(i32) = G_ZEXT [[UV10]](i8)
+ ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT7]], [[SHL4]]
+ ; CHECK-NEXT: [[ZEXT8:%[0-9]+]]:_(i32) = G_ZEXT [[UV12]](i8)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT8]], [[C2]](i64)
+ ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[OR4]], [[SHL5]]
+ ; CHECK-NEXT: [[ZEXT9:%[0-9]+]]:_(i32) = G_ZEXT [[UV14]](i8)
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ZEXT9]], [[C1]](i64)
+ ; CHECK-NEXT: [[ZEXT10:%[0-9]+]]:_(i32) = G_ZEXT [[UV13]](i8)
+ ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[ZEXT10]], [[SHL6]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C2]](i64)
+ ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[OR6]], [[SHL7]]
+ ; CHECK-NEXT: [[UV16:%[0-9]+]]:_(i8), [[UV17:%[0-9]+]]:_(i8), [[UV18:%[0-9]+]]:_(i8), [[UV19:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[OR1]](i32)
+ ; CHECK-NEXT: [[UV20:%[0-9]+]]:_(i8), [[UV21:%[0-9]+]]:_(i8), [[UV22:%[0-9]+]]:_(i8), [[UV23:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[OR3]](i32)
+ ; CHECK-NEXT: [[UV24:%[0-9]+]]:_(i8), [[UV25:%[0-9]+]]:_(i8), [[UV26:%[0-9]+]]:_(i8), [[UV27:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[OR5]](i32)
+ ; CHECK-NEXT: [[UV28:%[0-9]+]]:_(i8), [[UV29:%[0-9]+]]:_(i8), [[UV30:%[0-9]+]]:_(i8), [[UV31:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[OR7]](i32)
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(i32) = G_MERGE_VALUES [[UV16]](i8), [[UV17]](i8), [[UV18]](i8), [[UV20]](i8)
+ ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(i32) = G_MERGE_VALUES [[UV21]](i8), [[UV22]](i8), [[UV24]](i8), [[UV25]](i8)
+ ; CHECK-NEXT: [[MV2:%[0-9]+]]:_(i32) = G_MERGE_VALUES [[UV26]](i8), [[UV28]](i8), [[UV29]](i8), [[UV30]](i8)
+ ; CHECK-NEXT: [[MV3:%[0-9]+]]:_(i32) = G_MERGE_VALUES [[DEF]](i8), [[DEF]](i8), [[DEF]](i8), [[DEF]](i8)
+ ; CHECK-NEXT: [[MV4:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[MV2]](i32), [[MV3]](i32)
+ ; CHECK-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(i64) = G_SEXT_INREG [[MV4]], 24
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 23
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(i64) = G_ASHR [[SEXT_INREG3]], [[C4]](i64)
; CHECK-NEXT: [[UADDO:%[0-9]+]]:_(i64), [[UADDO1:%[0-9]+]]:_(i32) = G_UADDO [[ASHR]], [[C]]
- ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i64) = G_SELECT [[ICMP]](i32), [[UADDO]], [[USUBO]]
+ ; CHECK-NEXT: [[MV5:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[MV]](i32), [[MV1]](i32)
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i64) = G_SELECT [[ICMP]](i32), [[UADDO]], [[MV5]]
; CHECK-NEXT: $x0 = COPY [[SELECT]](i64)
; CHECK-NEXT: RET_ReallyLR implicit $x0
%copy_1:_(i128) = COPY $q0
>From 1aa135a597811154df028f44eeb7b9de63581532 Mon Sep 17 00:00:00 2001
From: Kacper Doga <doga.kacper at gmail.com>
Date: Mon, 10 Aug 2026 01:09:32 +0200
Subject: [PATCH 2/3] drop non-pow2 lowering condition; move legality check to
the top
- dropped non-pow2 makes the output shorter for cases like v4i3 to v3i4
with no i8-piece unmerge left behind.
- moving the legality prevents checking size != 32 in lowering condition.
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 22 +++++++++----------
.../AArch64/GlobalISel/legalize-bitcast.mir | 19 +++++++---------
.../GlobalISel/legalize-merge-values.mir | 11 ++++++----
.../GlobalISel/legalize-unmerge-values.mir | 6 ++---
4 files changed, 28 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index cc2bce010c4de..27b19cee953d8 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1183,16 +1183,6 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
getActionDefinitionsBuilder(Op)
- .widenScalarToNextPow2(LitTyIdx, 8)
- // Above s64 lowered shifts narrow back to a merge and never terminate
- .lowerIf([=](const LegalityQuery &Q) {
- const LLT BigTy = Q.Types[BigTyIdx];
- return BigTy.isScalar() && !isPowerOf2_32(BigTy.getSizeInBits()) &&
- BigTy.getSizeInBits() < 64;
- })
- .widenScalarToNextPow2(BigTyIdx, 32)
- .clampScalar(LitTyIdx, s8, s64)
- .clampScalar(BigTyIdx, s32, s128)
.legalIf([=](const LegalityQuery &Q) {
switch (Q.Types[BigTyIdx].getSizeInBits()) {
case 32:
@@ -1211,7 +1201,17 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
default:
return false;
}
- });
+ })
+ .widenScalarToNextPow2(LitTyIdx, 8)
+ // below 64-bit build with shifts, above it the lowered shifts narrow
+ // back to a merge and never terminate
+ .lowerIf([=](const LegalityQuery &Q) {
+ const LLT BigTy = Q.Types[BigTyIdx];
+ return BigTy.isScalar() && BigTy.getSizeInBits() < 64;
+ })
+ .widenScalarToNextPow2(BigTyIdx, 32)
+ .clampScalar(LitTyIdx, s8, s64)
+ .clampScalar(BigTyIdx, s32, s128);
}
// TODO : nxv4s16, nxv2s16, nxv2s32
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
index 7a6726d4ad679..dbf4778c687f4 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
@@ -224,20 +224,17 @@ body: |
; CHECK: liveins: $w0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %w:_(i32) = COPY $w0
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i8), [[UV1:%[0-9]+]]:_(i8), [[UV2:%[0-9]+]]:_(i8), [[UV3:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES %w(i32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UV]](i8)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[ZEXT]], [[C]](i64)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 15
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR %w, [[C]](i64)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR %w, [[C1]](i64)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 15
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C]](i64)
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV]](i8)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ANYEXT]], [[C1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND %w, [[C2]]
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL]]
- ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV1]](i8)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ANYEXT1]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C2]](i64)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C1]](i64)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
; CHECK-NEXT: $w0 = COPY [[OR1]](i32)
; CHECK-NEXT: RET_ReallyLR implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir
index c661df0fc1d1b..f30c3606a2443 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir
@@ -29,10 +29,13 @@ body: |
; CHECK-LABEL: name: test_merge_s16_s8
; CHECK: %a:_(i32) = COPY $w0
; CHECK-NEXT: %b:_(i32) = COPY $w1
- ; CHECK-NEXT: %a_t:_(i8) = G_TRUNC %a(i32)
- ; CHECK-NEXT: %b_t:_(i8) = G_TRUNC %b(i32)
- ; CHECK-NEXT: %m:_(i16) = G_MERGE_VALUES %a_t(i8), %b_t(i8)
- ; CHECK-NEXT: %ext:_(i64) = G_ANYEXT %m(i16)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND %b, [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C1]](i64)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND %a, [[C]]
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL]]
+ ; CHECK-NEXT: %ext:_(i64) = G_ANYEXT [[OR]](i32)
; CHECK-NEXT: $x0 = COPY %ext(i64)
%a:_(i32) = COPY $w0
%b:_(i32) = COPY $w1
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir
index a4a6a13fb1f1c..c25c173a05faa 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir
@@ -32,11 +32,9 @@ body: |
; CHECK: liveins: $w0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $w0
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i8), [[UV1:%[0-9]+]]:_(i8), [[UV2:%[0-9]+]]:_(i8), [[UV3:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[COPY]](i32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UV]](i8)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[ZEXT]], [[C]](i64)
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(i64) = G_ANYEXT [[UV]](i8)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY]], [[C]](i64)
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(i64) = G_ANYEXT [[COPY]](i32)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(i64) = G_ANYEXT [[LSHR]](i32)
; CHECK-NEXT: $x0 = COPY [[ANYEXT]](i64)
; CHECK-NEXT: $x1 = COPY [[ANYEXT1]](i64)
>From 98725f8ac858ec0f810ef4acbb4912fc34f2ceac Mon Sep 17 00:00:00 2001
From: "Doga, Kacper" <kacper.doga at intel.com>
Date: Mon, 10 Aug 2026 12:08:46 +0200
Subject: [PATCH 3/3] Revert "drop non-pow2 lowering condition; move legality
check to the top"
This reverts commit 1aa135a597811154df028f44eeb7b9de63581532.
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 22 +++++++++----------
.../AArch64/GlobalISel/legalize-bitcast.mir | 19 +++++++++-------
.../GlobalISel/legalize-merge-values.mir | 11 ++++------
.../GlobalISel/legalize-unmerge-values.mir | 6 +++--
4 files changed, 30 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 27b19cee953d8..cc2bce010c4de 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1183,6 +1183,16 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
getActionDefinitionsBuilder(Op)
+ .widenScalarToNextPow2(LitTyIdx, 8)
+ // Above s64 lowered shifts narrow back to a merge and never terminate
+ .lowerIf([=](const LegalityQuery &Q) {
+ const LLT BigTy = Q.Types[BigTyIdx];
+ return BigTy.isScalar() && !isPowerOf2_32(BigTy.getSizeInBits()) &&
+ BigTy.getSizeInBits() < 64;
+ })
+ .widenScalarToNextPow2(BigTyIdx, 32)
+ .clampScalar(LitTyIdx, s8, s64)
+ .clampScalar(BigTyIdx, s32, s128)
.legalIf([=](const LegalityQuery &Q) {
switch (Q.Types[BigTyIdx].getSizeInBits()) {
case 32:
@@ -1201,17 +1211,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
default:
return false;
}
- })
- .widenScalarToNextPow2(LitTyIdx, 8)
- // below 64-bit build with shifts, above it the lowered shifts narrow
- // back to a merge and never terminate
- .lowerIf([=](const LegalityQuery &Q) {
- const LLT BigTy = Q.Types[BigTyIdx];
- return BigTy.isScalar() && BigTy.getSizeInBits() < 64;
- })
- .widenScalarToNextPow2(BigTyIdx, 32)
- .clampScalar(LitTyIdx, s8, s64)
- .clampScalar(BigTyIdx, s32, s128);
+ });
}
// TODO : nxv4s16, nxv2s16, nxv2s32
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
index dbf4778c687f4..7a6726d4ad679 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir
@@ -224,17 +224,20 @@ body: |
; CHECK: liveins: $w0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %w:_(i32) = COPY $w0
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i8), [[UV1:%[0-9]+]]:_(i8), [[UV2:%[0-9]+]]:_(i8), [[UV3:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES %w(i32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UV]](i8)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR %w, [[C]](i64)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR %w, [[C1]](i64)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 15
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[ZEXT]], [[C]](i64)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 15
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C]](i64)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND %w, [[C2]]
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV]](i8)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ANYEXT]], [[C1]]
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C1]](i64)
+ ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV1]](i8)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ANYEXT1]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C2]](i64)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
; CHECK-NEXT: $w0 = COPY [[OR1]](i32)
; CHECK-NEXT: RET_ReallyLR implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir
index f30c3606a2443..c661df0fc1d1b 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir
@@ -29,13 +29,10 @@ body: |
; CHECK-LABEL: name: test_merge_s16_s8
; CHECK: %a:_(i32) = COPY $w0
; CHECK-NEXT: %b:_(i32) = COPY $w1
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND %b, [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C1]](i64)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND %a, [[C]]
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL]]
- ; CHECK-NEXT: %ext:_(i64) = G_ANYEXT [[OR]](i32)
+ ; CHECK-NEXT: %a_t:_(i8) = G_TRUNC %a(i32)
+ ; CHECK-NEXT: %b_t:_(i8) = G_TRUNC %b(i32)
+ ; CHECK-NEXT: %m:_(i16) = G_MERGE_VALUES %a_t(i8), %b_t(i8)
+ ; CHECK-NEXT: %ext:_(i64) = G_ANYEXT %m(i16)
; CHECK-NEXT: $x0 = COPY %ext(i64)
%a:_(i32) = COPY $w0
%b:_(i32) = COPY $w1
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir
index c25c173a05faa..a4a6a13fb1f1c 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir
@@ -32,9 +32,11 @@ body: |
; CHECK: liveins: $w0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $w0
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i8), [[UV1:%[0-9]+]]:_(i8), [[UV2:%[0-9]+]]:_(i8), [[UV3:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[COPY]](i32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UV]](i8)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY]], [[C]](i64)
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(i64) = G_ANYEXT [[COPY]](i32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[ZEXT]], [[C]](i64)
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(i64) = G_ANYEXT [[UV]](i8)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(i64) = G_ANYEXT [[LSHR]](i32)
; CHECK-NEXT: $x0 = COPY [[ANYEXT]](i64)
; CHECK-NEXT: $x1 = COPY [[ANYEXT1]](i64)
More information about the llvm-commits
mailing list