[llvm] [AArch64] Fix correctness of FORM_TRANSPOSED_REG_TUPLE expansions (PR #205528)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 24 06:06:03 PDT 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/205528
>From b394ab80c5196d405af53228dfe56b738a6ff467 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 24 Jun 2026 10:42:59 +0000
Subject: [PATCH 1/6] Precommit tests
---
...pand-form-transposed-reg-tuple-pseudos.mir | 254 ++++++++++++++++++
1 file changed, 254 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir
diff --git a/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir b/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir
new file mode 100644
index 0000000000000..09aedaccf8197
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir
@@ -0,0 +1,254 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -run-pass=aarch64-expand-pseudo -verify-machineinstrs %s -o - | FileCheck %s
+
+# Post-RA tests for FORM_TRANSPOSED_REG_TUPLE_*_PSEUDO expansion.
+#
+# This test covers unlikely cases that ideally we'll never encounter in
+# practise, but still need to handle for correctness.
+#
+# Ideally, we should make it impossible for the register allocator to pick cases
+# with costly expansions (i.e., those that require swaps).
+
+
+# Naive copy lowering would clobber $z19 before $z20 is formed.
+---
+name: form_transposed_reg_tuple_x2_overlap
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z17, $z19
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x2_overlap
+ ; CHECK: liveins: $z17, $z19
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $z19 = ORR_ZZZ $z17, $z17
+ ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20
+ $z19_z20 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO $z17, $z19
+ RET_ReallyLR implicit killed $z19, implicit killed $z20
+...
+
+# Pure X2 permutation: this must use the fallback swap sequence.
+---
+name: form_transposed_reg_tuple_x2_swap
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z19, $z20
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x2_swap
+ ; CHECK: liveins: $z19, $z20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $z19 = ORR_ZZZ $z20, $z20
+ ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20
+ $z19_z20 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO $z20, $z19
+ RET_ReallyLR implicit killed $z19, implicit killed $z20
+...
+
+# Identity tuple: the pseudo should disappear without emitting copies.
+---
+name: form_transposed_reg_tuple_x2_identity
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z19, $z20
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x2_identity
+ ; CHECK: liveins: $z19, $z20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20
+ $z19_z20 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO $z19, $z20
+ RET_ReallyLR implicit killed $z19, implicit killed $z20
+...
+
+# Duplicate the first tuple source into the second lane.
+---
+name: form_transposed_reg_tuple_x2_duplicate_first_source
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z19
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x2_duplicate_first_source
+ ; CHECK: liveins: $z19
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20
+ $z19_z20 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO $z19, $z19
+ RET_ReallyLR implicit killed $z19, implicit killed $z20
+...
+
+# Duplicate the second tuple source into the first lane.
+---
+name: form_transposed_reg_tuple_x2_duplicate_second_source
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z20
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x2_duplicate_second_source
+ ; CHECK: liveins: $z20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $z19 = ORR_ZZZ $z20, $z20
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20
+ $z19_z20 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO $z20, $z20
+ RET_ReallyLR implicit killed $z19, implicit killed $z20
+...
+
+# Identity tuple: the pseudo should disappear without emitting copies.
+---
+name: form_transposed_reg_tuple_x4_identity
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z19, $z20, $z21, $z22
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x4_identity
+ ; CHECK: liveins: $z19, $z20, $z21, $z22
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+ $z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z19, $z20, $z21, $z22
+ RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+...
+
+# Splat one tuple member across the whole X4 tuple.
+---
+name: form_transposed_reg_tuple_x4_splat_from_tuple_member
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z19
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x4_splat_from_tuple_member
+ ; CHECK: liveins: $z19
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z21 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z22 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+ $z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z19, $z19, $z19, $z19
+ RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+...
+
+# Mixed external source plus tuple-member splat.
+---
+name: form_transposed_reg_tuple_x4_external_and_splat
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z17, $z19
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x4_external_and_splat
+ ; CHECK: liveins: $z17, $z19
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $z19 = ORR_ZZZ $z17, $z17
+ ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z21 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z22 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+ $z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z17, $z19, $z19, $z19
+ RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+...
+
+# Overlapping X4 tuple with no permutation: copies must be emitted in reverse dependency order.
+---
+name: form_transposed_reg_tuple_x4_overlap
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z17, $z19, $z20, $z21
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x4_overlap
+ ; CHECK: liveins: $z17, $z19, $z20, $z21
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $z19 = ORR_ZZZ $z17, $z17
+ ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z21 = ORR_ZZZ $z20, $z20
+ ; CHECK-NEXT: $z22 = ORR_ZZZ $z21, $z21
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+ $z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z17, $z19, $z20, $z21
+ RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+...
+
+# Mixed X4 case: independent copies first, then swap the remaining 2-cycle.
+---
+name: form_transposed_reg_tuple_x4_contains_swap
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z17, $z18, $z19, $z20
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x4_contains_swap
+ ; CHECK: liveins: $z17, $z18, $z19, $z20
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $z19 = ORR_ZZZ $z20, $z20
+ ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z21 = ORR_ZZZ $z17, $z17
+ ; CHECK-NEXT: $z22 = ORR_ZZZ $z18, $z18
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+ $z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z20, $z19, $z17, $z18
+ RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+...
+
+# Pure X4 permutation: expand into a sequence of swaps over the cycle.
+---
+name: form_transposed_reg_tuple_x4_rotate
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z19, $z20, $z21, $z22
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x4_rotate
+ ; CHECK: liveins: $z19, $z20, $z21, $z22
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $z19 = ORR_ZZZ $z20, $z20
+ ; CHECK-NEXT: $z20 = ORR_ZZZ $z21, $z21
+ ; CHECK-NEXT: $z21 = ORR_ZZZ $z22, $z22
+ ; CHECK-NEXT: $z22 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+ $z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z20, $z21, $z22, $z19
+ RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+...
+
+# Non-contiguous remaining 2-cycle after removing an identity and an external copy.
+---
+name: form_transposed_reg_tuple_x4_noncontiguous_2cycle
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z18, $z19, $z20, $z21
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x4_noncontiguous_2cycle
+ ; CHECK: liveins: $z18, $z19, $z20, $z21
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $z19 = ORR_ZZZ $z21, $z21
+ ; CHECK-NEXT: $z21 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z22 = ORR_ZZZ $z18, $z18
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+ $z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z21, $z20, $z19, $z18
+ RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+...
+
+# Non-contiguous remaining 3-cycle with one identity tuple member.
+---
+name: form_transposed_reg_tuple_x4_noncontiguous_3cycle
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z19, $z20, $z21, $z22
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x4_noncontiguous_3cycle
+ ; CHECK: liveins: $z19, $z20, $z21, $z22
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $z19 = ORR_ZZZ $z22, $z22
+ ; CHECK-NEXT: $z21 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z22 = ORR_ZZZ $z21, $z21
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+ $z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z22, $z20, $z19, $z21
+ RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+...
+
+# Mixed identity-plus-cycle case where only non-adjacent tuple members must swap.
+---
+name: form_transposed_reg_tuple_x4_identity_plus_cycle
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z19, $z20, $z21, $z22
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x4_identity_plus_cycle
+ ; CHECK: liveins: $z19, $z20, $z21, $z22
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $z20 = ORR_ZZZ $z22, $z22
+ ; CHECK-NEXT: $z22 = ORR_ZZZ $z20, $z20
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+ $z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z19, $z22, $z21, $z20
+ RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
+...
>From aba5498004cf5b4861e7717b931c0c4953e86096 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 24 Jun 2026 10:49:08 +0000
Subject: [PATCH 2/6] [AArch64] Fix correctness of FORM_TRANSPOSED_REG_TUPLE
expansions
This patch is a "quick fix" to the correctness of the
FORM_TRANSPOSED_REG_TUPLE expansions. These pseudos are meant to act
like a parallel copy, but the current simple expansion does not account
for the cases where the operands of the pseudo are also members of the
tuple. In these cases the expansion can clobber members of the tuple
which results in miscompiles.
This patch fixes these cases by emitting the copies in reverse
dependency order (if possible). That is only write a destination once
all users of that register have been handled.
However, in the worse case the tuple pseudo could be an arbitrary
permutation. This case is very unlikely, but for correctness we handle
it as a sequence of swaps.
Note: This is a temporary workaround. Ideally, we constrain the
FORM_TRANSPOSED_REG_TUPLE pseudos so that cases that need complex
expansions are impossible.
---
.../AArch64/AArch64ExpandPseudoInsts.cpp | 117 ++++++++++++++++--
...pand-form-transposed-reg-tuple-pseudos.mir | 62 ++++++----
2 files changed, 144 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 5fa93da1544fc..5e100b0a84bfb 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -1282,26 +1282,123 @@ bool AArch64ExpandPseudoImpl::expandMultiVecPseudo(
return true;
}
+struct CopyReg {
+ Register Dest;
+ Register Src;
+};
+
bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
MachineBasicBlock::iterator &NextMBBI, unsigned Size) {
assert((Size == 2 || Size == 4) && "Invalid Tuple Size");
+ const TargetRegisterInfo *TRI =
+ MBB.getParent()->getSubtarget().getRegisterInfo();
MachineInstr &MI = *MBBI;
Register ReturnTuple = MI.getOperand(0).getReg();
+ Register TupleZSub0 = TRI->getSubReg(ReturnTuple, AArch64::zsub0);
- const TargetRegisterInfo *TRI =
- MBB.getParent()->getSubtarget().getRegisterInfo();
+ auto IsTupleReturnMember = [&](Register ZReg) {
+ return ZReg >= TupleZSub0 && ZReg < (TupleZSub0 + Size);
+ };
+
+ // Collect the copies required to form the tuple. Count if any return tuple
+ // members appear as operands to the FORM_TRANSPOSED_REG_TUPLE. If a
+ // destination appears as a source operand, we can't write to that register
+ // until we've handled the other copies that use it.
+ SmallVector<CopyReg, 4> Copies;
+ unsigned ReturnOpUseCount[4] = {};
for (unsigned I = 0; I < Size; ++I) {
- Register FormTupleOpReg = MI.getOperand(I + 1).getReg();
Register ReturnTupleSubReg =
TRI->getSubReg(ReturnTuple, AArch64::zsub0 + I);
- // Add copies to ensure the subregisters remain in the correct order
- // for any contigious operation they are used by.
- if (FormTupleOpReg != ReturnTupleSubReg)
- BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORR_ZZZ))
- .addReg(ReturnTupleSubReg, RegState::Define)
- .addReg(FormTupleOpReg)
- .addReg(FormTupleOpReg);
+ Register FormTupleOpReg = MI.getOperand(I + 1).getReg();
+ Copies.push_back({ReturnTupleSubReg, FormTupleOpReg});
+ if (FormTupleOpReg != ReturnTupleSubReg &&
+ IsTupleReturnMember(FormTupleOpReg))
+ ++ReturnOpUseCount[FormTupleOpReg - TupleZSub0];
+ }
+
+ SmallVector<CopyReg, 4> Worklist;
+ for (auto [I, Copy] : enumerate(Copies))
+ if (ReturnOpUseCount[I] == 0)
+ Worklist.push_back(Copy);
+
+ // Emit the copies in reverse dependency order.
+ auto DL = MI.getDebugLoc();
+ unsigned CopiesHandled = 0;
+ while (!Worklist.empty()) {
+ CopyReg Copy = Worklist.pop_back_val();
+ if (Copy.Dest != Copy.Src) {
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORR_ZZZ))
+ .addReg(Copy.Dest, RegState::Define)
+ .addReg(Copy.Src)
+ .addReg(Copy.Src);
+ if (IsTupleReturnMember(Copy.Src)) {
+ unsigned TupleIdx = Copy.Src - TupleZSub0;
+ if (--ReturnOpUseCount[TupleIdx] == 0)
+ Worklist.push_back(Copies[TupleIdx]);
+ }
+ }
+ ++CopiesHandled;
+ }
+
+ if (CopiesHandled == Copies.size()) {
+ MI.eraseFromParent();
+ return true;
+ }
+
+ // This FORM_TRANSPOSED_REG_TUPLE contains a permutation. FIXME: This really
+ // should _never_ happer. This is an issue with FORM_TRANSPOSED_REG_TUPLE
+ // being undercontrained.
+
+ // We don't have many options here. It's too late to allocate a spill slot,
+ // so we can't rely on scavenging. So, we handle this my emitting a sequence
+ // of XOR swaps. Again FIXME -- we should not need to do this!
+
+ auto emitZRegSwap = [&](Register ZA, Register ZB) {
+ assert(ZA != ZB && "xor-swap of same reg clobbers it");
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::EOR_ZZZ), ZA)
+ .addReg(ZA)
+ .addReg(ZB);
+
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::EOR_ZZZ), ZB)
+ .addReg(ZB)
+ .addReg(ZA);
+
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::EOR_ZZZ), ZA)
+ .addReg(ZA)
+ .addReg(ZB);
+ };
+
+ // Extract the permutation.
+ SmallVector<int, 4> Perm;
+ for (unsigned I = 0; I < Size; ++I) {
+ if (ReturnOpUseCount[I] > 0) {
+ assert(ReturnOpUseCount[I] == 1);
+ Perm.push_back(Copies[I].Src - TupleZSub0);
+ } else {
+ Perm.push_back(I);
+ }
+ }
+
+ // Decompose the permutation to swaps and emit.
+ SmallBitVector Visited(4);
+ for (unsigned I = 0; I < Perm.size(); I++) {
+ if (Visited[I])
+ continue;
+
+ int J = I;
+ SmallVector<int, 4> Cycle;
+ while (!Visited[J]) {
+ Visited[J] = true;
+ Cycle.push_back(J);
+ J = Perm[J];
+ }
+
+ for (int K = Cycle.size() - 1; K > 0; --K) {
+ Register ZA = Cycle[0] + TupleZSub0;
+ Register ZB = Cycle[K] + TupleZSub0;
+ emitZRegSwap(ZA, ZB);
+ }
}
MI.eraseFromParent();
diff --git a/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir b/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir
index 09aedaccf8197..8573ede8e32e7 100644
--- a/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir
+++ b/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir
@@ -20,8 +20,8 @@ body: |
; CHECK-LABEL: name: form_transposed_reg_tuple_x2_overlap
; CHECK: liveins: $z17, $z19
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $z19 = ORR_ZZZ $z17, $z17
; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z19 = ORR_ZZZ $z17, $z17
; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20
$z19_z20 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO $z17, $z19
RET_ReallyLR implicit killed $z19, implicit killed $z20
@@ -37,8 +37,9 @@ body: |
; CHECK-LABEL: name: form_transposed_reg_tuple_x2_swap
; CHECK: liveins: $z19, $z20
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $z19 = ORR_ZZZ $z20, $z20
- ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z20
+ ; CHECK-NEXT: $z20 = EOR_ZZZ $z20, $z19
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z20
; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20
$z19_z20 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO $z20, $z19
RET_ReallyLR implicit killed $z19, implicit killed $z20
@@ -116,9 +117,9 @@ body: |
; CHECK-LABEL: name: form_transposed_reg_tuple_x4_splat_from_tuple_member
; CHECK: liveins: $z19
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
- ; CHECK-NEXT: $z21 = ORR_ZZZ $z19, $z19
; CHECK-NEXT: $z22 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z21 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
$z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z19, $z19, $z19, $z19
RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
@@ -134,10 +135,10 @@ body: |
; CHECK-LABEL: name: form_transposed_reg_tuple_x4_external_and_splat
; CHECK: liveins: $z17, $z19
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $z19 = ORR_ZZZ $z17, $z17
- ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
- ; CHECK-NEXT: $z21 = ORR_ZZZ $z19, $z19
; CHECK-NEXT: $z22 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z21 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z19 = ORR_ZZZ $z17, $z17
; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
$z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z17, $z19, $z19, $z19
RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
@@ -153,10 +154,10 @@ body: |
; CHECK-LABEL: name: form_transposed_reg_tuple_x4_overlap
; CHECK: liveins: $z17, $z19, $z20, $z21
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $z19 = ORR_ZZZ $z17, $z17
- ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
- ; CHECK-NEXT: $z21 = ORR_ZZZ $z20, $z20
; CHECK-NEXT: $z22 = ORR_ZZZ $z21, $z21
+ ; CHECK-NEXT: $z21 = ORR_ZZZ $z20, $z20
+ ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z19 = ORR_ZZZ $z17, $z17
; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
$z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z17, $z19, $z20, $z21
RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
@@ -172,10 +173,11 @@ body: |
; CHECK-LABEL: name: form_transposed_reg_tuple_x4_contains_swap
; CHECK: liveins: $z17, $z18, $z19, $z20
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $z19 = ORR_ZZZ $z20, $z20
- ; CHECK-NEXT: $z20 = ORR_ZZZ $z19, $z19
- ; CHECK-NEXT: $z21 = ORR_ZZZ $z17, $z17
; CHECK-NEXT: $z22 = ORR_ZZZ $z18, $z18
+ ; CHECK-NEXT: $z21 = ORR_ZZZ $z17, $z17
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z20
+ ; CHECK-NEXT: $z20 = EOR_ZZZ $z20, $z19
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z20
; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
$z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z20, $z19, $z17, $z18
RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
@@ -191,10 +193,15 @@ body: |
; CHECK-LABEL: name: form_transposed_reg_tuple_x4_rotate
; CHECK: liveins: $z19, $z20, $z21, $z22
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $z19 = ORR_ZZZ $z20, $z20
- ; CHECK-NEXT: $z20 = ORR_ZZZ $z21, $z21
- ; CHECK-NEXT: $z21 = ORR_ZZZ $z22, $z22
- ; CHECK-NEXT: $z22 = ORR_ZZZ $z19, $z19
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z22
+ ; CHECK-NEXT: $z22 = EOR_ZZZ $z22, $z19
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z22
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z21
+ ; CHECK-NEXT: $z21 = EOR_ZZZ $z21, $z19
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z21
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z20
+ ; CHECK-NEXT: $z20 = EOR_ZZZ $z20, $z19
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z20
; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
$z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z20, $z21, $z22, $z19
RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
@@ -210,9 +217,10 @@ body: |
; CHECK-LABEL: name: form_transposed_reg_tuple_x4_noncontiguous_2cycle
; CHECK: liveins: $z18, $z19, $z20, $z21
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $z19 = ORR_ZZZ $z21, $z21
- ; CHECK-NEXT: $z21 = ORR_ZZZ $z19, $z19
; CHECK-NEXT: $z22 = ORR_ZZZ $z18, $z18
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z21
+ ; CHECK-NEXT: $z21 = EOR_ZZZ $z21, $z19
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z21
; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
$z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z21, $z20, $z19, $z18
RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
@@ -228,9 +236,12 @@ body: |
; CHECK-LABEL: name: form_transposed_reg_tuple_x4_noncontiguous_3cycle
; CHECK: liveins: $z19, $z20, $z21, $z22
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $z19 = ORR_ZZZ $z22, $z22
- ; CHECK-NEXT: $z21 = ORR_ZZZ $z19, $z19
- ; CHECK-NEXT: $z22 = ORR_ZZZ $z21, $z21
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z21
+ ; CHECK-NEXT: $z21 = EOR_ZZZ $z21, $z19
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z21
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z22
+ ; CHECK-NEXT: $z22 = EOR_ZZZ $z22, $z19
+ ; CHECK-NEXT: $z19 = EOR_ZZZ $z19, $z22
; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
$z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z22, $z20, $z19, $z21
RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
@@ -246,8 +257,9 @@ body: |
; CHECK-LABEL: name: form_transposed_reg_tuple_x4_identity_plus_cycle
; CHECK: liveins: $z19, $z20, $z21, $z22
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $z20 = ORR_ZZZ $z22, $z22
- ; CHECK-NEXT: $z22 = ORR_ZZZ $z20, $z20
+ ; CHECK-NEXT: $z20 = EOR_ZZZ $z20, $z22
+ ; CHECK-NEXT: $z22 = EOR_ZZZ $z22, $z20
+ ; CHECK-NEXT: $z20 = EOR_ZZZ $z20, $z22
; CHECK-NEXT: RET undef $lr, implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
$z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z19, $z22, $z21, $z20
RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
>From a792185cd0de037a24e5ed63c5aab670308b9121 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 24 Jun 2026 11:37:05 +0000
Subject: [PATCH 3/6] Tweaks
---
.../AArch64/AArch64ExpandPseudoInsts.cpp | 28 +++++++++----------
...pand-form-transposed-reg-tuple-pseudos.mir | 2 +-
2 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 5e100b0a84bfb..f8c1a9e03c972 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -1305,25 +1305,25 @@ bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
// members appear as operands to the FORM_TRANSPOSED_REG_TUPLE. If a
// destination appears as a source operand, we can't write to that register
// until we've handled the other copies that use it.
- SmallVector<CopyReg, 4> Copies;
+ CopyReg Copies[4] = {};
unsigned ReturnOpUseCount[4] = {};
for (unsigned I = 0; I < Size; ++I) {
Register ReturnTupleSubReg =
TRI->getSubReg(ReturnTuple, AArch64::zsub0 + I);
Register FormTupleOpReg = MI.getOperand(I + 1).getReg();
- Copies.push_back({ReturnTupleSubReg, FormTupleOpReg});
+ Copies[I] = {ReturnTupleSubReg, FormTupleOpReg};
if (FormTupleOpReg != ReturnTupleSubReg &&
IsTupleReturnMember(FormTupleOpReg))
++ReturnOpUseCount[FormTupleOpReg - TupleZSub0];
}
SmallVector<CopyReg, 4> Worklist;
- for (auto [I, Copy] : enumerate(Copies))
+ for (unsigned I = 0; I < Size; ++I)
if (ReturnOpUseCount[I] == 0)
- Worklist.push_back(Copy);
+ Worklist.push_back(Copies[I]);
// Emit the copies in reverse dependency order.
- auto DL = MI.getDebugLoc();
+ DebugLoc DL = MI.getDebugLoc();
unsigned CopiesHandled = 0;
while (!Worklist.empty()) {
CopyReg Copy = Worklist.pop_back_val();
@@ -1341,17 +1341,17 @@ bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
++CopiesHandled;
}
- if (CopiesHandled == Copies.size()) {
+ if (CopiesHandled == Size) {
MI.eraseFromParent();
return true;
}
// This FORM_TRANSPOSED_REG_TUPLE contains a permutation. FIXME: This really
- // should _never_ happer. This is an issue with FORM_TRANSPOSED_REG_TUPLE
- // being undercontrained.
+ // should _never_ happen. This is an issue with FORM_TRANSPOSED_REG_TUPLE
+ // being underconstrained.
// We don't have many options here. It's too late to allocate a spill slot,
- // so we can't rely on scavenging. So, we handle this my emitting a sequence
+ // so we can't rely on scavenging. So, we handle this by emitting a sequence
// of XOR swaps. Again FIXME -- we should not need to do this!
auto emitZRegSwap = [&](Register ZA, Register ZB) {
@@ -1370,19 +1370,19 @@ bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
};
// Extract the permutation.
- SmallVector<int, 4> Perm;
+ unsigned Perm[4] = {};
for (unsigned I = 0; I < Size; ++I) {
if (ReturnOpUseCount[I] > 0) {
assert(ReturnOpUseCount[I] == 1);
- Perm.push_back(Copies[I].Src - TupleZSub0);
+ Perm[I] = Copies[I].Src - TupleZSub0;
} else {
- Perm.push_back(I);
+ Perm[I] = I;
}
}
// Decompose the permutation to swaps and emit.
- SmallBitVector Visited(4);
- for (unsigned I = 0; I < Perm.size(); I++) {
+ bool Visited[4] = {};
+ for (unsigned I = 0; I < Size; ++I) {
if (Visited[I])
continue;
diff --git a/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir b/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir
index 8573ede8e32e7..3da30790cf785 100644
--- a/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir
+++ b/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir
@@ -4,7 +4,7 @@
# Post-RA tests for FORM_TRANSPOSED_REG_TUPLE_*_PSEUDO expansion.
#
# This test covers unlikely cases that ideally we'll never encounter in
-# practise, but still need to handle for correctness.
+# practice, but still need to handle for correctness.
#
# Ideally, we should make it impossible for the register allocator to pick cases
# with costly expansions (i.e., those that require swaps).
>From 243369d466ce0f2a51407c1e755151e5fe5fe2a0 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 24 Jun 2026 11:54:41 +0000
Subject: [PATCH 4/6] Tweaks
---
.../AArch64/AArch64ExpandPseudoInsts.cpp | 27 +++++++++----------
...pand-form-transposed-reg-tuple-pseudos.mir | 19 +++++++++++--
2 files changed, 30 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index f8c1a9e03c972..7ddf62bce31d5 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -1285,6 +1285,7 @@ bool AArch64ExpandPseudoImpl::expandMultiVecPseudo(
struct CopyReg {
Register Dest;
Register Src;
+ int SrcTupleIdx = -1;
};
bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
@@ -1295,11 +1296,6 @@ bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
MBB.getParent()->getSubtarget().getRegisterInfo();
MachineInstr &MI = *MBBI;
Register ReturnTuple = MI.getOperand(0).getReg();
- Register TupleZSub0 = TRI->getSubReg(ReturnTuple, AArch64::zsub0);
-
- auto IsTupleReturnMember = [&](Register ZReg) {
- return ZReg >= TupleZSub0 && ZReg < (TupleZSub0 + Size);
- };
// Collect the copies required to form the tuple. Count if any return tuple
// members appear as operands to the FORM_TRANSPOSED_REG_TUPLE. If a
@@ -1313,8 +1309,12 @@ bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
Register FormTupleOpReg = MI.getOperand(I + 1).getReg();
Copies[I] = {ReturnTupleSubReg, FormTupleOpReg};
if (FormTupleOpReg != ReturnTupleSubReg &&
- IsTupleReturnMember(FormTupleOpReg))
- ++ReturnOpUseCount[FormTupleOpReg - TupleZSub0];
+ TRI->isSubRegister(ReturnTuple, FormTupleOpReg)) {
+ int SrcTupleIdx =
+ TRI->getSubRegIndex(ReturnTuple, FormTupleOpReg) - AArch64::zsub0;
+ ++ReturnOpUseCount[SrcTupleIdx];
+ Copies[I].SrcTupleIdx = SrcTupleIdx;
+ }
}
SmallVector<CopyReg, 4> Worklist;
@@ -1332,10 +1332,9 @@ bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
.addReg(Copy.Dest, RegState::Define)
.addReg(Copy.Src)
.addReg(Copy.Src);
- if (IsTupleReturnMember(Copy.Src)) {
- unsigned TupleIdx = Copy.Src - TupleZSub0;
- if (--ReturnOpUseCount[TupleIdx] == 0)
- Worklist.push_back(Copies[TupleIdx]);
+ if (Copy.SrcTupleIdx != -1) {
+ if (--ReturnOpUseCount[Copy.SrcTupleIdx] == 0)
+ Worklist.push_back(Copies[Copy.SrcTupleIdx]);
}
}
++CopiesHandled;
@@ -1374,7 +1373,7 @@ bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
for (unsigned I = 0; I < Size; ++I) {
if (ReturnOpUseCount[I] > 0) {
assert(ReturnOpUseCount[I] == 1);
- Perm[I] = Copies[I].Src - TupleZSub0;
+ Perm[I] = Copies[I].SrcTupleIdx;
} else {
Perm[I] = I;
}
@@ -1395,8 +1394,8 @@ bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
}
for (int K = Cycle.size() - 1; K > 0; --K) {
- Register ZA = Cycle[0] + TupleZSub0;
- Register ZB = Cycle[K] + TupleZSub0;
+ Register ZA = TRI->getSubReg(ReturnTuple, AArch64::zsub0 + Cycle[0]);
+ Register ZB = TRI->getSubReg(ReturnTuple, AArch64::zsub0 + Cycle[K]);
emitZRegSwap(ZA, ZB);
}
}
diff --git a/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir b/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir
index 3da30790cf785..25cebd765e337 100644
--- a/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir
+++ b/llvm/test/CodeGen/AArch64/expand-form-transposed-reg-tuple-pseudos.mir
@@ -44,7 +44,23 @@ body: |
$z19_z20 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO $z20, $z19
RET_ReallyLR implicit killed $z19, implicit killed $z20
...
-
+# Pure X2 permutation: this must use the fallback swap sequence (z31_z0).
+---
+name: form_transposed_reg_tuple_x2_swap_reg_wrap
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $z0, $z31
+ ; CHECK-LABEL: name: form_transposed_reg_tuple_x2_swap_reg_wrap
+ ; CHECK: liveins: $z0, $z31
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $z31 = EOR_ZZZ $z31, $z0
+ ; CHECK-NEXT: $z0 = EOR_ZZZ $z0, $z31
+ ; CHECK-NEXT: $z31 = EOR_ZZZ $z31, $z0
+ ; CHECK-NEXT: RET undef $lr, implicit killed $z31, implicit killed $z0
+ $z31_z0 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO $z0, $z31
+ RET_ReallyLR implicit killed $z31, implicit killed $z0
+...
# Identity tuple: the pseudo should disappear without emitting copies.
---
name: form_transposed_reg_tuple_x2_identity
@@ -143,7 +159,6 @@ body: |
$z19_z20_z21_z22 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO $z17, $z19, $z19, $z19
RET_ReallyLR implicit killed $z19, implicit killed $z20, implicit killed $z21, implicit killed $z22
...
-
# Overlapping X4 tuple with no permutation: copies must be emitted in reverse dependency order.
---
name: form_transposed_reg_tuple_x4_overlap
>From ff8e5af3fa8eac38302b4d20c1cfdf1cfd4e9869 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 24 Jun 2026 12:25:58 +0000
Subject: [PATCH 5/6] Tweaks
---
llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 7ddf62bce31d5..6b131cae9bbf0 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -1302,7 +1302,7 @@ bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
// destination appears as a source operand, we can't write to that register
// until we've handled the other copies that use it.
CopyReg Copies[4] = {};
- unsigned ReturnOpUseCount[4] = {};
+ unsigned DestUseCount[4] = {};
for (unsigned I = 0; I < Size; ++I) {
Register ReturnTupleSubReg =
TRI->getSubReg(ReturnTuple, AArch64::zsub0 + I);
@@ -1312,14 +1312,14 @@ bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
TRI->isSubRegister(ReturnTuple, FormTupleOpReg)) {
int SrcTupleIdx =
TRI->getSubRegIndex(ReturnTuple, FormTupleOpReg) - AArch64::zsub0;
- ++ReturnOpUseCount[SrcTupleIdx];
+ ++DestUseCount[SrcTupleIdx];
Copies[I].SrcTupleIdx = SrcTupleIdx;
}
}
SmallVector<CopyReg, 4> Worklist;
for (unsigned I = 0; I < Size; ++I)
- if (ReturnOpUseCount[I] == 0)
+ if (DestUseCount[I] == 0)
Worklist.push_back(Copies[I]);
// Emit the copies in reverse dependency order.
@@ -1333,7 +1333,7 @@ bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
.addReg(Copy.Src)
.addReg(Copy.Src);
if (Copy.SrcTupleIdx != -1) {
- if (--ReturnOpUseCount[Copy.SrcTupleIdx] == 0)
+ if (--DestUseCount[Copy.SrcTupleIdx] == 0)
Worklist.push_back(Copies[Copy.SrcTupleIdx]);
}
}
@@ -1371,8 +1371,8 @@ bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
// Extract the permutation.
unsigned Perm[4] = {};
for (unsigned I = 0; I < Size; ++I) {
- if (ReturnOpUseCount[I] > 0) {
- assert(ReturnOpUseCount[I] == 1);
+ if (DestUseCount[I] > 0) {
+ assert(DestUseCount[I] == 1);
Perm[I] = Copies[I].SrcTupleIdx;
} else {
Perm[I] = I;
>From 753a49dfb97f021a898b529ea0cf8661508bfc80 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 24 Jun 2026 13:05:46 +0000
Subject: [PATCH 6/6] Update test
---
.../AArch64/sme2-multivec-regalloc.mir | 26 +++++++++----------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sme2-multivec-regalloc.mir b/llvm/test/CodeGen/AArch64/sme2-multivec-regalloc.mir
index c3338b14522cb..7e7d5e7407753 100644
--- a/llvm/test/CodeGen/AArch64/sme2-multivec-regalloc.mir
+++ b/llvm/test/CodeGen/AArch64/sme2-multivec-regalloc.mir
@@ -32,19 +32,19 @@ body: |
; CHECK-NEXT: add x10, x9, x1
; CHECK-NEXT: ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x9]
; CHECK-NEXT: ld1b { z4.b - z7.b }, pn8/z, [x0, x10]
- ; CHECK-NEXT: mov z8.d, z16.d
- ; CHECK-NEXT: mov z9.d, z18.d
- ; CHECK-NEXT: mov z21.d, z22.d
- ; CHECK-NEXT: mov z10.d, z19.d
- ; CHECK-NEXT: mov z22.d, z23.d
- ; CHECK-NEXT: mov z25.d, z26.d
- ; CHECK-NEXT: mov z11.d, z4.d
- ; CHECK-NEXT: mov z23.d, z5.d
- ; CHECK-NEXT: mov z26.d, z27.d
- ; CHECK-NEXT: mov z27.d, z6.d
- ; CHECK-NEXT: mov z29.d, z30.d
- ; CHECK-NEXT: mov z30.d, z31.d
- ; CHECK-NEXT: mov z31.d, z7.d
+ ; CHECK-NEXT: mov z9.d, z18.d
+ ; CHECK-NEXT: mov z8.d, z16.d
+ ; CHECK-NEXT: mov z21.d, z22.d
+ ; CHECK-NEXT: mov z10.d, z19.d
+ ; CHECK-NEXT: mov z22.d, z23.d
+ ; CHECK-NEXT: mov z25.d, z26.d
+ ; CHECK-NEXT: mov z11.d, z4.d
+ ; CHECK-NEXT: mov z23.d, z5.d
+ ; CHECK-NEXT: mov z26.d, z27.d
+ ; CHECK-NEXT: mov z27.d, z6.d
+ ; CHECK-NEXT: mov z29.d, z30.d
+ ; CHECK-NEXT: mov z30.d, z31.d
+ ; CHECK-NEXT: mov z31.d, z7.d
; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z8.b - z11.b }, z0.b[0]
; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]
; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]
More information about the llvm-commits
mailing list