[llvm] [AArch64] Expand FORM_TRANSPOSED_REG_TUPLE to copies before regalloc (PR #207205)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 2 08:15:24 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Benjamin Maxwell (MacDue)
<details>
<summary>Changes</summary>
Previously, we kept the FORM_TRANSPOSED_REG_TUPLE nodes around during register allocation. The problem with this approach is that it does not model the potential overlap in live ranges between the destination and source operands.
As a result, the register allocator assumes it has complete freedom to allocate registers to the operands. For example, there is nothing stopping it from allocating:
```
z0, z1, z2, z3 FORM_TRANSPOSED_X4 z3, z2, z1, z0
```
However, such cases are hard to expand later, either requiring spills or complex shuffles. The current expansions of FORM_TRANSPOSED_REG_TUPLE miscompile in cases like this because, when naively expanded into copies after register allocation, earlier copies can clobber values that are still needed by later ones.
For the above case, the incorrect expansion would be:
```
z0 = COPY z3 // z0 clobbered
z1 = COPY z2 // z1 clobbered
z2 = COPY z1 // reads the wrong value of z1
z3 = COPY z0 // reads the wrong value of z0
```
This patch fixes the issue by expanding FORM_TRANSPOSED_REG_TUPLEs into copy sequences immediately before register allocation (in `aarch64-post-coalescer`).
For example:
```
%v4:zpr4mul4 = FORM_TRANSPOSED_X4 %v0:0, %v1:0, %v2:0, %v3:0
```
Expands to:
```
undef %v4.zsub0:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v0:0
%v4.zsub1:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v1:0
%v4.zsub2:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v2:0
%v4.zsub3:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v3:0
```
This is similar to how REG_SEQUENCE is expanded and allows the register allocator to reason about how the copies may interfere with one another.
To ensure our register allocation hints still apply, we encourage the scheduler to place FORM_TRANSPOSED_REG_TUPLE nodes immediately before their users. This keeps the live ranges of the hint nodes (COPY_INTO_TRANSPOSED_TUPLE) short while keeping the live ranges of their operands long. As a result, the register allocator is more likely to allocate registers to the copy sources first, which works best for our allocation hints.
---
Patch is 29.77 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/207205.diff
7 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp (+14-28)
- (modified) llvm/lib/Target/AArch64/AArch64PostCoalescerPass.cpp (+54-2)
- (modified) llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp (+64-60)
- (modified) llvm/lib/Target/AArch64/AArch64TargetMachine.cpp (+34)
- (modified) llvm/lib/Target/AArch64/SMEInstrFormats.td (+9)
- (added) llvm/test/CodeGen/AArch64/expand-form-transposed-tuple.mir (+69)
- (modified) llvm/test/CodeGen/AArch64/sme2-multivec-regalloc.mir (+29-32)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 5fa93da1544fc..16b7b2e74fcd2 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -59,10 +59,8 @@ class AArch64ExpandPseudoImpl {
TargetRegisterClass ContiguousClass,
TargetRegisterClass StridedClass,
unsigned ContiguousOpc, unsigned StridedOpc);
- bool expandFormTuplePseudo(MachineBasicBlock &MBB,
- MachineBasicBlock::iterator MBBI,
- MachineBasicBlock::iterator &NextMBBI,
- unsigned Size);
+ bool expandCopyIntoTuplePseudo(MachineInstr &MI, MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI);
bool expandMOVImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
unsigned BitSize);
@@ -1282,27 +1280,17 @@ bool AArch64ExpandPseudoImpl::expandMultiVecPseudo(
return true;
}
-bool AArch64ExpandPseudoImpl::expandFormTuplePseudo(
- MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
- MachineBasicBlock::iterator &NextMBBI, unsigned Size) {
- assert((Size == 2 || Size == 4) && "Invalid Tuple Size");
- MachineInstr &MI = *MBBI;
- Register ReturnTuple = MI.getOperand(0).getReg();
+bool AArch64ExpandPseudoImpl::expandCopyIntoTuplePseudo(
+ MachineInstr &MI, MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI) {
+ Register Src = MI.getOperand(1).getReg();
+ Register Dest = MI.getOperand(0).getReg();
- const TargetRegisterInfo *TRI =
- MBB.getParent()->getSubtarget().getRegisterInfo();
- for (unsigned I = 0; I < Size; ++I) {
- Register FormTupleOpReg = MI.getOperand(I + 1).getReg();
- Register ReturnTupleSubReg =
- TRI->getSubReg(ReturnTuple, AArch64::zsub0 + I);
- // Add copies to ensure the subregisters remain in the correct order
- // for any contigious operation they are used by.
- if (FormTupleOpReg != ReturnTupleSubReg)
- BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORR_ZZZ))
- .addReg(ReturnTupleSubReg, RegState::Define)
- .addReg(FormTupleOpReg)
- .addReg(FormTupleOpReg);
- }
+ if (Src != Dest)
+ BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORR_ZZZ))
+ .addReg(Dest, RegState::Define)
+ .addReg(Src)
+ .addReg(Src);
MI.eraseFromParent();
return true;
@@ -1940,10 +1928,8 @@ bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR4RegClass,
AArch64::ZPR4StridedRegClass,
AArch64::LDNT1D_4Z, AArch64::LDNT1D_4Z_STRIDED);
- case AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO:
- return expandFormTuplePseudo(MBB, MBBI, NextMBBI, 2);
- case AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO:
- return expandFormTuplePseudo(MBB, MBBI, NextMBBI, 4);
+ case AArch64::COPY_INTO_TRANSPOSED_TUPLE:
+ return expandCopyIntoTuplePseudo(MI, MBB, MBBI);
case AArch64::EON_ZZZ:
case AArch64::NAND_ZZZ:
case AArch64::NOR_ZZZ:
diff --git a/llvm/lib/Target/AArch64/AArch64PostCoalescerPass.cpp b/llvm/lib/Target/AArch64/AArch64PostCoalescerPass.cpp
index 7815b7526ca4b..56e271d38440d 100644
--- a/llvm/lib/Target/AArch64/AArch64PostCoalescerPass.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PostCoalescerPass.cpp
@@ -5,7 +5,6 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
-//===----------------------------------------------------------------------===//
#include "AArch64.h"
#include "AArch64MachineFunctionInfo.h"
@@ -21,9 +20,58 @@ using namespace llvm;
namespace {
+static bool expandFormTransposedRegTuple(MachineBasicBlock &MBB,
+ MachineInstr &MI, LiveIntervals &LIS) {
+ const TargetInstrInfo *TII =
+ MBB.getParent()->getSubtarget<AArch64Subtarget>().getInstrInfo();
+ unsigned TupleSize =
+ MI.getOpcode() == AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO ? 2 : 4;
+
+ DebugLoc DL = MI.getDebugLoc();
+ Register TupleReg = MI.getOperand(0).getReg();
+ SmallVector<Register, 5> OrigRegs{TupleReg};
+ MachineBasicBlock::iterator FirstCopyMBBI;
+
+ for (unsigned I = 0; I < TupleSize; ++I) {
+ MachineOperand &SrcOp = MI.getOperand(I + 1);
+ OrigRegs.push_back(SrcOp.getReg());
+
+ // Ensure that an if operand is killed the kill flag is placed on the final
+ // copy for that operand. TODO: Can we remove this? Requesting the live
+ // intervals seems to clear the kill flags anyway.
+ if (SrcOp.isKill()) {
+ for (unsigned J = I + 2; J < MI.getNumOperands(); ++J) {
+ MachineOperand &LaterOp = MI.getOperand(J);
+ if (LaterOp.getReg() == SrcOp.getReg()) {
+ LaterOp.setIsKill();
+ SrcOp.setIsKill(false);
+ }
+ }
+ }
+
+ RegState DefState = I == 0 ? RegState::Undef : RegState::NoFlags;
+ MachineInstr *CopyMI =
+ BuildMI(MBB, MI, DL, TII->get(AArch64::COPY_INTO_TRANSPOSED_TUPLE))
+ .addDef(TupleReg, DefState, AArch64::zsub0 + I)
+ .add(SrcOp)
+ .addImm(TupleSize);
+
+ if (I == 0)
+ FirstCopyMBBI = CopyMI;
+ }
+
+ MachineBasicBlock::iterator EndMBBI = std::next(MI.getIterator());
+ LIS.RemoveMachineInstrFromMaps(MI);
+ MI.eraseFromParent();
+
+ LIS.repairIntervalsInRange(&MBB, FirstCopyMBBI, EndMBBI, OrigRegs);
+ return true;
+}
+
bool runAArch64PostCoalescer(MachineFunction &MF, LiveIntervals &LIS) {
AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
- if (!FuncInfo->hasStreamingModeChanges())
+ if (!FuncInfo->hasStreamingModeChanges() &&
+ !MF.getSubtarget<AArch64Subtarget>().isStreaming())
return false;
MachineRegisterInfo &MRI = MF.getRegInfo();
@@ -34,6 +82,10 @@ bool runAArch64PostCoalescer(MachineFunction &MF, LiveIntervals &LIS) {
switch (MI.getOpcode()) {
default:
break;
+ case AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO:
+ case AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO:
+ Changed |= expandFormTransposedRegTuple(MBB, MI, LIS);
+ break;
case AArch64::COALESCER_BARRIER_FPR16:
case AArch64::COALESCER_BARRIER_FPR32:
case AArch64::COALESCER_BARRIER_FPR64:
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 9e55b29426dd2..9417e3d2df75f 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -1176,7 +1176,7 @@ static bool HandleDestructivePredicateHint(
// * Improve register allocation for SME multi-vector instructions where we can
// benefit from the strided- and contiguous register multi-vector tuples.
//
-// Here FORM_TRANSPOSED_REG_TUPLE nodes are created to improve register
+// Here COPY_INTO_TRANSPOSED_TUPLE nodes are created to improve register
// allocation where a consecutive multi-vector tuple is constructed from the
// same indices of multiple strided loads. This may still result in
// unnecessary copies between the loads and the tuple. Here we try to return a
@@ -1274,20 +1274,18 @@ bool AArch64RegisterInfo::getRegAllocationHints(
// callee-saved registers and so by default these will be pushed to the back
// of the allocation order for the ZPRStridedOrContiguous classes.
// If any of the instructions which define VirtReg are used by the
- // FORM_TRANSPOSED_REG_TUPLE pseudo, we want to favour reducing copy
+ // COPY_INTO_TRANSPOSED_TUPLE pseudos, we want to favour reducing copy
// instructions over reducing the number of clobbered callee-save registers,
// so we add the strided registers as a hint.
unsigned RegID = RegRC->getID();
if (RegID == AArch64::ZPR2StridedOrContiguousRegClassID ||
RegID == AArch64::ZPR4StridedOrContiguousRegClassID) {
- // Look through uses of the register for FORM_TRANSPOSED_REG_TUPLE.
+ // Look through uses of the register for COPY_INTO_TRANSPOSED_TUPLE.
for (const MachineInstr &Use : MRI.use_nodbg_instructions(VirtReg)) {
- if (Use.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO &&
- Use.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO)
+ if (Use.getOpcode() != AArch64::COPY_INTO_TRANSPOSED_TUPLE)
continue;
- unsigned UseOps = Use.getNumOperands() - 1;
const TargetRegisterClass *StridedRC;
switch (RegID) {
case AArch64::ZPR2StridedOrContiguousRegClassID:
@@ -1305,17 +1303,19 @@ bool AArch64RegisterInfo::getRegAllocationHints(
if (StridedRC->contains(Reg))
StridedOrder.push_back(Reg);
- int OpIdx = Use.findRegisterUseOperandIdx(VirtReg, this);
- assert(OpIdx != -1 && "Expected operand index from register use.");
+ unsigned TupleSize = Use.getOperand(2).getImm();
+ unsigned TupIdx = Use.getOperand(0).getSubReg() - AArch64::zsub0;
unsigned TupleID = MRI.getRegClass(Use.getOperand(0).getReg())->getID();
bool IsMulZPR = TupleID == AArch64::ZPR2Mul2RegClassID ||
TupleID == AArch64::ZPR4Mul4RegClassID;
- const MachineOperand *AssignedRegOp = llvm::find_if(
- make_range(Use.operands_begin() + 1, Use.operands_end()),
- [&VRM](const MachineOperand &Op) {
- return VRM->hasPhys(Op.getReg());
+ auto Copies = MRI.def_instructions(Use.getOperand(0).getReg());
+ auto CopyWithAssignedSrc =
+ llvm::find_if(Copies, [&](const MachineInstr &Def) {
+ auto &Src = Def.getOperand(1);
+ return Def.getOpcode() == Use.getOpcode() &&
+ VRM->hasPhys(Src.getReg());
});
// Example:
@@ -1326,7 +1326,10 @@ bool AArch64RegisterInfo::getRegAllocationHints(
// %v1:zpr2stridedorcontiguous = ld1 p0/z, [...]
// %v2:zpr2stridedorcontiguous = ld1 p0/z, [...]
// %v3:zpr2stridedorcontiguous = ld1 p0/z, [...]
- // %v4:zpr4mul4 = FORM_TRANSPOSED_X4 %v0:0, %v1:0, %v2:0, %v3:0
+ // %v4.zsub0:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v0:0
+ // %v4.zsub1:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v1:0
+ // %v4.zsub2:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v2:0
+ // %v4.zsub3:zpr4mul4 = COPY_INTO_TRANSPOSED_TUPLE %v3:0
//
// One such suitable allocation would be:
//
@@ -1334,58 +1337,62 @@ bool AArch64RegisterInfo::getRegAllocationHints(
// { z1, z9 } = ld1 p0/z, [...]
// { z2, z10 } = ld1 p0/z, [...]
// { z3, z11 } = ld1 p0/z, [...]
- // { z0, z1, z2, z3 } =
- // FORM_TRANSPOSED_X4 {z0, z8}:0, {z1, z9}:0, {z2, z10}:0, {z3, z11}:0
+ // z0 = COPY_INTO_TRANSPOSED_TUPLE {z0, z8}:0
+ // z1 = COPY_INTO_TRANSPOSED_TUPLE {z1, z9}:0
+ // z2 = COPY_INTO_TRANSPOSED_TUPLE {z2, z10}:0
+ // z3 = COPY_INTO_TRANSPOSED_TUPLE {z3, z11}:0
//
// Below we distinguish two cases when trying to find a register:
- // * None of the registers used by FORM_TRANSPOSED_X4 have been assigned
- // yet. In this case the code muse ensure that there are at least UseOps
+ // * None of the sources of the copies have been assigned a register yet.
+ // In this case the code must ensure that there are at least TupleSize
// free consecutive registers. If IsMulZPR is true, then the first of
- // registers must also be a multiple of UseOps, e.g. { z0, z1, z2, z3 }
- // is valid but { z1, z2, z3, z5 } is not.
- // * One or more of the registers used by FORM_TRANSPOSED_X4 is already
- // assigned a physical register, which means only checking that a
- // consecutive range of free tuple registers exists which includes
- // the assigned register.
+ // registers must also be a multiple of TupleSize, e.g.
+ // { z0, z1, z2, z3 } is valid but { z1, z2, z3, z5 } is not.
+ // * One or more copies already have registers assigned to their sources,
+ // which means only checking that a consecutive range of free tuple
+ // registers exists which includes the assigned register.
// e.g. in the example above, if { z0, z8 } is already allocated for
// %v0, we just need to ensure that { z1, z9 }, { z2, z10 } and
// { z3, z11 } are also free. If so, we add { z2, z10 }.
- if (AssignedRegOp == Use.operands_end()) {
+ if (CopyWithAssignedSrc == Copies.end()) {
// There are no registers already assigned to any of the pseudo
// operands. Look for a valid starting register for the group.
for (unsigned I = 0; I < StridedOrder.size(); ++I) {
MCPhysReg Reg = StridedOrder[I];
- // If the FORM_TRANSPOSE nodes use the ZPRMul classes, the starting
- // register of the first load should be a multiple of 2 or 4.
- unsigned SubRegIdx = Use.getOperand(OpIdx).getSubReg();
- if (IsMulZPR && (getSubReg(Reg, SubRegIdx) - AArch64::Z0) % UseOps !=
- ((unsigned)OpIdx - 1))
+ // If the COPY_INTO_TRANSPOSED_TUPLE nodes use the ZPRMul classes, the
+ // starting register of the first load should be a multiple of 2 or 4.
+ unsigned SubRegIdx = Use.getOperand(1).getSubReg();
+ if (IsMulZPR &&
+ (getSubReg(Reg, SubRegIdx) - AArch64::Z0) % TupleSize != TupIdx)
continue;
// In the example above, if VirtReg is the third operand of the
// tuple (%v2) and Reg == Z2_Z10, then we need to make sure that
// Z0_Z8, Z1_Z9 and Z3_Z11 are also available.
- auto IsFreeConsecutiveReg = [&](unsigned UseOp) {
- unsigned R = Reg - (OpIdx - 1) + UseOp;
+ auto IsFreeConsecutiveReg = [&](unsigned I) {
+ unsigned R = Reg - TupIdx + I;
return StridedRC->contains(R) &&
- (UseOp == 0 ||
+ (I == 0 ||
((getSubReg(R, AArch64::zsub0) - AArch64::Z0) ==
(getSubReg(R - 1, AArch64::zsub0) - AArch64::Z0) + 1)) &&
!Matrix->isPhysRegUsed(R);
};
- if (all_of(iota_range<unsigned>(0U, UseOps, /*Inclusive=*/false),
+ if (all_of(iota_range<unsigned>(0U, TupleSize, /*Inclusive=*/false),
IsFreeConsecutiveReg))
Hints.push_back(Reg);
}
} else {
- // At least one operand already has a physical register assigned.
+ // At least copy already has a physical register assigned to its source.
// Find the starting sub-register of this and use it to work out the
// correct strided register to suggest based on the current op index.
+ unsigned AssignedTupIdx =
+ CopyWithAssignedSrc->getOperand(0).getSubReg() - AArch64::zsub0;
MCPhysReg TargetStartReg =
- getSubReg(VRM->getPhys(AssignedRegOp->getReg()), AArch64::zsub0) +
- (OpIdx - AssignedRegOp->getOperandNo());
+ getSubReg(VRM->getPhys(CopyWithAssignedSrc->getOperand(1).getReg()),
+ AArch64::zsub0) +
+ (TupIdx - AssignedTupIdx);
for (unsigned I = 0; I < StridedOrder.size(); ++I)
if (getSubReg(StridedOrder[I], AArch64::zsub0) == TargetStartReg)
@@ -1398,34 +1405,31 @@ bool AArch64RegisterInfo::getRegAllocationHints(
}
}
- for (MachineInstr &MI : MRI.def_instructions(VirtReg)) {
- if (MI.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO &&
- MI.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO)
- return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints,
- MF, VRM);
-
- unsigned FirstOpSubReg = MI.getOperand(1).getSubReg();
- switch (FirstOpSubReg) {
- case AArch64::zsub0:
- case AArch64::zsub1:
- case AArch64::zsub2:
- case AArch64::zsub3:
- break;
- default:
+ for (auto &Def : MRI.def_instructions(VirtReg)) {
+ if (Def.getOpcode() != AArch64::COPY_INTO_TRANSPOSED_TUPLE)
continue;
- }
- // Look up the physical register mapped to the first operand of the pseudo.
- Register FirstOpVirtReg = MI.getOperand(1).getReg();
- if (!VRM->hasPhys(FirstOpVirtReg))
+ MachineOperand &Src = Def.getOperand(1);
+ MachineOperand &Dst = Def.getOperand(0);
+
+ if (!Src.getSubReg() || !Dst.getSubReg())
continue;
- MCRegister TupleStartReg =
- getSubReg(VRM->getPhys(FirstOpVirtReg), FirstOpSubReg);
- for (unsigned I = 0; I < Order.size(); ++I)
- if (MCRegister R = getSubReg(Order[I], AArch64::zsub0))
- if (R == TupleStartReg)
- Hints.push_back(Order[I]);
+ // FIXME: This is fragile. If we allocate a register to the Dst before Src,
+ // our hints are won't have any effect... This is currently mitigated by
+ // by trying to schedule copies immediately before their uses. This gives
+ // them a short live range (so they're low priority to allocate).
+ if (!VRM->hasPhys(Src.getReg()))
+ continue;
+
+ // Find the ZPR register mapped to the source of the copy.
+ MCPhysReg SrcZPR = getSubReg(VRM->getPhys(Src.getReg()), Src.getSubReg());
+
+ // Try to pick a tuple register for Dst with Src as a member.
+ for (MCPhysReg R : Order) {
+ if (getSubReg(R, Dst.getSubReg()) == SrcZPR)
+ Hints.push_back(R);
+ }
}
return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 898d92e74c85c..5b2c48c860206 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -510,6 +510,37 @@ AArch64TargetMachine::getSubtargetImpl(const Function &F) const {
return I.get();
}
+// Encourage placing FORM_TRANSPOSED_REG immediately before the instruction that
+// uses/consumes it. This ensures it has a short live range, which means we're
+// more likely to allocate registers its operands first (which works best for
+// the hints in AArch64RegisterInfo::getRegAllocationHints).
+static bool scheduleFormTransposedTupleAdjacentToUsers(
+ const TargetInstrInfo &TII, const TargetSubtargetInfo &TSI,
+ const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
+
+ auto *TRI = TSI.getRegisterInfo();
+ if (!FirstMI) {
+ // The SecondMI must be a multi-vector operation. So limit this to
+ // instructions that use full tuple registers (not a sub-register).
+ const MachineRegisterInfo &MRI = SecondMI.getMF()->getRegInfo();
+ for (const MachineOperand &Use : SecondMI.uses()) {
+ if (Use.isReg() && Use.getReg().isVirtual() && !Use.getSubReg() &&
+ TRI->isSubRegValidForRegClass(MRI.getRegClass(Use.getReg()),
+ AArch64::zsub0))
+ return true;
+ }
+ return false;
+ }
+
+ if (FirstMI->getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO &&
+ FirstMI->getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO)
+ return false;
+
+ Register TupleDef = FirstMI->getOperand(0).getReg();
+ return SecondMI.findRegisterUseOperandIdx(TupleDef, TSI.getRegisterInfo()) !=
+ -1;
+}
+
ScheduleDAGInstrs *
AArch64TargetMachine::createMachineScheduler(MachineSchedContext *C) const {
const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>();
@@ -518,6 +549,9 @@ AArch64TargetMachine::createMachineScheduler(MachineSchedContext *C) const {
DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
if (ST.hasFusion())
DAG->addMutation(createAArch64MacroFusionDAGMutation());
+ if (ST.hasSME() && ST.isStreaming())
+ DAG->addMutation(createMacroFusionDAGMutation(
+ scheduleFormTransposedTupleAdjacentToUsers));
return DAG;
}
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index f07fb8ad81f63..041e5d8973e8a 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -58,6 +58,15 @@ def FORM_TRANS...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/207205
More information about the llvm-commits
mailing list