[llvm] [AMDGPU][DAG] Allow extract_subvector for free register copy (PR #214152)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 01:15:51 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Shoreshen
<details>
<summary>Changes</summary>
---
Patch is 2.93 MiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/214152.diff
110 Files Affected:
- (modified) llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp (+8-1)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp (+22)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h (+1)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp (+19)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h (+7)
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+156-29)
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.h (+3)
- (modified) llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp (+11)
- (modified) llvm/lib/Target/AMDGPU/SIRegisterInfo.h (+6)
- (modified) llvm/test/CodeGen/AMDGPU/agpr-csr.ll (+353-281)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll (+88-77)
- (modified) llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll (+94-94)
- (modified) llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll (+63-63)
- (modified) llvm/test/CodeGen/AMDGPU/bf16.ll (+6-6)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll (+45-30)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll (+11-7)
- (modified) llvm/test/CodeGen/AMDGPU/combine-vload-extract.ll (+6-10)
- (modified) llvm/test/CodeGen/AMDGPU/dag-divergence-atomic.ll (+60-104)
- (modified) llvm/test/CodeGen/AMDGPU/ds-alignment.ll (+36-22)
- (modified) llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll (+14-14)
- (added) llvm/test/CodeGen/AMDGPU/extract_subvector_to_subreg.ll (+139)
- (modified) llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll (+4-7)
- (modified) llvm/test/CodeGen/AMDGPU/fence-barrier-latency.ll (+1-1)
- (modified) llvm/test/CodeGen/AMDGPU/fma.f64.ll (+20-8)
- (modified) llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll (+168-156)
- (modified) llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll (+32-32)
- (modified) llvm/test/CodeGen/AMDGPU/freeze.ll (+920-1135)
- (modified) llvm/test/CodeGen/AMDGPU/function-returns.ll (+23-23)
- (modified) llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll (+7-7)
- (modified) llvm/test/CodeGen/AMDGPU/image-load-d16-tfe.ll (+2-2)
- (modified) llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll (+690-881)
- (modified) llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll (+8-10)
- (modified) llvm/test/CodeGen/AMDGPU/inlineasm-sgmask.ll (+3-1)
- (modified) llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll (+825-1135)
- (modified) llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll (+265-259)
- (modified) llvm/test/CodeGen/AMDGPU/invalid-cast-load-i1.ll (+1-2)
- (modified) llvm/test/CodeGen/AMDGPU/kernel-args.ll (+202-217)
- (modified) llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll (+11-11)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll (+12-12)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll (+136-136)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll (+2-2)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll (+57-58)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll (+12-24)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll (+20-40)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll (+144-224)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll (+556-914)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll (+20-21)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll (+911-921)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll (+108-194)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll (+236-404)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ptr.ll (+9-10)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.iterative.ll (+51-51)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll (+136-136)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll (+40-80)
- (modified) llvm/test/CodeGen/AMDGPU/load-constant-i16.ll (+372-374)
- (modified) llvm/test/CodeGen/AMDGPU/load-constant-i32.ll (+97-99)
- (modified) llvm/test/CodeGen/AMDGPU/load-constant-i64.ll (+203-212)
- (modified) llvm/test/CodeGen/AMDGPU/load-global-f32.ll (+50-49)
- (modified) llvm/test/CodeGen/AMDGPU/load-global-i16.ll (+137-137)
- (modified) llvm/test/CodeGen/AMDGPU/load-global-i32.ll (+149-150)
- (modified) llvm/test/CodeGen/AMDGPU/load-local-i16.ll (+784-788)
- (modified) llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-nontemporal-metadata.ll (+18-8)
- (modified) llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll (+2258-2250)
- (modified) llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll (+56-64)
- (modified) llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll (+159-161)
- (modified) llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll (+36-36)
- (modified) llvm/test/CodeGen/AMDGPU/no-folding-imm-to-inst-with-fi.ll (+37-38)
- (modified) llvm/test/CodeGen/AMDGPU/packed-fp32.ll (+184-64)
- (modified) llvm/test/CodeGen/AMDGPU/packed-fp64.ll (+62-118)
- (modified) llvm/test/CodeGen/AMDGPU/packed-u64.ll (+54-110)
- (modified) llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll (+20-20)
- (modified) llvm/test/CodeGen/AMDGPU/preload-kernargs.ll (+14-24)
- (modified) llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll (+10-19)
- (modified) llvm/test/CodeGen/AMDGPU/rewrite-mfma-single-exit-copy.ll (+382-387)
- (modified) llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll (+5-6)
- (modified) llvm/test/CodeGen/AMDGPU/scratch-simple.ll (+992-961)
- (modified) llvm/test/CodeGen/AMDGPU/shl.v2i64.ll (+82-81)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll (+12-11)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v4f32.ll (+12-20)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v8f32.ll (+16-28)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll (+12-20)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v8i32.ll (+8-12)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll (+12-20)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v8p3.ll (+8-12)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v2f32.ll (+128-95)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v3f32.ll (+97-72)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v4f32.v4f32.ll (+397-271)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v2i32.ll (+128-95)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v3i32.ll (+97-72)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v4i32.ll (+397-271)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v2p3.ll (+128-95)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v3p3.ll (+97-72)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v4p3.ll (+397-271)
- (modified) llvm/test/CodeGen/AMDGPU/smfmac_alloc_failure_no_agpr_O0.ll (+90-111)
- (modified) llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll (+3-3)
- (modified) llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll (+1489-1447)
- (modified) llvm/test/CodeGen/AMDGPU/spill-vgpr.ll (+1-1)
- (modified) llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll (+17-17)
- (modified) llvm/test/CodeGen/AMDGPU/sra.ll (+15-15)
- (modified) llvm/test/CodeGen/AMDGPU/srem.ll (+587-532)
- (modified) llvm/test/CodeGen/AMDGPU/store-local.128.ll (+10-10)
- (modified) llvm/test/CodeGen/AMDGPU/store-local.96.ll (+39-47)
- (modified) llvm/test/CodeGen/AMDGPU/store-to-constant.ll (+2-4)
- (modified) llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll (+10-12)
- (modified) llvm/test/CodeGen/AMDGPU/vector-reduce-fmaximum.ll (+149-151)
- (modified) llvm/test/CodeGen/AMDGPU/vector-reduce-fminimum.ll (+149-151)
- (modified) llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll (+21-21)
- (modified) llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll (+19-19)
- (modified) llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll (+70-62)
- (modified) llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll (+12-2)
``````````diff
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 418ef38daac29..5261b3e9f40fe 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -28268,8 +28268,15 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
V.getOperand(0), NewIndex);
return DAG.getBitcast(NVT, NewExtract);
}
+ // The action for EXTRACT_VECTOR_ELT is registered on the type of the
+ // vector being read, so once operations have been legalized SrcVT is
+ // what decides whether the replacement is really legal. Reading from
+ // an illegal source vector would be promoted straight back into the
+ // extract_subvector we started from.
if (NewExtEC.isScalar() &&
- TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, ScalarVT)) {
+ TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, ScalarVT) &&
+ (!LegalOperations ||
+ TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, SrcVT))) {
SDValue NewIndex = DAG.getVectorIdxConstant(IndexValScaled, DL);
SDValue NewExtract =
DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index a6fa716fd1598..9c18354937801 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -679,6 +679,25 @@ void AMDGPUDAGToDAGISel::SelectVectorShuffle(SDNode *N) {
CurDAG->SelectNodeTo(N, TargetOpcode::REG_SEQUENCE, VT, Ops);
}
+// Extracts that LowerEXTRACT_SUBVECTOR left alone are plain subregister
+// references.
+void AMDGPUDAGToDAGISel::SelectExtractSubvector(SDNode *N) {
+ const auto &TLI =
+ static_cast<const AMDGPUTargetLowering &>(*getTargetLowering());
+
+ EVT VT = N->getValueType(0);
+ SDValue Src = N->getOperand(0);
+ unsigned SubReg = TLI.getExtractSubvectorSubReg(VT, Src.getValueType(),
+ N->getConstantOperandVal(1));
+ if (SubReg == AMDGPU::NoSubRegister) {
+ SelectCode(N);
+ return;
+ }
+
+ ReplaceNode(
+ N, CurDAG->getTargetExtractSubreg(SubReg, SDLoc(N), VT, Src).getNode());
+}
+
void AMDGPUDAGToDAGISel::Select(SDNode *N) {
unsigned int Opc = N->getOpcode();
if (N->isMachineOpcode()) {
@@ -757,6 +776,9 @@ void AMDGPUDAGToDAGISel::Select(SDNode *N) {
case ISD::VECTOR_SHUFFLE:
SelectVectorShuffle(N);
return;
+ case ISD::EXTRACT_SUBVECTOR:
+ SelectExtractSubvector(N);
+ return;
case ISD::BUILD_PAIR: {
SDValue RC, SubReg0, SubReg1;
SDLoc DL(N);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
index 95f85a6151375..2a32f12ac84e1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
@@ -74,6 +74,7 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
protected:
void SelectBuildVector(SDNode *N, unsigned RegClassID);
void SelectVectorShuffle(SDNode *N);
+ void SelectExtractSubvector(SDNode *N);
bool isSDWAOperand(const SDNode *N) const;
private:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 962988ff97e39..0c34aa399d861 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -832,6 +832,11 @@ bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
}
+unsigned AMDGPUTargetLowering::getExtractSubvectorSubReg(EVT ResVT, EVT SrcVT,
+ unsigned Index) const {
+ return AMDGPU::NoSubRegister;
+}
+
bool AMDGPUTargetLowering::shouldReduceLoadWidth(
SDNode *N, ISD::LoadExtType ExtTy, EVT NewVT,
std::optional<unsigned> ByteOffset) const {
@@ -839,6 +844,13 @@ bool AMDGPUTargetLowering::shouldReduceLoadWidth(
if (!TargetLoweringBase::shouldReduceLoadWidth(N, ExtTy, NewVT, ByteOffset))
return false;
+ // WidenOrSplitVectorLoad legalizes a 3 element vector load by loading 4
+ // elements and extracting the low 3 back out. Narrowing a load down to 3
+ // elements would just be undone by that, and the two would keep undoing each
+ // other.
+ if (NewVT.isVector() && NewVT.getVectorNumElements() == 3)
+ return false;
+
unsigned NewSize = NewVT.getStoreSizeInBits();
// If we are reducing to a 32-bit load or a smaller multi-dword load,
@@ -1642,6 +1654,13 @@ SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
EVT VT = Op.getValueType();
EVT SrcVT = Op.getOperand(0).getValueType();
+ // A free extract is a sub register reference, which isel turns into an
+ // EXTRACT_SUBREG. This has to be checked before the 16-bit case below:
+ // rewriting a free extract into a build_vector only gets folded straight back
+ // into the extract by the combiner, and legalizing that rewrites it again.
+ if (getExtractSubvectorSubReg(VT, SrcVT, Start) != AMDGPU::NoSubRegister)
+ return Op;
+
if (VT.getScalarSizeInBits() == 16 && Start % 2 == 0) {
unsigned NumElt = VT.getVectorNumElements();
unsigned NumSrcElt = SrcVT.getVectorNumElements();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index f24f33e7dcb70..c25276580c4ec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -233,6 +233,13 @@ class AMDGPUTargetLowering : public TargetLowering {
bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtType, EVT ExtVT,
std::optional<unsigned> ByteOffset) const override;
+ /// If extracting a subvector of type \p ResVT starting at element \p Index
+ /// out of a vector of type \p SrcVT is a plain sub register reference,
+ /// return the sub register index that describes it. Otherwise return
+ /// AMDGPU::NoSubRegister.
+ virtual unsigned getExtractSubvectorSubReg(EVT ResVT, EVT SrcVT,
+ unsigned Index) const;
+
bool isLoadBitCastBeneficial(EVT, EVT, const SelectionDAG &DAG,
const MachineMemOperand &MMO) const final;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 9159e425dfffd..5ff4720dbc6ee 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1108,6 +1108,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
ISD::SIGN_EXTEND_INREG,
ISD::ANY_EXTEND,
ISD::EXTRACT_VECTOR_ELT,
+ ISD::EXTRACT_SUBVECTOR,
ISD::INSERT_VECTOR_ELT,
ISD::FCOPYSIGN});
@@ -2407,11 +2408,61 @@ bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
return true;
}
+unsigned SITargetLowering::getExtractSubvectorSubReg(EVT ResVT, EVT SrcVT,
+ unsigned Index) const {
+ if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
+ return AMDGPU::NoSubRegister;
+
+ // Both types have to be assigned a register class, otherwise the result
+ // cannot be named as a sub register of the source register.
+ if (!ResVT.isSimple() || !SrcVT.isSimple() || !isTypeLegal(ResVT) ||
+ !isTypeLegal(SrcVT))
+ return AMDGPU::NoSubRegister;
+
+ // A single register wide result is deliberately not described as a sub
+ // register reference: reading one register out of a tuple costs the same
+ // whether it is spelled as an extract_subvector or as an extract_vector_elt,
+ // and keeping the element form lets the combiner see what each element holds.
+ unsigned SizeInBits = ResVT.getSizeInBits();
+ if (SizeInBits <= 32)
+ return AMDGPU::NoSubRegister;
+
+ // Registers are 32 bits wide, so a piece that does not start and end on a
+ // register boundary has to be assembled by real instructions.
+ unsigned OffsetInBits = Index * SrcVT.getScalarSizeInBits();
+ if (SizeInBits % 32 != 0 || OffsetInBits % 32 != 0)
+ return AMDGPU::NoSubRegister;
+
+ unsigned SubReg = SIRegisterInfo::getSubRegFromChannelOrNone(
+ OffsetInBits / 32, SizeInBits / 32);
+ if (SubReg == AMDGPU::NoSubRegister)
+ return AMDGPU::NoSubRegister;
+
+ // Divergence is not known here, so require the sub register to be usable
+ // with the source register class regardless of which register file the
+ // value ends up in.
+ const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
+ for (bool IsDivergent : {false, true}) {
+ const TargetRegisterClass *SrcRC =
+ getRegClassFor(SrcVT.getSimpleVT(), IsDivergent);
+ if (!TRI->getSubClassWithSubReg(SrcRC, SubReg))
+ return AMDGPU::NoSubRegister;
+ }
+
+ return SubReg;
+}
+
bool SITargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
unsigned Index) const {
if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
return false;
+ // A plain sub register reference is free, and isel turns it into an
+ // EXTRACT_SUBREG. Everything else still goes through the elementwise
+ // lowering in AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR.
+ if (getExtractSubvectorSubReg(ResVT, SrcVT, Index) != AMDGPU::NoSubRegister)
+ return true;
+
// TODO: Add more cases that are cheap.
return Index == 0;
}
@@ -16888,6 +16939,33 @@ SITargetLowering::performExtractVectorEltCombine(SDNode *N,
unsigned VecSize = VecVT.getSizeInBits();
unsigned VecEltSize = VecEltVT.getSizeInBits();
+ // extract_vector_elt (extract_subvector X, I), J
+ // -> extract_vector_elt X, I + J
+ //
+ // A free extract is a subregister reference, so reading an element out of it
+ // reads the same register as reading that element out of the vector it came
+ // from. Going straight to the source lets the subvector stop being
+ // referenced, so it no longer needs a register of its own.
+ if (Vec.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
+ SDValue Src = Vec.getOperand(0);
+ EVT SrcVT = Src.getValueType();
+ if (auto *EltIdx = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
+ // The action for EXTRACT_VECTOR_ELT is registered on the type of the
+ // vector being read, so the source is what decides whether reading the
+ // element straight out of it is really legal.
+ if (getExtractSubvectorSubReg(VecVT, SrcVT,
+ Vec.getConstantOperandVal(1)) !=
+ AMDGPU::NoSubRegister &&
+ isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, SrcVT)) {
+ SDValue NewExt = DAG.getExtractVectorElt(SDLoc(N), ResVT, Src,
+ Vec.getConstantOperandVal(1) +
+ EltIdx->getZExtValue());
+ NewExt->setIROrder(Src->getIROrder());
+ return NewExt;
+ }
+ }
+ }
+
if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
allUsesHaveSourceMods(N)) {
SDLoc SL(N);
@@ -17038,6 +17116,38 @@ SITargetLowering::performExtractVectorEltCombine(SDNode *N,
return SDValue();
}
+// extract_subvector (extract_subvector X, I), J -> extract_subvector X, I + J
+//
+// The generic combiner only does this when the inner extract has a single use,
+// because in general each extract it leaves behind is an instruction. A free
+// extract is just a subregister reference, so here it is worth repeating for
+// every use: the intermediate vector stops being referenced and no longer needs
+// a register of its own, which matters most for the wide tuples this reads
+// from.
+SDValue
+SITargetLowering::performExtractSubvectorCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ SDValue Src = N->getOperand(0);
+ if (Src.getOpcode() != ISD::EXTRACT_SUBVECTOR || Src.hasOneUse())
+ return SDValue();
+
+ EVT VT = N->getValueType(0);
+ SDValue Outer = Src.getOperand(0);
+ unsigned Start = Src.getConstantOperandVal(1) + N->getConstantOperandVal(1);
+
+ // The combined index is relative to a wider vector than the one the inner
+ // extract indexed into, and an extract_subvector index has to stay a multiple
+ // of the number of elements it produces.
+ if (Start % VT.getVectorMinNumElements() != 0)
+ return SDValue();
+
+ if (getExtractSubvectorSubReg(VT, Outer.getValueType(), Start) ==
+ AMDGPU::NoSubRegister)
+ return SDValue();
+
+ return DCI.DAG.getExtractSubvector(SDLoc(N), VT, Outer, Start);
+}
+
SDValue
SITargetLowering::performInsertVectorEltCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
@@ -19026,6 +19136,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
}
case ISD::EXTRACT_VECTOR_ELT:
return performExtractVectorEltCombine(N, DCI);
+ case ISD::EXTRACT_SUBVECTOR:
+ return performExtractSubvectorCombine(N, DCI);
case ISD::INSERT_VECTOR_ELT:
return performInsertVectorEltCombine(N, DCI);
case ISD::FP_ROUND:
@@ -19048,22 +19160,45 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
}
-/// Helper function for adjustWritemask
-static unsigned SubIdx2Lane(unsigned Idx) {
- switch (Idx) {
- default:
- return ~0u;
- case AMDGPU::sub0:
- return 0;
- case AMDGPU::sub1:
- return 1;
- case AMDGPU::sub2:
- return 2;
- case AMDGPU::sub3:
- return 3;
- case AMDGPU::sub4:
- return 4; // Possible with TFE/LWE
+/// Helper function for adjustWritemask. Collect the EXTRACT_SUBREG users of
+/// \p Node that read a single 32-bit register into \p Lanes, together with the
+/// lane they read. \p LaneOffset is the lane \p Node itself starts at.
+///
+/// EXTRACT_SUBREGs that name a wider piece are looked through, since they only
+/// rename a range of lanes; the reads of the individual registers are what
+/// decides which channels are live.
+///
+/// \returns false if any use cannot be described as a lane read.
+static bool
+collectMIMGLaneUsers(SDNode *Node, unsigned LaneOffset,
+ const SIRegisterInfo &TRI,
+ SmallVectorImpl<std::pair<SDNode *, unsigned>> &Lanes) {
+ for (SDUse &Use : Node->uses()) {
+ // Don't look at users of the chain.
+ if (Use.getResNo() != 0)
+ continue;
+
+ SDNode *User = Use.getUser();
+ if (!User->isMachineOpcode() ||
+ User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
+ return false;
+
+ unsigned SubReg = User->getConstantOperandVal(1);
+ unsigned Offset = TRI.getSubRegIdxOffset(SubReg);
+ unsigned Size = TRI.getSubRegIdxSize(SubReg);
+ if (Offset % 32 != 0 || Size % 32 != 0)
+ return false;
+
+ if (Size == 32) {
+ Lanes.emplace_back(User, LaneOffset + Offset / 32);
+ continue;
+ }
+
+ if (!collectMIMGLaneUsers(User, LaneOffset + Offset / 32, TRI, Lanes))
+ return false;
}
+
+ return true;
}
/// Adjust the writemask of MIMG, VIMAGE or VSAMPLE instructions
@@ -19101,25 +19236,17 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
}
// Try to figure out the used register components
- for (SDUse &Use : Node->uses()) {
-
- // Don't look at users of the chain.
- if (Use.getResNo() != 0)
- continue;
-
- SDNode *User = Use.getUser();
-
- // Abort if we can't understand the usage
- if (!User->isMachineOpcode() ||
- User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
- return Node;
+ SmallVector<std::pair<SDNode *, unsigned>, 8> LaneUses;
+ if (!collectMIMGLaneUsers(Node, 0, *Subtarget->getRegisterInfo(), LaneUses))
+ return Node;
+ for (auto [User, UserLane] : LaneUses) {
// Lane means which subreg of %vgpra_vgprb_vgprc_vgprd is used.
// Note that subregs are packed, i.e. Lane==0 is the first bit set
// in OldDmask, so it can be any of X,Y,Z,W; Lane==1 is the second bit
// set, etc.
- Lane = SubIdx2Lane(User->getConstantOperandVal(1));
- if (Lane == ~0u)
+ Lane = UserLane;
+ if (Lane >= std::size(Users))
return Node;
// Check if the use is for the TFE/LWE generated result at VGPRn+1.
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index f358f59311c79..8413d74dc823a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -234,6 +234,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
SDValue performFMed3Combine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performCvtPkRTZCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performExtractVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+ SDValue performExtractSubvectorCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performInsertVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performFPRoundCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
@@ -405,6 +406,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
Type *Ty) const override;
+ unsigned getExtractSubvectorSubReg(EVT ResVT, EVT SrcVT,
+ unsigned Index) const override;
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
unsigned Index) const override;
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override;
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index a3d6509628d50..8462bdd69303f 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -578,6 +578,17 @@ unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel,
return SubRegFromChannelTable[NumRegIndex - 1][Channel];
}
+unsigned SIRegisterInfo::getSubRegFromChannelOrNone(unsigned Channel,
+ unsigned NumRegs) {
+ if (NumRegs >= SubRegFromChannelTableWidthMap.size())
+ return AMDGPU::NoSubRegister;
+ unsigned NumRegIndex = SubRegFromChannelTableWidthMap[NumRegs];
+ if (NumRegIndex == 0 ||
+ Channel >= SubRegFromChannelTable[NumRegIndex - 1].size())
+ return AMDGPU::NoSubRegister;
+ return SubRegFromChannelTable[NumRegIndex - 1][Channel];
+}
+
bool SIRegisterInfo::isCFISavedRegsSpillEnabled() const {
return EnableSpillCFISavedRegs;
}
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index e464c9334ffea..4502bfc7d4509 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -76,6 +76,12 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
/// (e.g. getSubRegFromChannel(0) -> AMDGPU::sub0)
static unsigned getSubRegFromChannel(unsigned Channel, unsigned NumRegs = 1);
+ /// Same as getSubRegFromChannel, but returns AMDGPU::NoSubRegister instead of
+ /// asserting when \p NumRegs registers starting at \p Channel are not
+ /// describable by a single sub register index.
+ static unsigned getSubRegFromChannelOrNone(unsigned Channel,
+ unsigned NumRegs = 1);
+
bool spillSGPRToVGPR() const {
return SpillSGPRToVGPR;
}
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-csr.ll b/llvm/test/CodeGen/AMDGPU/agpr-csr.ll
index 016ac4eeb8867..78549067ff5e7 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-csr.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-csr.ll
@@ -180,55 +180,63 @@ define amdgpu_kernel void @test_call_empty() #0 {
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def a[0:31]
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a6
-; GFX908-NEXT: v_accvgpr_read_b32 ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/214152
More information about the llvm-commits
mailing list