[llvm] 275ecaa - [AMDGPU] Cluster MIMG instructions
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 8 06:02:09 PDT 2020
Author: Jay Foad
Date: 2020-06-08T14:01:53+01:00
New Revision: 275ecaae1653dc03d95958add6ed0bfee0582d4f
URL: https://github.com/llvm/llvm-project/commit/275ecaae1653dc03d95958add6ed0bfee0582d4f
DIFF: https://github.com/llvm/llvm-project/commit/275ecaae1653dc03d95958add6ed0bfee0582d4f.diff
LOG: [AMDGPU] Cluster MIMG instructions
Differential Revision: https://reviews.llvm.org/D74035
Added:
Modified:
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
llvm/test/CodeGen/AMDGPU/cluster_stores.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 2c385a2c1b2f..01ffcfaa640d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -382,6 +382,21 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
return true;
}
+ if (isMIMG(LdSt)) {
+ int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
+ BaseOps.push_back(&LdSt.getOperand(SRsrcIdx));
+ int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
+ if (VAddr0Idx >= 0) {
+ // GFX10 possible NSA encoding.
+ for (int I = VAddr0Idx; I < SRsrcIdx; ++I)
+ BaseOps.push_back(&LdSt.getOperand(I));
+ } else {
+ BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr));
+ }
+ Offset = 0;
+ return true;
+ }
+
if (isSMRD(LdSt)) {
BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
if (!BaseOp) // e.g. S_MEMTIME
@@ -415,22 +430,14 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
return false;
}
-static bool
-memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,
- ArrayRef<const MachineOperand *> BaseOps2) {
- if (BaseOps1.size() != BaseOps2.size())
- return false;
- for (size_t I = 0, E = BaseOps1.size(); I < E; ++I)
- if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I]))
- return false;
- return true;
-}
-
static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
ArrayRef<const MachineOperand *> BaseOps1,
const MachineInstr &MI2,
ArrayRef<const MachineOperand *> BaseOps2) {
- if (memOpsHaveSameBaseOperands(BaseOps1, BaseOps2))
+ // Only examine the first "base" operand of each instruction, on the
+ // assumption that it represents the real base address of the memory access.
+ // Other operands are typically offsets or indices from this base address.
+ if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front()))
return true;
if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
@@ -472,6 +479,7 @@ bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
(isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
+ (isMIMG(FirstLdSt) && isMIMG(SecondLdSt)) ||
(isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
const unsigned MaxGlobalLoadCluster = 7;
if (NumLoads > MaxGlobalLoadCluster)
@@ -2747,6 +2755,18 @@ bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
return false;
}
+static bool
+memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,
+ ArrayRef<const MachineOperand *> BaseOps2) {
+ if (BaseOps1.size() != BaseOps2.size())
+ return false;
+ for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) {
+ if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I]))
+ return false;
+ }
+ return true;
+}
+
static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
int WidthB, int OffsetB) {
int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
diff --git a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
index 466d69002af5..67d6f686f459 100644
--- a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
+++ b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -debug-only=machine-scheduler -o /dev/null %s 2>&1 | FileCheck --enable-var-scope --check-prefixes=CHECK,DBG %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --enable-var-scope --check-prefixes=CHECK,GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -debug-only=machine-scheduler < %s 2> %t | FileCheck --enable-var-scope --check-prefixes=CHECK,GCN %s
+; RUN: FileCheck --enable-var-scope --check-prefixes=CHECK,DBG %s < %t
; REQUIRES: asserts
; CHECK-LABEL: {{^}}cluster_load_cluster_store:
@@ -83,3 +83,61 @@ bb:
ret void
}
+
+; Cluster loads from the same texture with
diff erent coordinates
+; CHECK-LABEL: {{^}}cluster_image_load:
+; DBG: {{^}}Cluster ld/st [[SU1:SU\([0-9]+\)]] - [[SU2:SU\([0-9]+\)]]
+; DBG: {{^}}[[SU1]]: {{.*}} IMAGE_LOAD
+; DBG: {{^}}[[SU2]]: {{.*}} IMAGE_LOAD
+; GCN: image_load v
+; GCN-NEXT: image_load v
+define amdgpu_ps void @cluster_image_load(<8 x i32> inreg %src, <8 x i32> inreg %dst, i32 %x, i32 %y) {
+entry:
+ %x1 = add i32 %x, 1
+ %y1 = add i32 %y, 1
+ %val1 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x1, i32 %y1, i32 0, <8 x i32> %src, i32 0, i32 0)
+ %x2 = add i32 %x, 2
+ %y2 = add i32 %y, 2
+ %val2 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x2, i32 %y2, i32 0, <8 x i32> %src, i32 0, i32 0)
+ %val = fadd fast <4 x float> %val1, %val2
+ call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %val, i32 15, i32 %x, i32 %y, <8 x i32> %dst, i32 0, i32 0)
+ ret void
+}
+
+; Don't cluster loads from
diff erent textures
+; CHECK-LABEL: {{^}}no_cluster_image_load:
+; DBG-NOT: {{^}}Cluster ld/st
+define amdgpu_ps void @no_cluster_image_load(<8 x i32> inreg %src1, <8 x i32> inreg %src2, <8 x i32> inreg %dst, i32 %x, i32 %y) {
+entry:
+ %val1 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x, i32 %y, i32 0, <8 x i32> %src1, i32 0, i32 0)
+ %val2 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x, i32 %y, i32 0, <8 x i32> %src2, i32 0, i32 0)
+ %val = fadd fast <4 x float> %val1, %val2
+ call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %val, i32 15, i32 %x, i32 %y, <8 x i32> %dst, i32 0, i32 0)
+ ret void
+}
+
+; Cluster loads from the same texture and sampler with
diff erent coordinates
+; CHECK-LABEL: {{^}}cluster_image_sample:
+; DBG: {{^}}Cluster ld/st [[SU1:SU\([0-9]+\)]] - [[SU2:SU\([0-9]+\)]]
+; DBG: {{^}}[[SU1]]: {{.*}} IMAGE_SAMPLE
+; DBG: {{^}}[[SU2]]: {{.*}} IMAGE_SAMPLE
+; GCN: image_sample_d
+; GCN-NEXT: image_sample_d
+define amdgpu_ps void @cluster_image_sample(<8 x i32> inreg %src, <4 x i32> inreg %smp, <8 x i32> inreg %dst, i32 %x, i32 %y) {
+entry:
+ %s = sitofp i32 %x to float
+ %t = sitofp i32 %y to float
+ %s1 = fadd float %s, 1.0
+ %t1 = fadd float %t, 1.0
+ %val1 = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32(i32 15, float %s1, float %t1, float 0.0, float 0.0, float 0.0, float 0.0, <8 x i32> %src, <4 x i32> %smp, i1 false, i32 0, i32 0)
+ %s2 = fadd float %s, 2.0
+ %t2 = fadd float %t, 2.0
+ %val2 = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32(i32 15, float %s2, float %t2, float 1.0, float 1.0, float 1.0, float 1.0, <8 x i32> %src, <4 x i32> %smp, i1 false, i32 0, i32 0)
+ %val = fadd fast <4 x float> %val1, %val2
+ call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %val, i32 15, i32 %x, i32 %y, <8 x i32> %dst, i32 0, i32 0)
+ ret void
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 immarg, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg)
+declare <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32(i32, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32)
+declare void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg)
More information about the llvm-commits
mailing list