[llvm] 31ef738 - [AMDGPU][GFX1250] Add cpol support for some prefetch instructions (#208368)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 9 06:19:42 PDT 2026
Author: Shilei Tian
Date: 2026-07-09T09:19:37-04:00
New Revision: 31ef7389463b1121ceecced9d8cf68e0b515bc41
URL: https://github.com/llvm/llvm-project/commit/31ef7389463b1121ceecced9d8cf68e0b515bc41
DIFF: https://github.com/llvm/llvm-project/commit/31ef7389463b1121ceecced9d8cf68e0b515bc41.diff
LOG: [AMDGPU][GFX1250] Add cpol support for some prefetch instructions (#208368)
Fixes ROCM-27516.
Added:
Modified:
llvm/lib/Target/AMDGPU/SMInstructions.td
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
llvm/test/MC/AMDGPU/gfx1250_asm_smem.s
llvm/test/MC/AMDGPU/gfx1250_asm_smem_err.s
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index 8e86d477f1926..19aeafe9b30cc 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -110,8 +110,8 @@ def SGPR_IMM_OptOffset : OffsetMode<1, 1, "_SGPR_IMM",
class SM_Probe_Pseudo <string opName, RegisterClass baseClass, OffsetMode offsets>
: SM_Pseudo<opName, (outs),
- !con((ins i8imm:$sdata, baseClass:$sbase), offsets.Ins),
- " $sdata, $sbase, " # offsets.Asm> {
+ !con((ins i8imm:$sdata, baseClass:$sbase), offsets.Ins, (ins CPol_0:$cpol)),
+ " $sdata, $sbase, " # offsets.Asm # "$cpol"> {
let mayLoad = 0;
let mayStore = 0;
let has_glc = 0;
@@ -239,8 +239,8 @@ class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo<
class SM_Prefetch_Pseudo <string opName, RegisterClass baseClass, bit hasSBase>
: SM_Pseudo<opName, (outs), !con(!if(hasSBase, (ins baseClass:$sbase), (ins)),
- (ins SMEMOffset:$offset, SReg_32:$soffset, i8imm:$sdata)),
- !if(hasSBase, " $sbase,", "") # " $offset, $soffset, $sdata"> {
+ (ins SMEMOffset:$offset, SReg_32:$soffset, i8imm:$sdata, CPol_0:$cpol)),
+ !if(hasSBase, " $sbase,", "") # " $offset, $soffset, $sdata$cpol"> {
// Mark prefetches as both load and store to prevent reordering with loads
// and stores. This is also needed for pattern to match prefetch intrinsic.
let mayLoad = 1;
@@ -1134,19 +1134,19 @@ def i32imm_one : TImmLeaf <i32, [{
multiclass SMPrefetchPat<string type, TImmLeaf cache_type> {
def : GCNPat <
(smrd_prefetch (SMRDImm i64:$sbase, i32:$offset), timm, timm, cache_type),
- (!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type) $sbase, $offset, (i32 SGPR_NULL), (i8 0))
+ (!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type) $sbase, $offset, (i32 SGPR_NULL), (i8 0), 0)
>;
def : GCNPat <
(smrd_prefetch (i64 SReg_64:$sbase), timm, timm, cache_type),
- (!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type) $sbase, 0, (i32 SGPR_NULL), (i8 0))
+ (!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type) $sbase, 0, (i32 SGPR_NULL), (i8 0), 0)
>;
def : GCNPat <
(smrd_prefetch (i32 SReg_32:$sbase), timm, timm, cache_type),
(!cast<SM_Prefetch_Pseudo>("S_PREFETCH_"#type)
(i64 (REG_SEQUENCE SReg_64, $sbase, sub0, (i32 (S_MOV_B32 (i32 0))), sub1)),
- 0, (i32 SGPR_NULL), (i8 0))
+ 0, (i32 SGPR_NULL), (i8 0), 0)
>;
}
@@ -1157,53 +1157,53 @@ defm : SMPrefetchPat<"DATA", i32imm_one>;
let SubtargetPredicate = HasSmemPrefetchInsts in {
def : GCNPat <
(int_amdgcn_s_prefetch_data (SMRDImm i64:$sbase, i32:$offset), (i32 SReg_32:$len)),
- (S_PREFETCH_DATA $sbase, $offset, $len, 0)
+ (S_PREFETCH_DATA $sbase, $offset, $len, 0, 0)
>;
def : GCNPat <
(int_amdgcn_s_prefetch_data (i64 SReg_64:$sbase), (i32 SReg_32:$len)),
- (S_PREFETCH_DATA $sbase, 0, $len, 0)
+ (S_PREFETCH_DATA $sbase, 0, $len, 0, 0)
>;
def : GCNPat <
(int_amdgcn_s_prefetch_data (SMRDImm i64:$sbase, i32:$offset), imm:$len),
- (S_PREFETCH_DATA $sbase, $offset, (i32 SGPR_NULL), (as_i8timm $len))
+ (S_PREFETCH_DATA $sbase, $offset, (i32 SGPR_NULL), (as_i8timm $len), 0)
>;
def : GCNPat <
(int_amdgcn_s_prefetch_data (i64 SReg_64:$sbase), imm:$len),
- (S_PREFETCH_DATA $sbase, 0, (i32 SGPR_NULL), (as_i8timm $len))
+ (S_PREFETCH_DATA $sbase, 0, (i32 SGPR_NULL), (as_i8timm $len), 0)
>;
def : GCNPat <
(SIsbuffer_prefetch v4i32:$sbase, (SMRDBufferImm i32:$offset), (i32 SReg_32:$len)),
- (S_BUFFER_PREFETCH_DATA SReg_128:$sbase, i32imm:$offset, $len, 0)
+ (S_BUFFER_PREFETCH_DATA SReg_128:$sbase, i32imm:$offset, $len, 0, 0)
>;
def : GCNPat <
(SIsbuffer_prefetch v4i32:$sbase, (SMRDBufferImm i32:$offset), imm:$len),
- (S_BUFFER_PREFETCH_DATA SReg_128:$sbase, i32imm:$offset, (i32 SGPR_NULL), (as_i8timm $len))
+ (S_BUFFER_PREFETCH_DATA SReg_128:$sbase, i32imm:$offset, (i32 SGPR_NULL), (as_i8timm $len), 0)
>;
def : GCNPat <
(int_amdgcn_s_prefetch_inst (SMRDImm i64:$sbase, i32:$offset), (i32 SReg_32:$len)),
- (S_PREFETCH_INST $sbase, $offset, $len, 0)
+ (S_PREFETCH_INST $sbase, $offset, $len, 0, 0)
>;
def : GCNPat <
(int_amdgcn_s_prefetch_inst (i64 SReg_64:$sbase), (i32 SReg_32:$len)),
- (S_PREFETCH_INST $sbase, 0, $len, 0)
+ (S_PREFETCH_INST $sbase, 0, $len, 0, 0)
>;
def : GCNPat <
(int_amdgcn_s_prefetch_inst (SMRDImm i64:$sbase, i32:$offset), imm:$len),
- (S_PREFETCH_INST $sbase, $offset, (i32 SGPR_NULL), (as_i8timm $len))
+ (S_PREFETCH_INST $sbase, $offset, (i32 SGPR_NULL), (as_i8timm $len), 0)
>;
def : GCNPat <
(int_amdgcn_s_prefetch_inst (i64 SReg_64:$sbase), imm:$len),
- (S_PREFETCH_INST $sbase, 0, (i32 SGPR_NULL), (as_i8timm $len))
+ (S_PREFETCH_INST $sbase, 0, (i32 SGPR_NULL), (as_i8timm $len), 0)
>;
} // End let SubtargetPredicate = HasSmemPrefetchInsts
@@ -1499,6 +1499,9 @@ class SMEM_Real_Prefetch_gfx12<bits<6> op, SM_Pseudo ps> :
let sdst = ?;
let Inst{12-11} = 0; // Unused sdata bits.
let Inst{10-6} = !if(ps.has_sdst, sdata{4-0}, ?);
+ let Inst{20} = cpol{CPolBit.NV}; // non-volatile
+ let Inst{22-21} = cpol{4-3}; // scope
+ let Inst{24-23} = cpol{1-0}; // th - only lower 2 bits are supported
}
class SMEM_Real_Load_gfx12<bits<6> op, string ps, string opName, OffsetMode offsets> :
@@ -1580,6 +1583,9 @@ class SMEM_Real_Prefetch_gfx13<bits<6> op, SM_Pseudo ps> :
let sdst = ?;
let Inst{12-11} = 0; // Unused sdata bits.
let Inst{10-6} = !if(ps.has_sdst, sdata{4-0}, ?);
+ let Inst{20} = cpol{CPolBit.NV}; // non-volatile
+ let Inst{22-21} = cpol{4-3}; // scope
+ let Inst{24-23} = cpol{1-0}; // th - only lower 2 bits are supported
}
class SMEM_Real_Load_gfx13<bits<6> op, string ps, string opName, OffsetMode offsets> :
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 3ae1ae40ca34a..71d95a23e30d3 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -3674,10 +3674,11 @@ getVGPRLoweringOperandTables(const MCInstrDesc &Desc) {
}
bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode) {
- uint64_t TSFlags = MII.get(Opcode).TSFlags;
+ const MCInstrDesc &Desc = MII.get(Opcode);
+ uint64_t TSFlags = Desc.TSFlags;
if (TSFlags & SIInstrFlags::SMRD)
- return !getSMEMIsBuffer(Opcode);
+ return Desc.mayLoad() && !Desc.mayStore() && !getSMEMIsBuffer(Opcode);
if (!(TSFlags & SIInstrFlags::FLAT))
return false;
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_smem.s b/llvm/test/MC/AMDGPU/gfx1250_asm_smem.s
index ddadc80cea6b3..917edeb993d9d 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_smem.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_smem.s
@@ -1,4 +1,5 @@
// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | %extract-encodings | llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding | FileCheck --check-prefix=GFX1250 %s
// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -filetype=null %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
s_load_b32 s4, s[2:3], 10 nv
@@ -13,6 +14,48 @@ s_buffer_load_i8 s5, s[4:7], s0 nv
// GFX12-ERR-NEXT:{{^}}s_buffer_load_i8 s5, s[4:7], s0 nv
// GFX12-ERR-NEXT:{{^}} ^
+s_prefetch_inst s[12:13], 16, s4, 2 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX1250: s_prefetch_inst s[12:13], 0x10, s4, 2 th:TH_LOAD_HT scope:SCOPE_DEV nv ; encoding: [0x86,0x80,0x54,0xf5,0x10,0x00,0x00,0x08]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: nv is not supported on this GPU
+// GFX12-ERR-NEXT:{{^}}s_prefetch_inst s[12:13], 16, s4, 2 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX12-ERR-NEXT:{{^}} ^
+
+s_prefetch_inst_pc_rel 100, s10, 7 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX1250: s_prefetch_inst_pc_rel 0x64, s10, 7 th:TH_LOAD_HT scope:SCOPE_DEV nv ; encoding: [0xc0,0xa1,0x54,0xf5,0x64,0x00,0x00,0x14]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: nv is not supported on this GPU
+// GFX12-ERR-NEXT:{{^}}s_prefetch_inst_pc_rel 100, s10, 7 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX12-ERR-NEXT:{{^}} ^
+
+s_prefetch_data s[18:19], 100, s10, 7 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX1250: s_prefetch_data s[18:19], 0x64, s10, 7 th:TH_LOAD_HT scope:SCOPE_DEV nv ; encoding: [0xc9,0xc1,0x54,0xf5,0x64,0x00,0x00,0x14]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: nv is not supported on this GPU
+// GFX12-ERR-NEXT:{{^}}s_prefetch_data s[18:19], 100, s10, 7 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX12-ERR-NEXT:{{^}} ^
+
+s_prefetch_data_pc_rel 100, s10, 7 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX1250: s_prefetch_data_pc_rel 0x64, s10, 7 th:TH_LOAD_HT scope:SCOPE_DEV nv ; encoding: [0xc0,0x01,0x55,0xf5,0x64,0x00,0x00,0x14]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: nv is not supported on this GPU
+// GFX12-ERR-NEXT:{{^}}s_prefetch_data_pc_rel 100, s10, 7 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX12-ERR-NEXT:{{^}} ^
+
+s_buffer_prefetch_data s[8:11], 100, s10, 7 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX1250: s_buffer_prefetch_data s[8:11], 0x64, s10, 7 th:TH_LOAD_HT scope:SCOPE_DEV nv ; encoding: [0xc4,0xe1,0x54,0xf5,0x64,0x00,0x00,0x14]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: nv is not supported on this GPU
+// GFX12-ERR-NEXT:{{^}}s_buffer_prefetch_data s[8:11], 100, s10, 7 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX12-ERR-NEXT:{{^}} ^
+
+s_atc_probe 7, s[4:5], s0 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX1250: s_atc_probe 7, s[4:5], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_DEV nv ; encoding: [0xc2,0x41,0x54,0xf5,0x00,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: nv is not supported on this GPU
+// GFX12-ERR-NEXT:{{^}}s_atc_probe 7, s[4:5], s0 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX12-ERR-NEXT:{{^}} ^
+
+s_atc_probe_buffer 1, s[8:11], s0 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX1250: s_atc_probe_buffer 1, s[8:11], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_DEV nv ; encoding: [0x44,0x60,0x54,0xf5,0x00,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: nv is not supported on this GPU
+// GFX12-ERR-NEXT:{{^}}s_atc_probe_buffer 1, s[8:11], s0 th:TH_LOAD_HT scope:SCOPE_DEV nv
+// GFX12-ERR-NEXT:{{^}} ^
+
s_load_b32 s4, s[2:3], 0xa scale_offset
// GFX1250: s_load_b32 s4, s[2:3], 0xa scale_offset ; encoding: [0x01,0x01,0x00,0xf4,0x0a,0x00,0x00,0xf9]
// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: scale_offset is not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_smem_err.s b/llvm/test/MC/AMDGPU/gfx1250_asm_smem_err.s
index aaa89a5cd52cb..ad4019080737c 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_smem_err.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_smem_err.s
@@ -5,12 +5,7 @@ s_buffer_load_i8 s5, s[4:7], s0 scale_offset
// GFX1250-ERR-NEXT:{{^}}s_buffer_load_i8 s5, s[4:7], s0 scale_offset
// GFX1250-ERR-NEXT:{{^}} ^
-s_prefetch_data s[18:19], 100, s10, 7 nv
-// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
-// GFX1250-ERR-NEXT:{{^}}s_prefetch_data s[18:19], 100, s10, 7 nv
-// GFX1250-ERR-NEXT:{{^}} ^
-
s_prefetch_data s[18:19], 100, s10, 7 scale_offset
-// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: scale_offset is not supported for this instruction
// GFX1250-ERR-NEXT:{{^}}s_prefetch_data s[18:19], 100, s10, 7 scale_offset
// GFX1250-ERR-NEXT:{{^}} ^
More information about the llvm-commits
mailing list