[llvm] [AMDGPU] Fix unaligned SMEM dword immediate offsets (PR #216269)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 01:00:22 PDT 2026
https://github.com/YanLiu0942 created https://github.com/llvm/llvm-project/pull/216269
GFX11+ requires immediate byte offsets used by S_LOAD_DWORD* instructions to be dword aligned.
SelectionDAG could split an aligned address into an unaligned base and a non-dword-aligned immediate offset. This occurs when an aligned f16 load is widened to an i32 SMEM load, for example with a 22-byte vector stride and a 2-byte component offset.
Keep scalar sub-dword load patterns and pre-GFX11 targets unchanged. Add GFX11 and GFX12 regression coverage for the widened f16 load case.
>From 026bcbb576a812dd14ac71f38ab1d244b6d76070 Mon Sep 17 00:00:00 2001
From: Yan Liu <YanLiu0942 at gmail.com>
Date: Fri, 14 Aug 2026 15:13:03 +0800
Subject: [PATCH] [AMDGPU] Fix unaligned SMEM dword immediate offsets
GFX11+ requires immediate byte offsets used by S_LOAD_DWORD* instructions
to be dword aligned.
SelectionDAG could split an aligned address into an unaligned base and a
non-dword-aligned immediate offset. This occurs when an aligned f16 load
is widened to an i32 SMEM load, for example with a 22-byte vector stride
and a 2-byte component offset.
Keep scalar sub-dword load patterns and pre-GFX11 targets unchanged.
Add GFX11 and GFX12 regression coverage for the widened f16 load case.
Signed-off-by: Yan Liu <YanLiu0942 at gmail.com>
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 21 +++++++
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h | 1 +
llvm/lib/Target/AMDGPU/SMInstructions.td | 7 ++-
.../AMDGPU/smrd-dword-unaligned-imm-offset.ll | 56 +++++++++++++++++++
4 files changed, 82 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/smrd-dword-unaligned-imm-offset.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 40345819a3f8d..99732b347ddeb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -2636,6 +2636,27 @@ bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
&Offset);
}
+bool AMDGPUDAGToDAGISel::SelectSMRDDwordImm(SDValue Addr, SDValue &SBase,
+ SDValue &Offset) const {
+ if (!SelectSMRD(/* N */ nullptr, Addr, SBase, /* SOffset */ nullptr,
+ &Offset))
+ return false;
+
+ // This selector is used for S_LOAD_DWORD* instructions. On GFX11+, their
+ // immediate byte offset must be dword aligned. If an aligned address is split
+ // into an unaligned base and a non-dword-aligned immediate (for example,
+ // 22 + 2), encoding them separately loses the carry from the low address bits.
+ // Materialize the complete address in sbase and use an immediate offset of 0.
+ auto *ConstantOffset = cast<ConstantSDNode>(Offset);
+ if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX11 &&
+ (ConstantOffset->getSExtValue() & 3) != 0) {
+ SBase = Expand32BitAddress(Addr);
+ Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
+ }
+
+ return true;
+}
+
bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
SDValue &Offset) const {
assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
index 3b7e76a508d1b..0638fc6affbc7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
@@ -205,6 +205,7 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
SDValue *Offset, bool Imm32Only = false,
bool *ScaleOffset = nullptr) const;
bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
+ bool SelectSMRDDwordImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
bool SelectScaleOffset(SDNode *N, SDValue &Offset, bool IsSigned) const;
bool SelectSMRDSgpr(SDNode *N, SDValue Addr, SDValue &SBase, SDValue &SOffset,
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index 19aeafe9b30cc..b89c9ab8a4729 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -861,8 +861,9 @@ def smrd_prefetch : PatFrag <(ops node:$ptr, node:$rw, node:$loc, node:$type),
}];
}
-def SMRDImm : ComplexPattern<iPTR, 2, "SelectSMRDImm">;
-def SMRDImm32 : ComplexPattern<iPTR, 2, "SelectSMRDImm32">;
+def SMRDImm : ComplexPattern<iPTR, 2, "SelectSMRDImm">;
+def SMRDDwordImm : ComplexPattern<iPTR, 2, "SelectSMRDDwordImm">;
+def SMRDImm32 : ComplexPattern<iPTR, 2, "SelectSMRDImm32">;
let WantsRoot = true in {
def SMRDSgpr : ComplexPattern<iPTR, 3, "SelectSMRDSgpr", [], [], -3>;
def SMRDSgprImm : ComplexPattern<iPTR, 4, "SelectSMRDSgprImm", [], []>;
@@ -890,7 +891,7 @@ multiclass SMRD_Patterns <string Instr, ValueType vt, PatFrag frag,
bit immci = true, string suffix = ""> {
// 1. IMM offset
def : GCNPat <
- (frag (SMRDImm i64:$sbase, i32:$offset)),
+ (frag (SMRDDwordImm i64:$sbase, i32:$offset)),
(vt (!cast<SM_Pseudo>(Instr#"_IMM"#suffix) $sbase, $offset, 0))>;
// 2. 32-bit IMM offset on CI
diff --git a/llvm/test/CodeGen/AMDGPU/smrd-dword-unaligned-imm-offset.ll b/llvm/test/CodeGen/AMDGPU/smrd-dword-unaligned-imm-offset.ll
new file mode 100644
index 0000000000000..ca726bee5f36b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/smrd-dword-unaligned-imm-offset.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -disable-separate-const-offset-from-gep=1 < %s | FileCheck --check-prefix=GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -disable-separate-const-offset-from-gep=1 < %s | FileCheck --check-prefix=GFX12 %s
+
+; Model one unrolled f16 component load from an 11-component vector. The vector
+; stride is 22 bytes and the fixed component offset is 2 bytes. Because the load
+; is known to be dword aligned, widenLoad promotes it to an i32 SMEM load.
+; GFX11+ must keep the complete address in sbase rather than encode the 2-byte
+; displacement as an immediate offset of s_load_b32.
+define amdgpu_kernel void @widen_f16_load_with_unaligned_imm_offset(
+; GFX11-LABEL: widen_f16_load_with_unaligned_imm_offset:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x10
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mul_i32 s6, s6, 22
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_ashr_i32 s4, s6, 31
+; GFX11-NEXT: s_add_u32 s2, s2, s6
+; GFX11-NEXT: s_addc_u32 s3, s3, s4
+; GFX11-NEXT: s_add_u32 s2, s2, 2
+; GFX11-NEXT: s_addc_u32 s3, s3, 0
+; GFX11-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: widen_f16_load_with_unaligned_imm_offset:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x10
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mul_i32 s4, s6, 22
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_ashr_i32 s5, s4, 31
+; GFX12-NEXT: s_add_nc_u64 s[2:3], s[2:3], s[4:5]
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_add_nc_u64 s[2:3], s[2:3], 2
+; GFX12-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX12-NEXT: s_endpgm
+ ptr addrspace(1) inreg %out, ptr addrspace(4) inreg %base,
+ i32 inreg %index) {
+ %dynamic.offset = mul i32 %index, 22
+ %vector.base = getelementptr i8, ptr addrspace(4) %base, i32 %dynamic.offset
+ %component.ptr = getelementptr i8, ptr addrspace(4) %vector.base, i32 2
+ %value = load half, ptr addrspace(4) %component.ptr, align 4
+ %bits = bitcast half %value to i16
+ store i16 %bits, ptr addrspace(1) %out, align 2
+ ret void
+}
More information about the llvm-commits
mailing list