[llvm] cddc107 - [AMDGPU] Do not widen constant loads that may not be dereferenceable (#184790)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 12:48:23 PDT 2026
Author: Arseniy Obolenskiy
Date: 2026-08-04T21:48:18+02:00
New Revision: cddc107899bdf09ee3ffd60b00b32455e699b130
URL: https://github.com/llvm/llvm-project/commit/cddc107899bdf09ee3ffd60b00b32455e699b130
DIFF: https://github.com/llvm/llvm-project/commit/cddc107899bdf09ee3ffd60b00b32455e699b130.diff
LOG: [AMDGPU] Do not widen constant loads that may not be dereferenceable (#184790)
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
llvm/test/CodeGen/AMDGPU/amdgpu-late-codegenprepare.ll
llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
index 54bc95653b314..be0b221456077 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
@@ -16,6 +16,7 @@
#include "AMDGPUMemoryUtils.h"
#include "AMDGPUTargetMachine.h"
#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/TargetPassConfig.h"
@@ -24,7 +25,6 @@
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/InitializePasses.h"
#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/KnownBits.h"
#include "llvm/Transforms/Utils/Local.h"
#define DEBUG_TYPE "amdgpu-late-codegenprepare"
@@ -61,10 +61,15 @@ class AMDGPULateCodeGenPrepare
bool run();
bool visitInstruction(Instruction &) { return false; }
- // Check if the specified value is at least DWORD aligned.
- bool isDWORDAligned(const Value *V) const {
- KnownBits Known = computeKnownBits(V, DL, AC);
- return Known.countMinTrailingZeros() >= 2;
+ // Widening may read padding bytes past the original access, so require the
+ // whole AccessSize-byte range at Base to be dereferenceable, not just Base
+ // itself aligned.
+ bool isSafeToWidenLoad(const Value *Base, uint64_t AccessSize,
+ const Instruction *CxtI) const {
+ return isDereferenceableAndAlignedPointer(
+ Base, Align(4),
+ APInt(DL.getIndexTypeSizeInBits(Base->getType()), AccessSize),
+ SimplifyQuery(DL, /*TLI=*/nullptr, /*DT=*/nullptr, AC, CxtI));
}
bool canWidenScalarExtLoad(LoadInst &LI) const;
@@ -512,18 +517,11 @@ bool AMDGPULateCodeGenPrepare::visitLoadInst(LoadInst &LI) {
int64_t Offset = 0;
auto *Base =
GetPointerBaseWithConstantOffset(LI.getPointerOperand(), Offset, DL);
- // If that base is not DWORD aligned, it's not safe to perform the following
- // transforms.
- if (!isDWORDAligned(Base))
- return false;
int64_t Adjust = Offset & 0x3;
- if (Adjust == 0) {
- // With a zero adjust, the original alignment could be promoted with a
- // better one.
- LI.setAlignment(Align(4));
- return true;
- }
+ int64_t AccessOffset = Offset - Adjust;
+ if (AccessOffset < 0 || !isSafeToWidenLoad(Base, AccessOffset + 4, &LI))
+ return false;
IRBuilder<> IRB(&LI);
IRB.SetCurrentDebugLocation(LI.getDebugLoc());
@@ -540,10 +538,11 @@ bool AMDGPULateCodeGenPrepare::visitLoadInst(LoadInst &LI) {
AMDGPU::copyMetadataForWidenedLoad(*NewLd, LI);
unsigned ShAmt = Adjust * 8;
+ Value *Shifted = ShAmt ? IRB.CreateLShr(NewLd, ShAmt) : NewLd;
Value *NewVal = IRB.CreateBitCast(
- IRB.CreateTrunc(IRB.CreateLShr(NewLd, ShAmt),
- DL.typeSizeEqualsStoreSize(LI.getType()) ? IntNTy
- : LI.getType()),
+ IRB.CreateTrunc(Shifted, DL.typeSizeEqualsStoreSize(LI.getType())
+ ? IntNTy
+ : LI.getType()),
LI.getType());
LI.replaceAllUsesWith(NewVal);
DeadInsts.emplace_back(&LI);
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-late-codegenprepare.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-late-codegenprepare.ll
index dd1cf68bda7b0..a5fecc4913432 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-late-codegenprepare.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-late-codegenprepare.ll
@@ -7,9 +7,7 @@
; address spaces
define amdgpu_kernel void @constant_from_offset_cast_generic_null() {
; GFX9-LABEL: @constant_from_offset_cast_generic_null(
-; GFX9-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(4) getelementptr (i8, ptr addrspace(4) addrspacecast (ptr null to ptr addrspace(4)), i64 4), align 4
-; GFX9-NEXT: [[TMP2:%.*]] = lshr i32 [[TMP1]], 16
-; GFX9-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
+; GFX9-NEXT: [[TMP3:%.*]] = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr null to ptr addrspace(4)), i64 6), align 1
; GFX9-NEXT: store i8 [[TMP3]], ptr addrspace(1) poison, align 1
; GFX9-NEXT: ret void
;
@@ -25,9 +23,7 @@ define amdgpu_kernel void @constant_from_offset_cast_generic_null() {
define amdgpu_kernel void @constant_from_offset_cast_global_null() {
; GFX9-LABEL: @constant_from_offset_cast_global_null(
-; GFX9-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(4) getelementptr (i8, ptr addrspace(4) addrspacecast (ptr addrspace(1) null to ptr addrspace(4)), i64 4), align 4
-; GFX9-NEXT: [[TMP2:%.*]] = lshr i32 [[TMP1]], 16
-; GFX9-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
+; GFX9-NEXT: [[TMP3:%.*]] = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr addrspace(1) null to ptr addrspace(4)), i64 6), align 1
; GFX9-NEXT: store i8 [[TMP3]], ptr addrspace(1) poison, align 1
; GFX9-NEXT: ret void
;
@@ -63,9 +59,7 @@ define amdgpu_kernel void @constant_from_offset_cast_global_gv() {
define amdgpu_kernel void @constant_from_offset_cast_generic_inttoptr() {
; GFX9-LABEL: @constant_from_offset_cast_generic_inttoptr(
-; GFX9-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(4) getelementptr (i8, ptr addrspace(4) addrspacecast (ptr inttoptr (i64 128 to ptr) to ptr addrspace(4)), i64 4), align 4
-; GFX9-NEXT: [[TMP2:%.*]] = lshr i32 [[TMP1]], 16
-; GFX9-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
+; GFX9-NEXT: [[TMP3:%.*]] = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr inttoptr (i64 128 to ptr) to ptr addrspace(4)), i64 6), align 1
; GFX9-NEXT: store i8 [[TMP3]], ptr addrspace(1) poison, align 1
; GFX9-NEXT: ret void
;
@@ -81,8 +75,8 @@ define amdgpu_kernel void @constant_from_offset_cast_generic_inttoptr() {
define amdgpu_kernel void @constant_from_inttoptr() {
; GFX9-LABEL: @constant_from_inttoptr(
-; GFX9-NEXT: [[LOAD:%.*]] = load i8, ptr addrspace(4) inttoptr (i64 128 to ptr addrspace(4)), align 4
-; GFX9-NEXT: store i8 [[LOAD]], ptr addrspace(1) poison, align 1
+; GFX9-NEXT: [[TMP2:%.*]] = load i8, ptr addrspace(4) inttoptr (i64 128 to ptr addrspace(4)), align 1
+; GFX9-NEXT: store i8 [[TMP2]], ptr addrspace(1) poison, align 1
; GFX9-NEXT: ret void
;
; GFX12-LABEL: @constant_from_inttoptr(
@@ -95,6 +89,38 @@ define amdgpu_kernel void @constant_from_inttoptr() {
ret void
}
+define amdgpu_kernel void @constant_i16_from_dword_aligned() {
+; GFX9-LABEL: @constant_i16_from_dword_aligned(
+; GFX9-NEXT: [[TMP2:%.*]] = load i16, ptr addrspace(4) inttoptr (i64 128 to ptr addrspace(4)), align 2
+; GFX9-NEXT: store i16 [[TMP2]], ptr addrspace(1) poison, align 2
+; GFX9-NEXT: ret void
+;
+; GFX12-LABEL: @constant_i16_from_dword_aligned(
+; GFX12-NEXT: [[LOAD:%.*]] = load i16, ptr addrspace(4) inttoptr (i64 128 to ptr addrspace(4)), align 2
+; GFX12-NEXT: store i16 [[LOAD]], ptr addrspace(1) poison, align 2
+; GFX12-NEXT: ret void
+;
+ %load = load i16, ptr addrspace(4) inttoptr (i64 128 to ptr addrspace(4)), align 2
+ store i16 %load, ptr addrspace(1) poison
+ ret void
+}
+
+define i1 @constant_i16_from_null() {
+; GFX9-LABEL: @constant_i16_from_null(
+; GFX9-NEXT: [[TMP2:%.*]] = load i16, ptr addrspace(4) null, align 2
+; GFX9-NEXT: [[CMP:%.*]] = icmp ne i16 [[TMP2]], 0
+; GFX9-NEXT: ret i1 [[CMP]]
+;
+; GFX12-LABEL: @constant_i16_from_null(
+; GFX12-NEXT: [[LOAD:%.*]] = load i16, ptr addrspace(4) null, align 2
+; GFX12-NEXT: [[CMP:%.*]] = icmp ne i16 [[LOAD]], 0
+; GFX12-NEXT: ret i1 [[CMP]]
+;
+ %load = load i16, ptr addrspace(4) null, align 2
+ %cmp = icmp ne i16 %load, 0
+ ret i1 %cmp
+}
+
define void @broken_phi() {
; GFX9-LABEL: @broken_phi(
; GFX9-NEXT: bb:
@@ -154,10 +180,8 @@ bb7:
; must not be carried over for GFX9.
define amdgpu_kernel void @no_widen_noundef(ptr addrspace(4) align 4 %p, ptr addrspace(1) %out) {
; GFX9-LABEL: @no_widen_noundef(
-; GFX9-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[P:%.*]], i64 0
-; GFX9-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4
-; GFX9-NEXT: [[TMP3:%.*]] = lshr i32 [[TMP2]], 8
-; GFX9-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP3]] to i8
+; GFX9-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[P:%.*]], i64 1
+; GFX9-NEXT: [[TMP4:%.*]] = load i8, ptr addrspace(4) [[P1]], align 1, !noundef [[META0:![0-9]+]]
; GFX9-NEXT: [[VZ:%.*]] = zext i8 [[TMP4]] to i32
; GFX9-NEXT: store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX9-NEXT: ret void
@@ -180,11 +204,8 @@ define amdgpu_kernel void @no_widen_noundef(ptr addrspace(4) align 4 %p, ptr add
; must not be carried over for GFX9.
define amdgpu_kernel void @no_widen_nofpclass(ptr addrspace(4) align 4 %p, ptr addrspace(1) %out) {
; GFX9-LABEL: @no_widen_nofpclass(
-; GFX9-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[P:%.*]], i64 0
-; GFX9-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4
-; GFX9-NEXT: [[TMP3:%.*]] = lshr i32 [[TMP2]], 16
-; GFX9-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP3]] to i16
-; GFX9-NEXT: [[TMP5:%.*]] = bitcast i16 [[TMP4]] to half
+; GFX9-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[P:%.*]], i64 2
+; GFX9-NEXT: [[TMP5:%.*]] = load half, ptr addrspace(4) [[P1]], align 2, !nofpclass [[META1:![0-9]+]]
; GFX9-NEXT: [[VB:%.*]] = bitcast half [[TMP5]] to i16
; GFX9-NEXT: [[VZ:%.*]] = zext i16 [[VB]] to i32
; GFX9-NEXT: store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
@@ -206,5 +227,47 @@ define amdgpu_kernel void @no_widen_nofpclass(ptr addrspace(4) align 4 %p, ptr a
ret void
}
+; dereferenceable(4) covers the padding bytes of the widened access, so widening is safe.
+define amdgpu_kernel void @widen_dereferenceable_dword(ptr addrspace(4) align 4 dereferenceable(4) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: @widen_dereferenceable_dword(
+; GFX9-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[P:%.*]], i64 0
+; GFX9-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4
+; GFX9-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i16
+; GFX9-NEXT: [[VZ:%.*]] = zext i16 [[TMP3]] to i32
+; GFX9-NEXT: store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
+; GFX9-NEXT: ret void
+;
+; GFX12-LABEL: @widen_dereferenceable_dword(
+; GFX12-NEXT: [[V:%.*]] = load i16, ptr addrspace(4) [[P:%.*]], align 2
+; GFX12-NEXT: [[VZ:%.*]] = zext i16 [[V]] to i32
+; GFX12-NEXT: store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
+; GFX12-NEXT: ret void
+;
+ %v = load i16, ptr addrspace(4) %p, align 2
+ %vz = zext i16 %v to i32
+ store i32 %vz, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+; dereferenceable(2) does not cover the padding bytes of the widened access, so widening must not occur.
+define amdgpu_kernel void @no_widen_insufficient_dereferenceable(ptr addrspace(4) align 4 dereferenceable(2) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: @no_widen_insufficient_dereferenceable(
+; GFX9-NEXT: [[V:%.*]] = load i16, ptr addrspace(4) [[P:%.*]], align 2
+; GFX9-NEXT: [[VZ:%.*]] = zext i16 [[V]] to i32
+; GFX9-NEXT: store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
+; GFX9-NEXT: ret void
+;
+; GFX12-LABEL: @no_widen_insufficient_dereferenceable(
+; GFX12-NEXT: [[V:%.*]] = load i16, ptr addrspace(4) [[P:%.*]], align 2
+; GFX12-NEXT: [[VZ:%.*]] = zext i16 [[V]] to i32
+; GFX12-NEXT: store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
+; GFX12-NEXT: ret void
+;
+ %v = load i16, ptr addrspace(4) %p, align 2
+ %vz = zext i16 %v to i32
+ store i32 %vz, ptr addrspace(1) %out, align 4
+ ret void
+}
+
!0 = !{}
!1 = !{i32 3}
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll b/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
index 4166f5a2ef3b2..9f67a48cf3f53 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
@@ -11,14 +11,15 @@
define amdgpu_kernel void @indirect_call_known_no_special_inputs() {
; GFX9-LABEL: indirect_call_known_no_special_inputs:
; GFX9: ; %bb.0: ; %bb
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: global_load_ubyte v3, v[3:4], off
; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s17
; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; GFX9-NEXT: s_add_u32 s0, s0, s17
; GFX9-NEXT: s_addc_u32 s1, s1, 0
; GFX9-NEXT: s_mov_b32 s13, s15
; GFX9-NEXT: s_mov_b32 s12, s14
-; GFX9-NEXT: s_mov_b64 s[14:15], 0
-; GFX9-NEXT: s_load_dword s17, s[14:15], 0x0
; GFX9-NEXT: s_getpc_b64 s[14:15]
; GFX9-NEXT: s_add_u32 s14, s14, wobble at gotpcrel32@lo+4
; GFX9-NEXT: s_addc_u32 s15, s15, wobble at gotpcrel32@hi+12
@@ -28,17 +29,19 @@ define amdgpu_kernel void @indirect_call_known_no_special_inputs() {
; GFX9-NEXT: s_load_dwordx2 s[20:21], s[18:19], 0x0
; GFX9-NEXT: s_load_dwordx2 s[22:23], s[14:15], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s14, 1, s17
-; GFX9-NEXT: s_cmp_eq_u32 s14, 1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT: s_cselect_b32 s19, s23, s21
-; GFX9-NEXT: s_cselect_b32 s18, s22, s20
; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2
-; GFX9-NEXT: s_mov_b32 s14, s16
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_mov_b32 s32, 0
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_readfirstlane_b32 s14, v3
+; GFX9-NEXT: s_and_b32 s14, 1, s14
+; GFX9-NEXT: s_cmp_eq_u32 s14, 1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_cselect_b32 s19, s23, s21
+; GFX9-NEXT: s_cselect_b32 s18, s22, s20
+; GFX9-NEXT: s_mov_b32 s14, s16
; GFX9-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX9-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
index db7f998270f16..0c4569276e98e 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
@@ -272,10 +272,9 @@ define amdgpu_kernel void @incorrect_type_i16_block_count_x(ptr addrspace(1) inr
; GFX942-NEXT: .p2align 8
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: .LBB6_0:
-; GFX942-NEXT: s_load_dword s0, s[0:1], 0x8
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s0
+; GFX942-NEXT: global_load_ushort v1, v0, s[0:1] offset:8
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_store_short v0, v1, s[2:3]
; GFX942-NEXT: s_endpgm
;
@@ -287,10 +286,9 @@ define amdgpu_kernel void @incorrect_type_i16_block_count_x(ptr addrspace(1) inr
; GFX90a-NEXT: .p2align 8
; GFX90a-NEXT: ; %bb.2:
; GFX90a-NEXT: .LBB6_0:
-; GFX90a-NEXT: s_load_dword s0, s[4:5], 0x8
; GFX90a-NEXT: v_mov_b32_e32 v0, 0
-; GFX90a-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90a-NEXT: v_mov_b32_e32 v1, s0
+; GFX90a-NEXT: global_load_ushort v1, v0, s[4:5] offset:8
+; GFX90a-NEXT: s_waitcnt vmcnt(0)
; GFX90a-NEXT: global_store_short v0, v1, s[8:9]
; GFX90a-NEXT: s_endpgm
;
@@ -1015,11 +1013,9 @@ define amdgpu_kernel void @no_free_sgprs_preloadremainder_z(ptr addrspace(1) inr
; GFX90a-NEXT: .p2align 8
; GFX90a-NEXT: ; %bb.2:
; GFX90a-NEXT: .LBB20_0:
-; GFX90a-NEXT: s_load_dword s0, s[8:9], 0x1c
; GFX90a-NEXT: v_mov_b32_e32 v0, 0
-; GFX90a-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90a-NEXT: s_lshr_b32 s0, s0, 16
-; GFX90a-NEXT: v_mov_b32_e32 v1, s0
+; GFX90a-NEXT: global_load_ushort v1, v0, s[8:9] offset:30
+; GFX90a-NEXT: s_waitcnt vmcnt(0)
; GFX90a-NEXT: global_store_dword v0, v1, s[14:15]
; GFX90a-NEXT: s_endpgm
;
More information about the llvm-commits
mailing list