[llvm-branch-commits] [llvm] [AMDGPU] PromoteAlloca: split scalar accesses that span several elements (PR #217056)
Domenic Nutile via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Aug 19 10:23:54 PDT 2026
https://github.com/saxlungs updated https://github.com/llvm/llvm-project/pull/217056
>From a58a80affd84fd58c6ea79ac296ad8b179d670c4 Mon Sep 17 00:00:00 2001
From: Domenic Nutile <domenic.nutile at gmail.com>
Date: Tue, 18 Aug 2026 11:31:37 -0400
Subject: [PATCH 1/2] [AMDGPU] PromoteAlloca: split scalar accesses that span
several elements
promoteAllocaToVector already splits a *vector* access across several
elements when it is a multiple of the element size, but a *scalar* access
had to be bitcastable to the element type, so an i64 load from an alloca
promoted to <8 x i32> was rejected as "not a supported access type" and
the object stayed in scratch.
Accept a scalar access that is a whole multiple of the element size and
route it through the existing subvector path, which already builds the
value from consecutive elements and bitcasts. Accesses with padding are
still rejected, since splitting those would put the pieces at the wrong
offsets, as are non-integer non-float types.
---
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 32 +++++++-
.../promote-alloca-mixed-width-access.ll | 78 +++++++++++++++++++
2 files changed, 106 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index f1a17bfa2e4d9..270f776571248 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -609,6 +609,25 @@ computeGEPToVectorIndex(GetElementPtrInst *GEP, AllocaInst *Alloca,
return Result;
}
+// Check if a scalar access of AccessTy spans several elements of a
+// promoted alloca whose element type is VecEltTy, and so has to be split
+// the same way a vector access is. This happens when an object is written
+// one element at a time but read back in wider pieces, e.g. i32 stores into
+// an alloca that is later loaded as i64.
+static bool isMultiElementScalarAccess(Type *VecEltTy, Type *AccessTy,
+ const DataLayout &DL) {
+ if (!AccessTy->isIntegerTy() && !AccessTy->isFloatingPointTy())
+ return false;
+
+ TypeSize AccTS = DL.getTypeStoreSize(AccessTy);
+ // Padding would leave the split pieces at the wrong offsets.
+ if (AccTS * 8 != DL.getTypeSizeInBits(AccessTy))
+ return false;
+
+ TypeSize EltTS = DL.getTypeStoreSize(VecEltTy);
+ return AccTS > EltTS && AccTS.isKnownMultipleOf(EltTS);
+}
+
/// Promotes a single user of the alloca to a vector form.
///
/// \param Inst Instruction to be promoted.
@@ -652,8 +671,9 @@ static Value *promoteAllocaUserToVector(Instruction *Inst, const DataLayout &DL,
}
}
- // Loading a subvector.
- if (isa<FixedVectorType>(AccessTy)) {
+ // Loading a subvector, or a scalar that spans several elements.
+ if (isa<FixedVectorType>(AccessTy) ||
+ isMultiElementScalarAccess(VecEltTy, AccessTy, DL)) {
assert(AccessSize.isKnownMultipleOf(DL.getTypeStoreSize(VecEltTy)));
const unsigned NumLoadedElts = AccessSize / DL.getTypeStoreSize(VecEltTy);
auto *SubVecTy = FixedVectorType::get(VecEltTy, NumLoadedElts);
@@ -728,8 +748,9 @@ static Value *promoteAllocaUserToVector(Instruction *Inst, const DataLayout &DL,
if (CI->isNullValue() && AccessSize == VecStoreSize)
return Builder.CreateBitPreservingCastChain(DL, Val, AA.Vector.Ty);
- // Storing a subvector.
- if (isa<FixedVectorType>(AccessTy)) {
+ // Storing a subvector, or a scalar that spans several elements.
+ if (isa<FixedVectorType>(AccessTy) ||
+ isMultiElementScalarAccess(VecEltTy, AccessTy, DL)) {
assert(AccessSize.isKnownMultipleOf(DL.getTypeStoreSize(VecEltTy)));
const unsigned NumWrittenElts =
AccessSize / DL.getTypeStoreSize(VecEltTy);
@@ -840,6 +861,9 @@ static bool isSupportedAccessType(FixedVectorType *VecTy, Type *AccessTy,
return AccTS.isKnownMultipleOf(VecTS);
}
+ if (isMultiElementScalarAccess(VecTy->getElementType(), AccessTy, DL))
+ return true;
+
return CastInst::isBitOrNoopPointerCastable(VecTy->getElementType(), AccessTy,
DL);
}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
new file mode 100644
index 0000000000000..889451017be38
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=sroa,amdgpu-promote-alloca < %s | FileCheck %s
+
+; An object written one element at a time but read back in wider pieces still
+; promotes: the wide access is split across elements the same way a vector
+; access is.
+
+define amdgpu_kernel void @store_i32_load_i64(ptr addrspace(1) %out, i32 %i, i32 %v) {
+; CHECK-LABEL: define amdgpu_kernel void @store_i32_load_i64(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i32 [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = freeze <8 x i32> poison
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i32> [[A]], i32 [[V]], i32 [[I]]
+; CHECK-NEXT: [[J:%.*]] = shl i32 [[I]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i32> [[TMP1]] to <4 x i64>
+; CHECK-NEXT: [[TMP3:%.*]] = lshr i32 [[J]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i32 [[TMP3]]
+; CHECK-NEXT: store i64 [[TMP4]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+ %a = alloca [8 x i32], align 16, addrspace(5)
+ %p = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %i
+ store i32 %v, ptr addrspace(5) %p, align 4
+ %j = shl i32 %i, 1
+ %q = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %j
+ %l = load i64, ptr addrspace(5) %q, align 8
+ store i64 %l, ptr addrspace(1) %out, align 8
+ ret void
+}
+
+; The same in the other direction: a wide store and narrow loads.
+define amdgpu_kernel void @store_i64_load_i32(ptr addrspace(1) %out, i32 %i, i64 %v) {
+; CHECK-LABEL: define amdgpu_kernel void @store_i64_load_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i64 [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = freeze <8 x i32> poison
+; CHECK-NEXT: [[J:%.*]] = shl i32 [[I]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[V]] to <2 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> [[A]], i32 [[TMP2]], i32 [[J]]
+; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[J]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[TMP5]], i32 [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[TMP6]], i32 [[I]]
+; CHECK-NEXT: store i32 [[TMP7]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+ %a = alloca [8 x i32], align 16, addrspace(5)
+ %j = shl i32 %i, 1
+ %p = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %j
+ store i64 %v, ptr addrspace(5) %p, align 8
+ %q = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %i
+ %l = load i32, ptr addrspace(5) %q, align 4
+ store i32 %l, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+; Negative: i48 has a store size of 8 bytes but only 48 value bits, so splitting
+; it across elements would put the pieces at the wrong offsets.
+define amdgpu_kernel void @padded_access_not_promoted(ptr addrspace(1) %out, i32 %i, i32 %v) {
+; CHECK-LABEL: define amdgpu_kernel void @padded_access_not_promoted(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i32 [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = alloca [8 x i32], align 16, addrspace(5)
+; CHECK-NEXT: [[P:%.*]] = getelementptr [8 x i32], ptr addrspace(5) [[A]], i32 0, i32 [[I]]
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: [[J:%.*]] = shl i32 [[I]], 1
+; CHECK-NEXT: [[Q:%.*]] = getelementptr [8 x i32], ptr addrspace(5) [[A]], i32 0, i32 [[J]]
+; CHECK-NEXT: [[L:%.*]] = load i48, ptr addrspace(5) [[Q]], align 8
+; CHECK-NEXT: store i48 [[L]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+ %a = alloca [8 x i32], align 16, addrspace(5)
+ %p = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %i
+ store i32 %v, ptr addrspace(5) %p, align 4
+ %j = shl i32 %i, 1
+ %q = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %j
+ %l = load i48, ptr addrspace(5) %q, align 8
+ store i48 %l, ptr addrspace(1) %out, align 8
+ ret void
+}
>From 512af05f35e0d0bea2d2e99180e58851e42f984d Mon Sep 17 00:00:00 2001
From: Domenic Nutile <domenic.nutile at gmail.com>
Date: Wed, 19 Aug 2026 11:37:51 -0400
Subject: [PATCH 2/2] Testing updates per PR feedback, add some new test cases
---
.../promote-alloca-mixed-width-access.ll | 93 +++++++++++++++++--
1 file changed, 87 insertions(+), 6 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
index 889451017be38..4ad78d999bfce 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
@@ -1,5 +1,5 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=sroa,amdgpu-promote-alloca < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=amdgpu7.00-amd-amdhsa -passes=sroa,amdgpu-promote-alloca < %s | FileCheck %s
; An object written one element at a time but read back in wider pieces still
; promotes: the wide access is split across elements the same way a vector
@@ -53,10 +53,67 @@ define amdgpu_kernel void @store_i64_load_i32(ptr addrspace(1) %out, i32 %i, i64
ret void
}
-; Negative: i48 has a store size of 8 bytes but only 48 value bits, so splitting
-; it across elements would put the pieces at the wrong offsets.
-define amdgpu_kernel void @padded_access_not_promoted(ptr addrspace(1) %out, i32 %i, i32 %v) {
-; CHECK-LABEL: define amdgpu_kernel void @padded_access_not_promoted(
+; A span that is not a power of two: i96 covers three elements
+define amdgpu_kernel void @store_i96_load_i32(ptr addrspace(1) %out, i32 %i, i96 %v) {
+; CHECK-LABEL: define amdgpu_kernel void @store_i96_load_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i96 [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = freeze <8 x i32> poison
+; CHECK-NEXT: [[J:%.*]] = mul i32 [[I]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i96 [[V]] to <3 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> [[A]], i32 [[TMP2]], i32 [[J]]
+; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[J]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <3 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[TMP5]], i32 [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[J]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <3 x i32> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[TMP8]], i32 [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP9]], i32 [[I]]
+; CHECK-NEXT: store i32 [[TMP10]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+ %a = alloca [8 x i32], align 16, addrspace(5)
+ %j = mul i32 %i, 3
+ %p = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %j
+ store i96 %v, ptr addrspace(5) %p, align 4
+ %q = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %i
+ %l = load i32, ptr addrspace(5) %q, align 4
+ store i32 %l, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+; <3 x i32> has the same 12-byte footprint as i96 and already went down the
+; subvector path before this change: the two must expand identically.
+define amdgpu_kernel void @store_v3i32_load_i32(ptr addrspace(1) %out, i32 %i, <3 x i32> %v) {
+; CHECK-LABEL: define amdgpu_kernel void @store_v3i32_load_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], <3 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = freeze <8 x i32> poison
+; CHECK-NEXT: [[J:%.*]] = mul i32 [[I]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <3 x i32> [[V]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> [[A]], i32 [[TMP1]], i32 [[J]]
+; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[J]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <3 x i32> [[V]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[TMP4]], i32 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[J]], 2
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <3 x i32> [[V]], i64 2
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[TMP7]], i32 [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP8]], i32 [[I]]
+; CHECK-NEXT: store i32 [[TMP9]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+ %a = alloca [8 x i32], align 16, addrspace(5)
+ %j = mul i32 %i, 3
+ %p = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %j
+ store <3 x i32> %v, ptr addrspace(5) %p, align 4
+ %q = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %i
+ %l = load i32, ptr addrspace(5) %q, align 4
+ store i32 %l, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+; Negative: i48 spans six bytes, which is not a whole number of i32 elements.
+define amdgpu_kernel void @unaligned_span_not_promoted(ptr addrspace(1) %out, i32 %i, i32 %v) {
+; CHECK-LABEL: define amdgpu_kernel void @unaligned_span_not_promoted(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i32 [[V:%.*]]) {
; CHECK-NEXT: [[A:%.*]] = alloca [8 x i32], align 16, addrspace(5)
; CHECK-NEXT: [[P:%.*]] = getelementptr [8 x i32], ptr addrspace(5) [[A]], i32 0, i32 [[I]]
@@ -76,3 +133,27 @@ define amdgpu_kernel void @padded_access_not_promoted(ptr addrspace(1) %out, i32
store i48 %l, ptr addrspace(1) %out, align 8
ret void
}
+
+; Negative: i33 is stored in five bytes but only 33 of those bits are value
+; bits, so there is no bitcast between it and a run of five elements.
+define amdgpu_kernel void @padded_access_not_promoted(ptr addrspace(1) %out, i32 %i, i8 %v) {
+; CHECK-LABEL: define amdgpu_kernel void @padded_access_not_promoted(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i8 [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = alloca [16 x i8], align 16, addrspace(5)
+; CHECK-NEXT: [[P:%.*]] = getelementptr [16 x i8], ptr addrspace(5) [[A]], i32 0, i32 [[I]]
+; CHECK-NEXT: store i8 [[V]], ptr addrspace(5) [[P]], align 1
+; CHECK-NEXT: [[J:%.*]] = mul i32 [[I]], 5
+; CHECK-NEXT: [[Q:%.*]] = getelementptr [16 x i8], ptr addrspace(5) [[A]], i32 0, i32 [[J]]
+; CHECK-NEXT: [[L:%.*]] = load i33, ptr addrspace(5) [[Q]], align 1
+; CHECK-NEXT: store i33 [[L]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+ %a = alloca [16 x i8], align 16, addrspace(5)
+ %p = getelementptr [16 x i8], ptr addrspace(5) %a, i32 0, i32 %i
+ store i8 %v, ptr addrspace(5) %p, align 1
+ %j = mul i32 %i, 5
+ %q = getelementptr [16 x i8], ptr addrspace(5) %a, i32 0, i32 %j
+ %l = load i33, ptr addrspace(5) %q, align 1
+ store i33 %l, ptr addrspace(1) %out, align 8
+ ret void
+}
More information about the llvm-branch-commits
mailing list