[llvm-branch-commits] [llvm] [AMDGPU] PromoteAlloca: split scalar accesses that span several elements (PR #217056)
Domenic Nutile via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Aug 27 07:53:38 PDT 2026
https://github.com/saxlungs updated https://github.com/llvm/llvm-project/pull/217056
>From 4f2cc7ee54bc94f6627e57c4ef84cff2bf21b475 Mon Sep 17 00:00:00 2001
From: Domenic Nutile <domenic.nutile at gmail.com>
Date: Tue, 18 Aug 2026 11:31:37 -0400
Subject: [PATCH 1/3] [AMDGPU] PromoteAlloca: split scalar accesses that span
several elements
promoteAllocaToVector already splits a *vector* access across several
elements when it is a multiple of the element size, but a *scalar* access
had to be bitcastable to the element type, so an i64 load from an alloca
promoted to <8 x i32> was rejected as "not a supported access type" and
the object stayed in scratch.
Accept a scalar access that is a whole multiple of the element size and
route it through the existing subvector path, which already builds the
value from consecutive elements and bitcasts. Accesses with padding are
still rejected, since splitting those would put the pieces at the wrong
offsets, as are non-integer non-float types.
---
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 32 +++++++-
.../promote-alloca-mixed-width-access.ll | 78 +++++++++++++++++++
2 files changed, 106 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index b2bd81c9a9ee0..7bc4dd5296e4e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -609,6 +609,25 @@ computeGEPToVectorIndex(GetElementPtrInst *GEP, AllocaInst *Alloca,
return Result;
}
+// Check if a scalar access of AccessTy spans several elements of a
+// promoted alloca whose element type is VecEltTy, and so has to be split
+// the same way a vector access is. This happens when an object is written
+// one element at a time but read back in wider pieces, e.g. i32 stores into
+// an alloca that is later loaded as i64.
+static bool isMultiElementScalarAccess(Type *VecEltTy, Type *AccessTy,
+ const DataLayout &DL) {
+ if (!AccessTy->isIntegerTy() && !AccessTy->isFloatingPointTy())
+ return false;
+
+ TypeSize AccTS = DL.getTypeStoreSize(AccessTy);
+ // Padding would leave the split pieces at the wrong offsets.
+ if (AccTS * 8 != DL.getTypeSizeInBits(AccessTy))
+ return false;
+
+ TypeSize EltTS = DL.getTypeStoreSize(VecEltTy);
+ return AccTS > EltTS && AccTS.isKnownMultipleOf(EltTS);
+}
+
/// Promotes a single user of the alloca to a vector form.
///
/// \param Inst Instruction to be promoted.
@@ -652,8 +671,9 @@ static Value *promoteAllocaUserToVector(Instruction *Inst, const DataLayout &DL,
}
}
- // Loading a subvector.
- if (isa<FixedVectorType>(AccessTy)) {
+ // Loading a subvector, or a scalar that spans several elements.
+ if (isa<FixedVectorType>(AccessTy) ||
+ isMultiElementScalarAccess(VecEltTy, AccessTy, DL)) {
assert(AccessSize.isKnownMultipleOf(DL.getTypeStoreSize(VecEltTy)));
const unsigned NumLoadedElts = AccessSize / DL.getTypeStoreSize(VecEltTy);
auto *SubVecTy = FixedVectorType::get(VecEltTy, NumLoadedElts);
@@ -728,8 +748,9 @@ static Value *promoteAllocaUserToVector(Instruction *Inst, const DataLayout &DL,
if (CI->isNullValue() && AccessSize == VecStoreSize)
return Builder.CreateBitPreservingCastChain(DL, Val, AA.Vector.Ty);
- // Storing a subvector.
- if (isa<FixedVectorType>(AccessTy)) {
+ // Storing a subvector, or a scalar that spans several elements.
+ if (isa<FixedVectorType>(AccessTy) ||
+ isMultiElementScalarAccess(VecEltTy, AccessTy, DL)) {
assert(AccessSize.isKnownMultipleOf(DL.getTypeStoreSize(VecEltTy)));
const unsigned NumWrittenElts =
AccessSize / DL.getTypeStoreSize(VecEltTy);
@@ -840,6 +861,9 @@ static bool isSupportedAccessType(FixedVectorType *VecTy, Type *AccessTy,
return AccTS.isKnownMultipleOf(VecTS);
}
+ if (isMultiElementScalarAccess(VecTy->getElementType(), AccessTy, DL))
+ return true;
+
return CastInst::isBitOrNoopPointerCastable(VecTy->getElementType(), AccessTy,
DL);
}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
new file mode 100644
index 0000000000000..889451017be38
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=sroa,amdgpu-promote-alloca < %s | FileCheck %s
+
+; An object written one element at a time but read back in wider pieces still
+; promotes: the wide access is split across elements the same way a vector
+; access is.
+
+define amdgpu_kernel void @store_i32_load_i64(ptr addrspace(1) %out, i32 %i, i32 %v) {
+; CHECK-LABEL: define amdgpu_kernel void @store_i32_load_i64(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i32 [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = freeze <8 x i32> poison
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i32> [[A]], i32 [[V]], i32 [[I]]
+; CHECK-NEXT: [[J:%.*]] = shl i32 [[I]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i32> [[TMP1]] to <4 x i64>
+; CHECK-NEXT: [[TMP3:%.*]] = lshr i32 [[J]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i32 [[TMP3]]
+; CHECK-NEXT: store i64 [[TMP4]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+ %a = alloca [8 x i32], align 16, addrspace(5)
+ %p = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %i
+ store i32 %v, ptr addrspace(5) %p, align 4
+ %j = shl i32 %i, 1
+ %q = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %j
+ %l = load i64, ptr addrspace(5) %q, align 8
+ store i64 %l, ptr addrspace(1) %out, align 8
+ ret void
+}
+
+; The same in the other direction: a wide store and narrow loads.
+define amdgpu_kernel void @store_i64_load_i32(ptr addrspace(1) %out, i32 %i, i64 %v) {
+; CHECK-LABEL: define amdgpu_kernel void @store_i64_load_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i64 [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = freeze <8 x i32> poison
+; CHECK-NEXT: [[J:%.*]] = shl i32 [[I]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[V]] to <2 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> [[A]], i32 [[TMP2]], i32 [[J]]
+; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[J]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[TMP5]], i32 [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[TMP6]], i32 [[I]]
+; CHECK-NEXT: store i32 [[TMP7]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+ %a = alloca [8 x i32], align 16, addrspace(5)
+ %j = shl i32 %i, 1
+ %p = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %j
+ store i64 %v, ptr addrspace(5) %p, align 8
+ %q = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %i
+ %l = load i32, ptr addrspace(5) %q, align 4
+ store i32 %l, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+; Negative: i48 has a store size of 8 bytes but only 48 value bits, so splitting
+; it across elements would put the pieces at the wrong offsets.
+define amdgpu_kernel void @padded_access_not_promoted(ptr addrspace(1) %out, i32 %i, i32 %v) {
+; CHECK-LABEL: define amdgpu_kernel void @padded_access_not_promoted(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i32 [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = alloca [8 x i32], align 16, addrspace(5)
+; CHECK-NEXT: [[P:%.*]] = getelementptr [8 x i32], ptr addrspace(5) [[A]], i32 0, i32 [[I]]
+; CHECK-NEXT: store i32 [[V]], ptr addrspace(5) [[P]], align 4
+; CHECK-NEXT: [[J:%.*]] = shl i32 [[I]], 1
+; CHECK-NEXT: [[Q:%.*]] = getelementptr [8 x i32], ptr addrspace(5) [[A]], i32 0, i32 [[J]]
+; CHECK-NEXT: [[L:%.*]] = load i48, ptr addrspace(5) [[Q]], align 8
+; CHECK-NEXT: store i48 [[L]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+ %a = alloca [8 x i32], align 16, addrspace(5)
+ %p = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %i
+ store i32 %v, ptr addrspace(5) %p, align 4
+ %j = shl i32 %i, 1
+ %q = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %j
+ %l = load i48, ptr addrspace(5) %q, align 8
+ store i48 %l, ptr addrspace(1) %out, align 8
+ ret void
+}
>From f1e4e4c2098b2f83a899b77d88a67ff17cc342e0 Mon Sep 17 00:00:00 2001
From: Domenic Nutile <domenic.nutile at gmail.com>
Date: Wed, 19 Aug 2026 11:37:51 -0400
Subject: [PATCH 2/3] Testing updates per PR feedback, add some new test cases
---
.../promote-alloca-mixed-width-access.ll | 93 +++++++++++++++++--
1 file changed, 87 insertions(+), 6 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
index 889451017be38..4ad78d999bfce 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
@@ -1,5 +1,5 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=sroa,amdgpu-promote-alloca < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=amdgpu7.00-amd-amdhsa -passes=sroa,amdgpu-promote-alloca < %s | FileCheck %s
; An object written one element at a time but read back in wider pieces still
; promotes: the wide access is split across elements the same way a vector
@@ -53,10 +53,67 @@ define amdgpu_kernel void @store_i64_load_i32(ptr addrspace(1) %out, i32 %i, i64
ret void
}
-; Negative: i48 has a store size of 8 bytes but only 48 value bits, so splitting
-; it across elements would put the pieces at the wrong offsets.
-define amdgpu_kernel void @padded_access_not_promoted(ptr addrspace(1) %out, i32 %i, i32 %v) {
-; CHECK-LABEL: define amdgpu_kernel void @padded_access_not_promoted(
+; A span that is not a power of two: i96 covers three elements
+define amdgpu_kernel void @store_i96_load_i32(ptr addrspace(1) %out, i32 %i, i96 %v) {
+; CHECK-LABEL: define amdgpu_kernel void @store_i96_load_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i96 [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = freeze <8 x i32> poison
+; CHECK-NEXT: [[J:%.*]] = mul i32 [[I]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i96 [[V]] to <3 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> [[A]], i32 [[TMP2]], i32 [[J]]
+; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[J]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <3 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[TMP5]], i32 [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[J]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <3 x i32> [[TMP1]], i64 2
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[TMP8]], i32 [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP9]], i32 [[I]]
+; CHECK-NEXT: store i32 [[TMP10]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+ %a = alloca [8 x i32], align 16, addrspace(5)
+ %j = mul i32 %i, 3
+ %p = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %j
+ store i96 %v, ptr addrspace(5) %p, align 4
+ %q = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %i
+ %l = load i32, ptr addrspace(5) %q, align 4
+ store i32 %l, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+; <3 x i32> has the same 12-byte footprint as i96 and already went down the
+; subvector path before this change: the two must expand identically.
+define amdgpu_kernel void @store_v3i32_load_i32(ptr addrspace(1) %out, i32 %i, <3 x i32> %v) {
+; CHECK-LABEL: define amdgpu_kernel void @store_v3i32_load_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], <3 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = freeze <8 x i32> poison
+; CHECK-NEXT: [[J:%.*]] = mul i32 [[I]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <3 x i32> [[V]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> [[A]], i32 [[TMP1]], i32 [[J]]
+; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[J]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <3 x i32> [[V]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[TMP4]], i32 [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[J]], 2
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <3 x i32> [[V]], i64 2
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[TMP7]], i32 [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP8]], i32 [[I]]
+; CHECK-NEXT: store i32 [[TMP9]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+ %a = alloca [8 x i32], align 16, addrspace(5)
+ %j = mul i32 %i, 3
+ %p = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %j
+ store <3 x i32> %v, ptr addrspace(5) %p, align 4
+ %q = getelementptr [8 x i32], ptr addrspace(5) %a, i32 0, i32 %i
+ %l = load i32, ptr addrspace(5) %q, align 4
+ store i32 %l, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+; Negative: i48 spans six bytes, which is not a whole number of i32 elements.
+define amdgpu_kernel void @unaligned_span_not_promoted(ptr addrspace(1) %out, i32 %i, i32 %v) {
+; CHECK-LABEL: define amdgpu_kernel void @unaligned_span_not_promoted(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i32 [[V:%.*]]) {
; CHECK-NEXT: [[A:%.*]] = alloca [8 x i32], align 16, addrspace(5)
; CHECK-NEXT: [[P:%.*]] = getelementptr [8 x i32], ptr addrspace(5) [[A]], i32 0, i32 [[I]]
@@ -76,3 +133,27 @@ define amdgpu_kernel void @padded_access_not_promoted(ptr addrspace(1) %out, i32
store i48 %l, ptr addrspace(1) %out, align 8
ret void
}
+
+; Negative: i33 is stored in five bytes but only 33 of those bits are value
+; bits, so there is no bitcast between it and a run of five elements.
+define amdgpu_kernel void @padded_access_not_promoted(ptr addrspace(1) %out, i32 %i, i8 %v) {
+; CHECK-LABEL: define amdgpu_kernel void @padded_access_not_promoted(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], i8 [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = alloca [16 x i8], align 16, addrspace(5)
+; CHECK-NEXT: [[P:%.*]] = getelementptr [16 x i8], ptr addrspace(5) [[A]], i32 0, i32 [[I]]
+; CHECK-NEXT: store i8 [[V]], ptr addrspace(5) [[P]], align 1
+; CHECK-NEXT: [[J:%.*]] = mul i32 [[I]], 5
+; CHECK-NEXT: [[Q:%.*]] = getelementptr [16 x i8], ptr addrspace(5) [[A]], i32 0, i32 [[J]]
+; CHECK-NEXT: [[L:%.*]] = load i33, ptr addrspace(5) [[Q]], align 1
+; CHECK-NEXT: store i33 [[L]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+ %a = alloca [16 x i8], align 16, addrspace(5)
+ %p = getelementptr [16 x i8], ptr addrspace(5) %a, i32 0, i32 %i
+ store i8 %v, ptr addrspace(5) %p, align 1
+ %j = mul i32 %i, 5
+ %q = getelementptr [16 x i8], ptr addrspace(5) %a, i32 0, i32 %j
+ %l = load i33, ptr addrspace(5) %q, align 1
+ store i33 %l, ptr addrspace(1) %out, align 8
+ ret void
+}
>From fd62b485214365236fe00f6b1feed96bd7cf64ca Mon Sep 17 00:00:00 2001
From: Domenic Nutile <domenic.nutile at gmail.com>
Date: Mon, 24 Aug 2026 20:44:11 -0400
Subject: [PATCH 3/3] Refactor to enable scalars inline instead of seperate
helper function
---
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 61 +++++++------------
.../promote-alloca-mixed-width-access.ll | 50 +++++++++++++++
2 files changed, 73 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 7bc4dd5296e4e..ba94dbbf1ddf1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -609,25 +609,6 @@ computeGEPToVectorIndex(GetElementPtrInst *GEP, AllocaInst *Alloca,
return Result;
}
-// Check if a scalar access of AccessTy spans several elements of a
-// promoted alloca whose element type is VecEltTy, and so has to be split
-// the same way a vector access is. This happens when an object is written
-// one element at a time but read back in wider pieces, e.g. i32 stores into
-// an alloca that is later loaded as i64.
-static bool isMultiElementScalarAccess(Type *VecEltTy, Type *AccessTy,
- const DataLayout &DL) {
- if (!AccessTy->isIntegerTy() && !AccessTy->isFloatingPointTy())
- return false;
-
- TypeSize AccTS = DL.getTypeStoreSize(AccessTy);
- // Padding would leave the split pieces at the wrong offsets.
- if (AccTS * 8 != DL.getTypeSizeInBits(AccessTy))
- return false;
-
- TypeSize EltTS = DL.getTypeStoreSize(VecEltTy);
- return AccTS > EltTS && AccTS.isKnownMultipleOf(EltTS);
-}
-
/// Promotes a single user of the alloca to a vector form.
///
/// \param Inst Instruction to be promoted.
@@ -672,10 +653,11 @@ static Value *promoteAllocaUserToVector(Instruction *Inst, const DataLayout &DL,
}
// Loading a subvector, or a scalar that spans several elements.
- if (isa<FixedVectorType>(AccessTy) ||
- isMultiElementScalarAccess(VecEltTy, AccessTy, DL)) {
- assert(AccessSize.isKnownMultipleOf(DL.getTypeStoreSize(VecEltTy)));
- const unsigned NumLoadedElts = AccessSize / DL.getTypeStoreSize(VecEltTy);
+ TypeSize EltSize = DL.getTypeStoreSize(VecEltTy);
+ assert(AccessSize.isKnownMultipleOf(EltSize) &&
+ "promotable access must cover a whole number of elements");
+ const unsigned NumLoadedElts = AccessSize / EltSize;
+ if (NumLoadedElts > 1) {
auto *SubVecTy = FixedVectorType::get(VecEltTy, NumLoadedElts);
assert(DL.getTypeStoreSize(SubVecTy) == DL.getTypeStoreSize(AccessTy));
@@ -749,11 +731,11 @@ static Value *promoteAllocaUserToVector(Instruction *Inst, const DataLayout &DL,
return Builder.CreateBitPreservingCastChain(DL, Val, AA.Vector.Ty);
// Storing a subvector, or a scalar that spans several elements.
- if (isa<FixedVectorType>(AccessTy) ||
- isMultiElementScalarAccess(VecEltTy, AccessTy, DL)) {
- assert(AccessSize.isKnownMultipleOf(DL.getTypeStoreSize(VecEltTy)));
- const unsigned NumWrittenElts =
- AccessSize / DL.getTypeStoreSize(VecEltTy);
+ TypeSize EltSize = DL.getTypeStoreSize(VecEltTy);
+ assert(AccessSize.isKnownMultipleOf(EltSize) &&
+ "promotable access must cover a whole number of elements");
+ const unsigned NumWrittenElts = AccessSize / EltSize;
+ if (NumWrittenElts > 1) {
const unsigned NumVecElts = AA.Vector.Ty->getNumElements();
auto *SubVecTy = FixedVectorType::get(VecEltTy, NumWrittenElts);
assert(DL.getTypeStoreSize(SubVecTy) == DL.getTypeStoreSize(AccessTy));
@@ -839,31 +821,34 @@ static Value *promoteAllocaUserToVector(Instruction *Inst, const DataLayout &DL,
static bool isSupportedAccessType(FixedVectorType *VecTy, Type *AccessTy,
const DataLayout &DL) {
- // Access as a vector type can work if the size of the access vector is a
- // multiple of the size of the alloca's vector element type.
+ // An access that covers several elements can work if its size is a multiple
+ // of the size of the alloca's vector element type, since it can be split
+ // across consecutive elements. This covers accesses by a vector type, as well
+ // as scalar accesses that are wider than one element, which happens when an
+ // object is written one element at a time but read back in wider pieces.
//
// Examples:
// - VecTy = <8 x float>, AccessTy = <4 x float> -> OK
// - VecTy = <4 x double>, AccessTy = <2 x float> -> OK
// - VecTy = <4 x double>, AccessTy = <3 x float> -> NOT OK
// - 3*32 is not a multiple of 64
+ // - VecTy = <8 x i32>, AccessTy = i64 -> OK
//
// We could handle more complicated cases, but it'd make things a lot more
// complicated.
- if (isa<FixedVectorType>(AccessTy)) {
+ if (isa<FixedVectorType>(AccessTy) || AccessTy->isIntegerTy() ||
+ AccessTy->isFloatingPointTy()) {
TypeSize AccTS = DL.getTypeStoreSize(AccessTy);
+ TypeSize VecTS = DL.getTypeStoreSize(VecTy->getElementType());
// If the type size and the store size don't match, we would need to do more
// than just bitcast to translate between an extracted/insertable subvectors
// and the accessed value.
- if (AccTS * 8 != DL.getTypeSizeInBits(AccessTy))
- return false;
- TypeSize VecTS = DL.getTypeStoreSize(VecTy->getElementType());
- return AccTS.isKnownMultipleOf(VecTS);
+ if (AccTS * 8 == DL.getTypeSizeInBits(AccessTy) && AccTS > VecTS &&
+ AccTS.isKnownMultipleOf(VecTS))
+ return true;
}
- if (isMultiElementScalarAccess(VecTy->getElementType(), AccessTy, DL))
- return true;
-
+ // An access that covers exactly one element only needs a cast.
return CastInst::isBitOrNoopPointerCastable(VecTy->getElementType(), AccessTy,
DL);
}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
index 4ad78d999bfce..4825957d2270a 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-mixed-width-access.ll
@@ -157,3 +157,53 @@ define amdgpu_kernel void @padded_access_not_promoted(ptr addrspace(1) %out, i32
store i33 %l, ptr addrspace(1) %out, align 8
ret void
}
+
+; The elements do not have to be integers: an i64 spans two 32-bit pointers,
+; and the pieces are recovered with a ptrtoint rather than a plain bitcast.
+define amdgpu_kernel void @store_ptr_load_i64(ptr addrspace(1) %out, i32 %i, ptr addrspace(5) %v) {
+; CHECK-LABEL: define amdgpu_kernel void @store_ptr_load_i64(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], ptr addrspace(5) [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = freeze <8 x ptr addrspace(5)> poison
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x ptr addrspace(5)> [[A]], ptr addrspace(5) [[V]], i32 [[I]]
+; CHECK-NEXT: [[J:%.*]] = shl i32 [[I]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = ptrtoint <8 x ptr addrspace(5)> [[TMP1]] to <8 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i32> [[TMP2]] to <4 x i64>
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i32 [[J]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 [[TMP4]]
+; CHECK-NEXT: store i64 [[TMP5]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+ %a = alloca [8 x ptr addrspace(5)], align 16, addrspace(5)
+ %p = getelementptr [8 x ptr addrspace(5)], ptr addrspace(5) %a, i32 0, i32 %i
+ store ptr addrspace(5) %v, ptr addrspace(5) %p, align 4
+ %j = shl i32 %i, 1
+ %q = getelementptr [8 x ptr addrspace(5)], ptr addrspace(5) %a, i32 0, i32 %j
+ %l = load i64, ptr addrspace(5) %q, align 8
+ store i64 %l, ptr addrspace(1) %out, align 8
+ ret void
+}
+
+; Negative: a double is the same size as a 64-bit pointer, so it covers exactly
+; one element and is not split. There is no cast between the two types, so the
+; access is still rejected.
+define amdgpu_kernel void @same_size_element_not_castable(ptr addrspace(1) %out, i32 %i, ptr %v) {
+; CHECK-LABEL: define amdgpu_kernel void @same_size_element_not_castable(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]], ptr [[V:%.*]]) {
+; CHECK-NEXT: [[A:%.*]] = alloca [8 x ptr], align 16, addrspace(5)
+; CHECK-NEXT: [[P:%.*]] = getelementptr [8 x ptr], ptr addrspace(5) [[A]], i32 0, i32 [[I]]
+; CHECK-NEXT: store ptr [[V]], ptr addrspace(5) [[P]], align 8
+; CHECK-NEXT: [[J:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT: [[Q:%.*]] = getelementptr [8 x ptr], ptr addrspace(5) [[A]], i32 0, i32 [[J]]
+; CHECK-NEXT: [[L:%.*]] = load double, ptr addrspace(5) [[Q]], align 8
+; CHECK-NEXT: store double [[L]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+ %a = alloca [8 x ptr], align 16, addrspace(5)
+ %p = getelementptr [8 x ptr], ptr addrspace(5) %a, i32 0, i32 %i
+ store ptr %v, ptr addrspace(5) %p, align 8
+ %j = add i32 %i, 1
+ %q = getelementptr [8 x ptr], ptr addrspace(5) %a, i32 0, i32 %j
+ %l = load double, ptr addrspace(5) %q, align 8
+ store double %l, ptr addrspace(1) %out, align 8
+ ret void
+}
More information about the llvm-branch-commits
mailing list