[llvm] 243c0e0 - [AMDGPU] Fix in-memory p7 layout in LowerBufferFatPointers (#218973)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 26 13:01:43 PDT 2026
Author: Arseniy Obolenskiy
Date: 2026-08-26T22:01:37+02:00
New Revision: 243c0e03ba79bfc121ea1e4f405c3692495d50bd
URL: https://github.com/llvm/llvm-project/commit/243c0e03ba79bfc121ea1e4f405c3692495d50bd
DIFF: https://github.com/llvm/llvm-project/commit/243c0e03ba79bfc121ea1e4f405c3692495d50bd.diff
LOG: [AMDGPU] Fix in-memory p7 layout in LowerBufferFatPointers (#218973)
i160 has a smaller alloc size and weaker alignment than ptr
addrspace(7), so rewriting aggregate types in place moved field offsets
that GEPs already baked in
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-p7-in-memory.ll
llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 39653fc985e5c..1d130dc2f77fa 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -51,10 +51,10 @@
// including aggregates containing such pointers, to ones that use `i160`. This
// is handled by `StoreFatPtrsAsIntsAndExpandMemcpyVisitor` , which visits
// loads, stores, and allocas and, if the loaded or stored type contains `ptr
-// addrspace(7)`, rewrites that type to one where the p7s are replaced by i160s,
-// copying other parts of aggregates as needed. In the case of a store, each
-// pointer is `ptrtoint`d to i160 before storing, and load integers are
-// `inttoptr`d back. This same transformation is applied to vectors of pointers.
+// addrspace(7)`, rewrites it to use i160, `ptrtoint`ing before stores and
+// `inttoptr`ing after loads. Vectors of pointers work the same way. Since i160
+// and p7
diff er in size and alignment, aggregates are split into one access per
+// leaf, and allocas and GEPs use byte offsets and sizes from the original type.
//
// Such a transformation allows the later phases of the pass to not need
// to handle buffer fat pointers moving to and from memory, where we load
@@ -437,23 +437,27 @@ class StoreFatPtrsAsIntsAndExpandMemcpyVisitor
IRBuilder<InstSimplifyFolder> IRB;
+ const DataLayout &DL;
+
// Used for memcpy() lowering.
const TargetTransformInfo *TTI;
ScalarEvolution *SE;
- // Convert all the buffer fat pointers within the input value to inttegers
- // so that it can be stored in memory.
- Value *fatPtrsToInts(Value *V, Type *From, Type *To, const Twine &Name);
- // Convert all the i160s that need to be buffer fat pointers (as specified)
- // by the To type) into those pointers to preserve the semantics of the rest
- // of the program.
- Value *intsToFatPtrs(Value *V, Type *From, Type *To, const Twine &Name);
+ Value *applyOffset(Value *Ptr, uint64_t Off);
+ // Visits each maximal subtree of `Ty` that is fat-ptr-free or is itself a
+ // [vector of] fat pointer(s), at its offset in `Ty`'s original layout.
+ void forEachAggLeaf(
+ Type *Ty, SmallVectorImpl<unsigned> &AggIdxs, uint64_t Off,
+ const Twine &Name,
+ function_ref<void(Type *LeafTy, Type *IntLeafTy, ArrayRef<unsigned> Idxs,
+ uint64_t Off, const Twine &Name)>
+ Visit);
public:
StoreFatPtrsAsIntsAndExpandMemcpyVisitor(BufferFatPtrToIntTypeMap *TypeMap,
const DataLayout &DL,
LLVMContext &Ctx)
- : TypeMap(TypeMap), IRB(Ctx, InstSimplifyFolder(DL)) {}
+ : TypeMap(TypeMap), IRB(Ctx, InstSimplifyFolder(DL)), DL(DL) {}
bool processFunction(Function &F, const TargetTransformInfo *TTI,
ScalarEvolution *SE);
@@ -470,68 +474,45 @@ class StoreFatPtrsAsIntsAndExpandMemcpyVisitor
};
} // namespace
-Value *StoreFatPtrsAsIntsAndExpandMemcpyVisitor::fatPtrsToInts(
- Value *V, Type *From, Type *To, const Twine &Name) {
- if (From == To)
- return V;
- if (isBufferFatPtrOrVector(From))
- return IRB.CreatePtrToInt(V, To, Name + ".int");
- if (From->getNumContainedTypes() == 0)
- return V;
- // Structs, arrays, and other compound types.
- Value *Ret = PoisonValue::get(To);
- if (auto *AT = dyn_cast<ArrayType>(From)) {
- Type *FromPart = AT->getArrayElementType();
- Type *ToPart = cast<ArrayType>(To)->getElementType();
- for (uint64_t I = 0, E = AT->getArrayNumElements(); I < E; ++I) {
- Value *Field = IRB.CreateExtractValue(V, I);
- Value *NewField =
- fatPtrsToInts(Field, FromPart, ToPart, Name + "." + Twine(I));
- Ret = IRB.CreateInsertValue(Ret, NewField, I);
- }
- } else {
- for (auto [Idx, FromPart, ToPart] :
- enumerate(From->subtypes(), To->subtypes())) {
- Value *Field = IRB.CreateExtractValue(V, Idx);
- Value *NewField =
- fatPtrsToInts(Field, FromPart, ToPart, Name + "." + Twine(Idx));
- Ret = IRB.CreateInsertValue(Ret, NewField, Idx);
- }
+Value *StoreFatPtrsAsIntsAndExpandMemcpyVisitor::applyOffset(Value *Ptr,
+ uint64_t Off) {
+ // The InstSimplifyFolder gives back `Ptr` itself when `Off` is 0.
+ return IRB.CreatePtrAdd(
+ Ptr, ConstantInt::get(DL.getIndexType(Ptr->getType()), Off),
+ Ptr->getName() + ".off." + Twine(Off), GEPNoWrapFlags::noUnsignedWrap());
+}
+
+void StoreFatPtrsAsIntsAndExpandMemcpyVisitor::forEachAggLeaf(
+ Type *Ty, SmallVectorImpl<unsigned> &AggIdxs, uint64_t Off,
+ const Twine &Name,
+ function_ref<void(Type *LeafTy, Type *IntLeafTy, ArrayRef<unsigned> Idxs,
+ uint64_t Off, const Twine &Name)>
+ Visit) {
+ Type *IntTy = TypeMap->remapType(Ty);
+ if (isBufferFatPtrOrVector(Ty) || Ty == IntTy) {
+ // Zero-sized leaves ({} or [0 x T]) access no bytes; skip them.
+ if (DL.getTypeStoreSize(Ty) != 0)
+ Visit(Ty, IntTy, AggIdxs, Off, Name);
+ return;
}
- return Ret;
-}
-
-Value *StoreFatPtrsAsIntsAndExpandMemcpyVisitor::intsToFatPtrs(
- Value *V, Type *From, Type *To, const Twine &Name) {
- if (From == To)
- return V;
- if (isBufferFatPtrOrVector(To)) {
- Value *Cast = IRB.CreateIntToPtr(V, To, Name + ".ptr");
- return Cast;
- }
- if (From->getNumContainedTypes() == 0)
- return V;
- // Structs, arrays, and other compound types.
- Value *Ret = PoisonValue::get(To);
- if (auto *AT = dyn_cast<ArrayType>(From)) {
- Type *FromPart = AT->getArrayElementType();
- Type *ToPart = cast<ArrayType>(To)->getElementType();
- for (uint64_t I = 0, E = AT->getArrayNumElements(); I < E; ++I) {
- Value *Field = IRB.CreateExtractValue(V, I);
- Value *NewField =
- intsToFatPtrs(Field, FromPart, ToPart, Name + "." + Twine(I));
- Ret = IRB.CreateInsertValue(Ret, NewField, I);
- }
- } else {
- for (auto [Idx, FromPart, ToPart] :
- enumerate(From->subtypes(), To->subtypes())) {
- Value *Field = IRB.CreateExtractValue(V, Idx);
- Value *NewField =
- intsToFatPtrs(Field, FromPart, ToPart, Name + "." + Twine(Idx));
- Ret = IRB.CreateInsertValue(Ret, NewField, Idx);
- }
+ auto Recurse = [&](unsigned I, Type *ElemTy, uint64_t ElemOff) {
+ AggIdxs.push_back(I);
+ forEachAggLeaf(ElemTy, AggIdxs, Off + ElemOff, Name + "." + Twine(I),
+ Visit);
+ AggIdxs.pop_back();
+ };
+ if (auto *ST = dyn_cast<StructType>(Ty)) {
+ const StructLayout *Layout = DL.getStructLayout(ST);
+ for (auto [I, ElemTy, ElemOff] :
+ enumerate(ST->elements(), Layout->getMemberOffsets()))
+ Recurse(I, ElemTy, ElemOff.getFixedValue());
+ return;
}
- return Ret;
+ auto *AT = cast<ArrayType>(Ty);
+ Type *ElemTy = AT->getElementType();
+ uint64_t Stride = DL.getTypeAllocSize(ElemTy).getFixedValue();
+ for (unsigned I : seq<unsigned>(AT->getNumElements()))
+ Recurse(I, ElemTy, I * Stride);
}
bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::processFunction(
@@ -561,6 +542,11 @@ bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitAllocaInst(AllocaInst &I) {
Type *NewTy = TypeMap->remapType(Ty);
if (Ty == NewTy)
return false;
+ // i160 is smaller than ptr addrspace(7) (24 bytes vs. 32); fall back to a
+ // byte array of the original size so sizes computed from Ty stay in bounds.
+ TypeSize AllocSize = DL.getTypeAllocSize(Ty);
+ if (AllocSize.isFixed() && DL.getTypeAllocSize(NewTy) != AllocSize)
+ NewTy = ArrayType::get(IRB.getInt8Ty(), AllocSize.getFixedValue());
I.setAllocatedType(NewTy);
return true;
}
@@ -568,13 +554,16 @@ bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitAllocaInst(AllocaInst &I) {
bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitGetElementPtrInst(
GetElementPtrInst &I) {
Type *Ty = I.getSourceElementType();
- Type *NewTy = TypeMap->remapType(Ty);
- if (Ty == NewTy)
+ if (Ty == TypeMap->remapType(Ty))
return false;
- // We'll be rewriting the type `ptr addrspace(7)` out of existence soon, so
- // make sure GEPs don't have
diff erent semantics with the new type.
- I.setSourceElementType(NewTy);
- I.setResultElementType(TypeMap->remapType(I.getResultElementType()));
+ // Lower to a byte offset now, before remapping changes p7's layout (see file
+ // header).
+ IRB.SetInsertPoint(&I);
+ Value *Off = emitGEPOffset(&IRB, DL, &I);
+ Value *NewGEP = IRB.CreatePtrAdd(I.getPointerOperand(), Off, I.getName(),
+ I.getNoWrapFlags());
+ I.replaceAllUsesWith(NewGEP);
+ I.eraseFromParent();
return true;
}
@@ -585,12 +574,37 @@ bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitLoadInst(LoadInst &LI) {
return false;
IRB.SetInsertPoint(&LI);
+ if (!isBufferFatPtrOrVector(Ty)) {
+ // i160 has the same 20-byte store size as p7, so loading each leaf at
+ // its original-layout offset accesses the same bytes as the unlowered load.
+ Value *Agg = PoisonValue::get(Ty);
+ AAMDNodes AATags = LI.getAAMetadata();
+ SmallVector<unsigned> AggIdxs;
+ forEachAggLeaf(
+ Ty, AggIdxs, 0, LI.getName(),
+ [&](Type *LeafTy, Type *IntLeafTy, ArrayRef<unsigned> Idxs,
+ uint64_t Off, const Twine &Name) {
+ Value *Ptr = applyOffset(LI.getPointerOperand(), Off);
+ LoadInst *NewLI = IRB.CreateAlignedLoad(
+ IntLeafTy, Ptr, commonAlignment(LI.getAlign(), Off), Name);
+ NewLI->setVolatile(LI.isVolatile());
+ copyMetadataForLoad(*NewLI, LI);
+ NewLI->setAAMetadata(AATags.adjustForAccess(Off, IntLeafTy, DL));
+ Value *V = NewLI;
+ if (LeafTy != IntLeafTy)
+ V = IRB.CreateIntToPtr(NewLI, LeafTy, Name + ".ptr");
+ Agg = IRB.CreateInsertValue(Agg, V, Idxs, Name + ".agg");
+ });
+ LI.replaceAllUsesWith(Agg);
+ LI.eraseFromParent();
+ return true;
+ }
auto *NLI = cast<LoadInst>(LI.clone());
NLI->mutateType(IntTy);
NLI = IRB.Insert(NLI);
NLI->takeName(&LI);
- Value *CastBack = intsToFatPtrs(NLI, IntTy, Ty, NLI->getName());
+ Value *CastBack = IRB.CreateIntToPtr(NLI, Ty, NLI->getName() + ".ptr");
LI.replaceAllUsesWith(CastBack);
LI.eraseFromParent();
return true;
@@ -604,7 +618,31 @@ bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitStoreInst(StoreInst &SI) {
return false;
IRB.SetInsertPoint(&SI);
- Value *IntV = fatPtrsToInts(V, Ty, IntTy, V->getName());
+ if (!isBufferFatPtrOrVector(Ty)) {
+ // Store each leaf at its byte offset in the original layout; see
+ // visitLoadInst.
+ AAMDNodes AATags = SI.getAAMetadata();
+ SmallVector<unsigned> AggIdxs;
+ forEachAggLeaf(
+ Ty, AggIdxs, 0, V->getName(),
+ [&](Type *LeafTy, Type *IntLeafTy, ArrayRef<unsigned> Idxs,
+ uint64_t Off, const Twine &Name) {
+ Value *Leaf = IRB.CreateExtractValue(V, Idxs, Name);
+ if (LeafTy != IntLeafTy)
+ Leaf = IRB.CreatePtrToInt(Leaf, IntLeafTy, Name + ".int");
+ auto *NewSI = cast<StoreInst>(SI.clone());
+ NewSI->setAlignment(commonAlignment(SI.getAlign(), Off));
+ NewSI->setOperand(0, Leaf);
+ NewSI->setOperand(1, applyOffset(SI.getPointerOperand(), Off));
+ // Each leaf covers only part of the original assignment.
+ NewSI->setMetadata(LLVMContext::MD_DIAssignID, nullptr);
+ IRB.Insert(NewSI);
+ NewSI->setAAMetadata(AATags.adjustForAccess(Off, IntLeafTy, DL));
+ });
+ SI.eraseFromParent();
+ return true;
+ }
+ Value *IntV = IRB.CreatePtrToInt(V, IntTy, V->getName() + ".int");
for (auto *Dbg : at::getDVRAssignmentMarkers(&SI))
Dbg->setRawLocation(ValueAsMetadata::get(IntV));
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
index e471178394303..970535c96745f 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
@@ -117,7 +117,7 @@ define ptr @gep_of_p7_struct() {
define ptr addrspace(7) @gep_p7_from_p7() {
; CHECK-LABEL: define { ptr addrspace(8), i32 } @gep_p7_from_p7() {
-; CHECK-NEXT: ret { ptr addrspace(8), i32 } { ptr addrspace(8) @buf, i32 48 }
+; CHECK-NEXT: ret { ptr addrspace(8), i32 } { ptr addrspace(8) @buf, i32 64 }
;
ret ptr addrspace(7) getelementptr (ptr addrspace(7),
ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)),
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-p7-in-memory.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-p7-in-memory.ll
index 071666e4a6d75..db259c60cc701 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-p7-in-memory.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-p7-in-memory.ll
@@ -12,8 +12,8 @@ define void @scalar_copy(ptr %a, ptr %b) {
; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128
; CHECK-NEXT: [[X_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8)
; CHECK-NEXT: [[X_PTR_OFF:%.*]] = trunc i160 [[X]] to i32
-; CHECK-NEXT: [[B1:%.*]] = getelementptr i160, ptr [[B]], i64 1
-; CHECK-NEXT: store i160 [[X]], ptr [[B1]], align 32
+; CHECK-NEXT: [[B11:%.*]] = getelementptr i8, ptr [[B]], i64 32
+; CHECK-NEXT: store i160 [[X]], ptr [[B11]], align 32
; CHECK-NEXT: ret void
;
%x = load ptr addrspace(7), ptr %a
@@ -30,8 +30,8 @@ define void @vector_copy(ptr %a, ptr %b) {
; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i160> [[TMP1]] to <4 x i128>
; CHECK-NEXT: [[X_PTR_RSRC:%.*]] = inttoptr <4 x i128> [[TMP2]] to <4 x ptr addrspace(8)>
; CHECK-NEXT: [[X_PTR_OFF:%.*]] = trunc <4 x i160> [[X]] to <4 x i32>
-; CHECK-NEXT: [[B1:%.*]] = getelementptr <4 x i160>, ptr [[B]], i64 2
-; CHECK-NEXT: store <4 x i160> [[X]], ptr [[B1]], align 128
+; CHECK-NEXT: [[B11:%.*]] = getelementptr i8, ptr [[B]], i64 256
+; CHECK-NEXT: store <4 x i160> [[X]], ptr [[B11]], align 128
; CHECK-NEXT: ret void
;
%x = load <4 x ptr addrspace(7)>, ptr %a
@@ -43,15 +43,15 @@ define void @vector_copy(ptr %a, ptr %b) {
define void @alloca(ptr %a, ptr %b) {
; CHECK-LABEL: define void @alloca
; CHECK-SAME: (ptr [[A:%.*]], ptr [[B:%.*]]) {
-; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [5 x i160], align 32, addrspace(5)
+; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [160 x i8], align 32, addrspace(5)
; CHECK-NEXT: [[X:%.*]] = load i160, ptr [[A]], align 32
; CHECK-NEXT: [[TMP1:%.*]] = lshr i160 [[X]], 32
; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128
; CHECK-NEXT: [[X_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8)
; CHECK-NEXT: [[X_PTR_OFF:%.*]] = trunc i160 [[X]] to i32
-; CHECK-NEXT: [[L:%.*]] = getelementptr i160, ptr addrspace(5) [[ALLOCA]], i32 1
-; CHECK-NEXT: store i160 [[X]], ptr addrspace(5) [[L]], align 32
-; CHECK-NEXT: [[Y:%.*]] = load i160, ptr addrspace(5) [[L]], align 32
+; CHECK-NEXT: [[L1:%.*]] = getelementptr i8, ptr addrspace(5) [[ALLOCA]], i32 32
+; CHECK-NEXT: store i160 [[X]], ptr addrspace(5) [[L1]], align 32
+; CHECK-NEXT: [[Y:%.*]] = load i160, ptr addrspace(5) [[L1]], align 32
; CHECK-NEXT: [[TMP3:%.*]] = lshr i160 [[Y]], 32
; CHECK-NEXT: [[TMP4:%.*]] = trunc i160 [[TMP3]] to i128
; CHECK-NEXT: [[Y_PTR_RSRC:%.*]] = inttoptr i128 [[TMP4]] to ptr addrspace(8)
@@ -71,36 +71,42 @@ define void @alloca(ptr %a, ptr %b) {
define void @complex_copy(ptr %a, ptr %b) {
; CHECK-LABEL: define void @complex_copy
; CHECK-SAME: (ptr [[A:%.*]], ptr [[B:%.*]]) {
-; CHECK-NEXT: [[X:%.*]] = load { [2 x i160], i32, i160 }, ptr [[A]], align 32
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { [2 x i160], i32, i160 } [[X]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = extractvalue [2 x i160] [[TMP1]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = lshr i160 [[TMP2]], 32
-; CHECK-NEXT: [[TMP4:%.*]] = trunc i160 [[TMP3]] to i128
-; CHECK-NEXT: [[X_0_0_PTR_RSRC:%.*]] = inttoptr i128 [[TMP4]] to ptr addrspace(8)
-; CHECK-NEXT: [[X_0_0_PTR_OFF:%.*]] = trunc i160 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[X_0_0_PTR_RSRC]], 0
-; CHECK-NEXT: [[X_0_0_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP5]], i32 [[X_0_0_PTR_OFF]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = insertvalue [2 x { ptr addrspace(8), i32 }] poison, { ptr addrspace(8), i32 } [[X_0_0_PTR]], 0
-; CHECK-NEXT: [[TMP7:%.*]] = extractvalue [2 x i160] [[TMP1]], 1
-; CHECK-NEXT: [[TMP8:%.*]] = lshr i160 [[TMP7]], 32
-; CHECK-NEXT: [[TMP9:%.*]] = trunc i160 [[TMP8]] to i128
-; CHECK-NEXT: [[X_0_1_PTR_RSRC:%.*]] = inttoptr i128 [[TMP9]] to ptr addrspace(8)
-; CHECK-NEXT: [[X_0_1_PTR_OFF:%.*]] = trunc i160 [[TMP7]] to i32
-; CHECK-NEXT: [[TMP10:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[X_0_1_PTR_RSRC]], 0
-; CHECK-NEXT: [[X_0_1_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP10]], i32 [[X_0_1_PTR_OFF]], 1
-; CHECK-NEXT: [[TMP11:%.*]] = insertvalue [2 x { ptr addrspace(8), i32 }] [[TMP6]], { ptr addrspace(8), i32 } [[X_0_1_PTR]], 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertvalue { [2 x { ptr addrspace(8), i32 }], i32, { ptr addrspace(8), i32 } } poison, [2 x { ptr addrspace(8), i32 }] [[TMP11]], 0
-; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { [2 x i160], i32, i160 } [[X]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = insertvalue { [2 x { ptr addrspace(8), i32 }], i32, { ptr addrspace(8), i32 } } [[TMP12]], i32 [[TMP13]], 1
-; CHECK-NEXT: [[TMP15:%.*]] = extractvalue { [2 x i160], i32, i160 } [[X]], 2
-; CHECK-NEXT: [[TMP16:%.*]] = lshr i160 [[TMP15]], 32
-; CHECK-NEXT: [[TMP17:%.*]] = trunc i160 [[TMP16]] to i128
-; CHECK-NEXT: [[X_2_PTR_RSRC:%.*]] = inttoptr i128 [[TMP17]] to ptr addrspace(8)
-; CHECK-NEXT: [[X_2_PTR_OFF:%.*]] = trunc i160 [[TMP15]] to i32
-; CHECK-NEXT: [[TMP18:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[X_2_PTR_RSRC]], 0
-; CHECK-NEXT: [[X_2_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP18]], i32 [[X_2_PTR_OFF]], 1
-; CHECK-NEXT: [[TMP19:%.*]] = insertvalue { [2 x { ptr addrspace(8), i32 }], i32, { ptr addrspace(8), i32 } } [[TMP14]], { ptr addrspace(8), i32 } [[X_2_PTR]], 2
-; CHECK-NEXT: store { [2 x i160], i32, i160 } [[X]], ptr [[B]], align 32
+; CHECK-NEXT: [[X_0_0:%.*]] = load i160, ptr [[A]], align 32
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i160 [[X_0_0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128
+; CHECK-NEXT: [[X_0_0_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8)
+; CHECK-NEXT: [[X_0_0_PTR_OFF:%.*]] = trunc i160 [[X_0_0]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[X_0_0_PTR_RSRC]], 0
+; CHECK-NEXT: [[X_0_0_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP3]], i32 [[X_0_0_PTR_OFF]], 1
+; CHECK-NEXT: [[X_0_0_AGG:%.*]] = insertvalue { [2 x { ptr addrspace(8), i32 }], i32, { ptr addrspace(8), i32 } } poison, { ptr addrspace(8), i32 } [[X_0_0_PTR]], 0, 0
+; CHECK-NEXT: [[A_OFF_32:%.*]] = getelementptr nuw i8, ptr [[A]], i64 32
+; CHECK-NEXT: [[X_0_1:%.*]] = load i160, ptr [[A_OFF_32]], align 32
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i160 [[X_0_1]], 32
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i160 [[TMP4]] to i128
+; CHECK-NEXT: [[X_0_1_PTR_RSRC:%.*]] = inttoptr i128 [[TMP5]] to ptr addrspace(8)
+; CHECK-NEXT: [[X_0_1_PTR_OFF:%.*]] = trunc i160 [[X_0_1]] to i32
+; CHECK-NEXT: [[TMP6:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[X_0_1_PTR_RSRC]], 0
+; CHECK-NEXT: [[X_0_1_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP6]], i32 [[X_0_1_PTR_OFF]], 1
+; CHECK-NEXT: [[X_0_1_AGG:%.*]] = insertvalue { [2 x { ptr addrspace(8), i32 }], i32, { ptr addrspace(8), i32 } } [[X_0_0_AGG]], { ptr addrspace(8), i32 } [[X_0_1_PTR]], 0, 1
+; CHECK-NEXT: [[A_OFF_64:%.*]] = getelementptr nuw i8, ptr [[A]], i64 64
+; CHECK-NEXT: [[X_1:%.*]] = load i32, ptr [[A_OFF_64]], align 32
+; CHECK-NEXT: [[X_1_AGG:%.*]] = insertvalue { [2 x { ptr addrspace(8), i32 }], i32, { ptr addrspace(8), i32 } } [[X_0_1_AGG]], i32 [[X_1]], 1
+; CHECK-NEXT: [[A_OFF_96:%.*]] = getelementptr nuw i8, ptr [[A]], i64 96
+; CHECK-NEXT: [[X_2:%.*]] = load i160, ptr [[A_OFF_96]], align 32
+; CHECK-NEXT: [[TMP7:%.*]] = lshr i160 [[X_2]], 32
+; CHECK-NEXT: [[TMP8:%.*]] = trunc i160 [[TMP7]] to i128
+; CHECK-NEXT: [[X_2_PTR_RSRC:%.*]] = inttoptr i128 [[TMP8]] to ptr addrspace(8)
+; CHECK-NEXT: [[X_2_PTR_OFF:%.*]] = trunc i160 [[X_2]] to i32
+; CHECK-NEXT: [[TMP9:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[X_2_PTR_RSRC]], 0
+; CHECK-NEXT: [[X_2_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP9]], i32 [[X_2_PTR_OFF]], 1
+; CHECK-NEXT: [[X_2_AGG:%.*]] = insertvalue { [2 x { ptr addrspace(8), i32 }], i32, { ptr addrspace(8), i32 } } [[X_1_AGG]], { ptr addrspace(8), i32 } [[X_2_PTR]], 2
+; CHECK-NEXT: store i160 [[X_0_0]], ptr [[B]], align 32
+; CHECK-NEXT: [[B_OFF_32:%.*]] = getelementptr nuw i8, ptr [[B]], i64 32
+; CHECK-NEXT: store i160 [[X_0_1]], ptr [[B_OFF_32]], align 32
+; CHECK-NEXT: [[B_OFF_64:%.*]] = getelementptr nuw i8, ptr [[B]], i64 64
+; CHECK-NEXT: store i32 [[X_1]], ptr [[B_OFF_64]], align 32
+; CHECK-NEXT: [[B_OFF_96:%.*]] = getelementptr nuw i8, ptr [[B]], i64 96
+; CHECK-NEXT: store i160 [[X_2]], ptr [[B_OFF_96]], align 32
; CHECK-NEXT: ret void
;
%x = load {[2 x ptr addrspace(7)], i32, ptr addrspace(7)}, ptr %a
@@ -145,3 +151,124 @@ else:
exit:
ret void
}
+
+%struct.has_p7 = type { i32, ptr addrspace(7) }
+
+;; ptr addrspace(7)'s 256-bit alignment puts field 1 at byte offset 32;
+;; instcombine bakes that into an i8 GEP before this pass runs, so both
+;; functions below must load at offset 32.
+define ptr addrspace(7) @struct_field_baked_offset(ptr addrspace(5) %agg) {
+; CHECK-LABEL: define { ptr addrspace(8), i32 } @struct_field_baked_offset
+; CHECK-SAME: (ptr addrspace(5) [[AGG:%.*]]) {
+; CHECK-NEXT: [[F:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[AGG]], i32 32
+; CHECK-NEXT: [[P:%.*]] = load i160, ptr addrspace(5) [[F]], align 32
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i160 [[P]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128
+; CHECK-NEXT: [[P_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8)
+; CHECK-NEXT: [[P_PTR_OFF:%.*]] = trunc i160 [[P]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[P_PTR_RSRC]], 0
+; CHECK-NEXT: [[P_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP3]], i32 [[P_PTR_OFF]], 1
+; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[P_PTR]]
+;
+ %f = getelementptr inbounds i8, ptr addrspace(5) %agg, i32 32
+ %p = load ptr addrspace(7), ptr addrspace(5) %f, align 32
+ ret ptr addrspace(7) %p
+}
+
+define ptr addrspace(7) @struct_field_gep(ptr addrspace(5) %agg) {
+; CHECK-LABEL: define { ptr addrspace(8), i32 } @struct_field_gep
+; CHECK-SAME: (ptr addrspace(5) [[AGG:%.*]]) {
+; CHECK-NEXT: [[F1:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[AGG]], i32 32
+; CHECK-NEXT: [[P:%.*]] = load i160, ptr addrspace(5) [[F1]], align 32
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i160 [[P]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128
+; CHECK-NEXT: [[P_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8)
+; CHECK-NEXT: [[P_PTR_OFF:%.*]] = trunc i160 [[P]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[P_PTR_RSRC]], 0
+; CHECK-NEXT: [[P_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP3]], i32 [[P_PTR_OFF]], 1
+; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[P_PTR]]
+;
+ %f = getelementptr inbounds %struct.has_p7, ptr addrspace(5) %agg, i32 0, i32 1
+ %p = load ptr addrspace(7), ptr addrspace(5) %f, align 32
+ ret ptr addrspace(7) %p
+}
+
+define void @struct_copy(ptr addrspace(5) %a, ptr addrspace(5) %b) {
+; CHECK-LABEL: define void @struct_copy
+; CHECK-SAME: (ptr addrspace(5) [[A:%.*]], ptr addrspace(5) [[B:%.*]]) {
+; CHECK-NEXT: [[X_0:%.*]] = load i32, ptr addrspace(5) [[A]], align 32
+; CHECK-NEXT: [[X_0_AGG:%.*]] = insertvalue [[TMP0:%.*]] poison, i32 [[X_0]], 0
+; CHECK-NEXT: [[A_OFF_32:%.*]] = getelementptr nuw i8, ptr addrspace(5) [[A]], i32 32
+; CHECK-NEXT: [[X_1:%.*]] = load i160, ptr addrspace(5) [[A_OFF_32]], align 32
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i160 [[X_1]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128
+; CHECK-NEXT: [[X_1_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8)
+; CHECK-NEXT: [[X_1_PTR_OFF:%.*]] = trunc i160 [[X_1]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[X_1_PTR_RSRC]], 0
+; CHECK-NEXT: [[X_1_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP3]], i32 [[X_1_PTR_OFF]], 1
+; CHECK-NEXT: [[X_1_AGG:%.*]] = insertvalue [[TMP0]] [[X_0_AGG]], { ptr addrspace(8), i32 } [[X_1_PTR]], 1
+; CHECK-NEXT: store i32 [[X_0]], ptr addrspace(5) [[B]], align 32
+; CHECK-NEXT: [[B_OFF_32:%.*]] = getelementptr nuw i8, ptr addrspace(5) [[B]], i32 32
+; CHECK-NEXT: store i160 [[X_1]], ptr addrspace(5) [[B_OFF_32]], align 32
+; CHECK-NEXT: ret void
+;
+ %x = load %struct.has_p7, ptr addrspace(5) %a, align 32
+ store %struct.has_p7 %x, ptr addrspace(5) %b, align 32
+ ret void
+}
+
+;; sizeof(%struct.has_p7) is 64; a frontend-emitted memcpy of that constant
+;; size must not overrun the lowered alloca.
+define void @struct_alloca_size() {
+; CHECK-LABEL: define void @struct_alloca_size() {
+; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [64 x i8], align 32, addrspace(5)
+; CHECK-NEXT: ret void
+;
+ %alloca = alloca %struct.has_p7, align 32, addrspace(5)
+ ret void
+}
+
+;; Zero-sized members access no bytes and must not produce leaf accesses of
+;; their own (the buffer intrinsics can't be mangled for {}).
+define void @zero_size_member(ptr %a, ptr %b) {
+; CHECK-LABEL: define void @zero_size_member
+; CHECK-SAME: (ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT: [[X_0:%.*]] = load i160, ptr [[A]], align 32
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i160 [[X_0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128
+; CHECK-NEXT: [[X_0_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8)
+; CHECK-NEXT: [[X_0_PTR_OFF:%.*]] = trunc i160 [[X_0]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[X_0_PTR_RSRC]], 0
+; CHECK-NEXT: [[X_0_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP3]], i32 [[X_0_PTR_OFF]], 1
+; CHECK-NEXT: [[X_0_AGG:%.*]] = insertvalue { { ptr addrspace(8), i32 }, {}, [0 x i32] } poison, { ptr addrspace(8), i32 } [[X_0_PTR]], 0
+; CHECK-NEXT: store i160 [[X_0]], ptr [[B]], align 32
+; CHECK-NEXT: ret void
+;
+ %x = load { ptr addrspace(7), {}, [0 x i32] }, ptr %a
+ store { ptr addrspace(7), {}, [0 x i32] } %x, ptr %b
+ ret void
+}
+
+;; <4 x i160> has the same allocation size as <4 x p7>, so vector allocas
+;; keep the remapped vector type instead of decaying to a byte array.
+define void @vector_alloca() {
+; CHECK-LABEL: define void @vector_alloca() {
+; CHECK-NEXT: [[ALLOCA:%.*]] = alloca <4 x i160>, align 128, addrspace(5)
+; CHECK-NEXT: ret void
+;
+ %alloca = alloca <4 x ptr addrspace(7)>, addrspace(5)
+ ret void
+}
+
+;; GEPs over p7 itself step by its 32-byte allocation size, not by the
+;; 24-byte allocation size of i160.
+define ptr addrspace(5) @gep_p7_stride(ptr addrspace(5) %p, i32 %i) {
+; CHECK-LABEL: define ptr addrspace(5) @gep_p7_stride
+; CHECK-SAME: (ptr addrspace(5) [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT: [[Q_IDX:%.*]] = mul i32 [[I]], 32
+; CHECK-NEXT: [[Q1:%.*]] = getelementptr i8, ptr addrspace(5) [[P]], i32 [[Q_IDX]]
+; CHECK-NEXT: ret ptr addrspace(5) [[Q1]]
+;
+ %q = getelementptr ptr addrspace(7), ptr addrspace(5) %p, i32 %i
+ ret ptr addrspace(5) %q
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
index 21d0caa40ea3e..38ba6d81e96a8 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
@@ -7,9 +7,9 @@ target triple = "amdgpu9.00--"
define float @debug_stash_pointer(ptr addrspace(8) %buf, i32 %idx, ptr addrspace(8) %aux) !dbg !5 {
; CHECK-LABEL: define float @debug_stash_pointer
; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[IDX:%.*]], ptr addrspace(8) [[AUX:%.*]]) #[[ATTR0:[0-9]+]] !dbg [[DBG6:![0-9]+]] {
-; CHECK-NEXT: [[BUF_PTR_VAR:%.*]] = alloca i160, align 32, addrspace(5), !dbg [[DBG22:![0-9]+]]
+; CHECK-NEXT: [[BUF_PTR_VAR:%.*]] = alloca [32 x i8], align 32, addrspace(5), !dbg [[DBG22:![0-9]+]]
; CHECK-NEXT: #dbg_value(ptr addrspace(5) [[BUF_PTR_VAR]], [[META11:![0-9]+]], !DIExpression(), [[DBG22]])
-; CHECK-NEXT: [[AUX_PTR_VAR:%.*]] = alloca i160, align 32, addrspace(5), !dbg [[DBG23:![0-9]+]]
+; CHECK-NEXT: [[AUX_PTR_VAR:%.*]] = alloca [32 x i8], align 32, addrspace(5), !dbg [[DBG23:![0-9]+]]
; CHECK-NEXT: #dbg_value(ptr addrspace(5) [[AUX_PTR_VAR]], [[META13:![0-9]+]], !DIExpression(), [[DBG23]])
; CHECK-NEXT: #dbg_value(i32 0, [[META14:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 128, 32), [[META24:![0-9]+]])
; CHECK-NEXT: #dbg_value(ptr addrspace(8) [[BUF]], [[META14]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[META24]])
More information about the llvm-commits
mailing list