[llvm-branch-commits] [clang] [CIR][AMDGPU] Lower Indirect arguments and sret returns in addrspace(5) (PR #239501)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Nov 6 00:53:38 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-clang
Author: Chaitanya (skc7)
<details>
<summary>Changes</summary>
Summary:
- AMDGPU passes big trivial returns (sret) and non-trivial C++ arguments (Indirect, non-byval) through a pointer in addrspace(5).
- Classifier rejected any Indirect pointer outside the default address space, so these hit an NYI error.
- This PR removes that guard and builds indirect and sret pointers in classified address space.
- Test: `amdgpu-indirect.cir`, which replaces `amdgpu-indirect-nyi.cir`.
Relates to: https://github.com/llvm/llvm-project/issues/220471
Assisted by: claude opus 5.5
---
Full diff: https://github.com/llvm/llvm-project/pull/239501.diff
5 Files Affected:
- (modified) clang/lib/CIR/Dialect/Transforms/CIRABIClassifier.cpp (-3)
- (modified) clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp (+62-34)
- (modified) clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h (+6-5)
- (removed) clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect-nyi.cir (-22)
- (added) clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect.cir (+93)
``````````diff
diff --git a/clang/lib/CIR/Dialect/Transforms/CIRABIClassifier.cpp b/clang/lib/CIR/Dialect/Transforms/CIRABIClassifier.cpp
index 0af64bf105db6..58717e98adc0a 100644
--- a/clang/lib/CIR/Dialect/Transforms/CIRABIClassifier.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/CIRABIClassifier.cpp
@@ -639,9 +639,6 @@ convertABIArgInfo(const llvm::abi::ArgInfo &info, MLIRContext *ctx,
mlir::Type extendedTy = abiTypeToCIR(info.getCoerceToType(), ctx);
return ArgClassification::getExtend(extendedTy, info.isSignExt());
}
- // The rewriter has no address space for the Indirect pointer yet.
- if (info.isIndirect() && info.getIndirectAddrSpace() != 0)
- return std::nullopt;
if (info.isIndirect())
return ArgClassification::getIndirect(info.getIndirectAlign(),
info.getIndirectByVal(),
diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
index 1b909f3cf47f0..3988498cd4586 100644
--- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
@@ -82,6 +82,14 @@ cir::PointerType getByRefPtrTy(mlir::Type pointee,
return getPtrTyInAddrSpace(pointee, ac.indirectAddrSpace);
}
+/// The pointer passed in place of an Indirect argument or sret return of type
+/// \p pointee.
+cir::PointerType getIndirectPtrTy(mlir::Type pointee,
+ const ArgClassification &ac) {
+ assert(ac.kind == ArgKind::Indirect && "expected an indirect classification");
+ return getPtrTyInAddrSpace(pointee, ac.indirectAddrSpace);
+}
+
/// Return the coerced RecordType for a Direct classification that should be
/// flattened into individual scalar arguments, or a null type if the
/// classification does not call for flattening.
@@ -168,7 +176,7 @@ buildNewArgTypes(ArrayRef<mlir::Type> oldArgTypes,
: origTy);
break;
case ArgKind::Indirect:
- newArgTypes.push_back(cir::PointerType::get(origTy));
+ newArgTypes.push_back(getIndirectPtrTy(origTy, ac));
break;
case ArgKind::IndirectAliased:
newArgTypes.push_back(getByRefPtrTy(origTy, ac));
@@ -1099,6 +1107,15 @@ static cir::StoreOp maybeFindParamSpill(mlir::BlockArgument blockArg) {
return store;
}
+/// The alloca a parameter spill writes to through \p addr, either directly or
+/// through an address_space cast of it. Null otherwise.
+static cir::AllocaOp getSpillSlot(mlir::Value addr) {
+ if (auto view = addr.getDefiningOp<cir::CastOp>();
+ view && view.getKind() == cir::CastKind::address_space)
+ addr = view.getSrc();
+ return addr.getDefiningOp<cir::AllocaOp>();
+}
+
/// The store that spills non-byval indirect parameter \p blockArg, and the
/// slot it spills into. prepareNonByvalParameters has already established
/// that the spill is the block argument's only use and that it stores into
@@ -1111,7 +1128,9 @@ findParamSpill(mlir::BlockArgument blockArg) {
return {};
cir::StoreOp store = maybeFindParamSpill(blockArg);
assert(store && "non-byval arg's only use must be its CIRGen param spill");
- return {store, cast<cir::AllocaOp>(store.getAddr().getDefiningOp())};
+ cir::AllocaOp slot = getSpillSlot(store.getAddr());
+ assert(slot && "non-byval arg's spill must store to an alloca");
+ return {store, slot};
}
/// For each Direct arg with a coerced type, change the block argument's type to
@@ -1312,18 +1331,16 @@ insertArgCoercion(mlir::FunctionOpInterface funcOp,
return mlir::failure();
} else if (ac.kind == ArgKind::Indirect ||
ac.kind == ArgKind::IndirectAliased) {
- // byval and non-byval both lower to !cir.ptr<T>, and which it is shows
- // up only in the attrs updateArgAttrs applies. Body lowering differs:
- // the body addresses a byval parameter through the CIRGen slot it was
- // spilled into, so that slot has to be filled from the incoming
- // pointer, while a non-byval parameter's slot is replaced by the
- // pointer so the body works on the caller's storage in place. A byref
- // parameter takes the byval path, through a pointer in its own address
- // space.
+ // byval and non-byval both lower to a pointer to T in the classified
+ // address space, and which it is shows up only in the attrs
+ // updateArgAttrs applies. Body lowering differs: the body addresses a
+ // byval parameter through the CIRGen slot it was spilled into, so that
+ // slot has to be filled from the incoming pointer, while a non-byval
+ // parameter's slot is replaced by the pointer so the body works on the
+ // caller's storage in place. A byref parameter takes the byval path.
bool byRef = ac.kind == ArgKind::IndirectAliased;
- cir::PointerType ptrTy = byRef
- ? getByRefPtrTy(blockArg.getType(), ac)
- : cir::PointerType::get(blockArg.getType());
+ cir::PointerType ptrTy = byRef ? getByRefPtrTy(blockArg.getType(), ac)
+ : getIndirectPtrTy(blockArg.getType(), ac);
if (!byRef && !ac.byVal) {
// Without byval, drop the spill store and let the slot's uses read the
@@ -1607,7 +1624,7 @@ rewriteIndirectReturnCall(cir::CallOp call, const FunctionClassification &fc,
mlir::OpBuilder &builder, const mlir::DataLayout &dl,
bool enableNoundefAttrs) {
mlir::MLIRContext *ctx = call->getContext();
- auto ptrTy = cir::PointerType::get(origRetTy);
+ cir::PointerType ptrTy = getIndirectPtrTy(origRetTy, fc.returnInfo);
builder.setInsertionPoint(call);
uint64_t sretAlign = fc.returnInfo.indirectAlign.value();
@@ -1621,18 +1638,25 @@ rewriteIndirectReturnCall(cir::CallOp call, const FunctionClassification &fc,
// %dest as the sret slot directly so construction flows into it,
// matching classic CodeGen's "pass %s as sret" pattern. %dest must
// dominate the call so the rewritten call (which takes it as operand 0)
- // does not use a value before its definition.
+ // does not use a value before its definition. An address_space cast of a
+ // slot that already has the sret pointer's type stands for that slot.
mlir::Value sretSlot = nullptr;
cir::StoreOp reuseStore = nullptr;
if (call.getResult().hasOneUse()) {
mlir::Operation *user = *call.getResult().getUsers().begin();
- if (auto store = mlir::dyn_cast<cir::StoreOp>(user))
- if (store.getValue() == call.getResult() &&
- store.getAddr().getType() == ptrTy &&
- mlir::DominanceInfo().properlyDominates(store.getAddr(), call)) {
- sretSlot = store.getAddr();
+ auto store = mlir::dyn_cast<cir::StoreOp>(user);
+ if (store && store.getValue() == call.getResult()) {
+ mlir::Value dest = store.getAddr();
+ if (auto view = dest.getDefiningOp<cir::CastOp>();
+ view && view.getKind() == cir::CastKind::address_space &&
+ view.getSrc().getType() == ptrTy)
+ dest = view.getSrc();
+ if (dest.getType() == ptrTy &&
+ mlir::DominanceInfo().properlyDominates(dest, call)) {
+ sretSlot = dest;
reuseStore = store;
}
+ }
}
if (!sretSlot) {
auto alloca = cir::AllocaOp::create(
@@ -1715,6 +1739,8 @@ static mlir::LogicalResult prepareOneNonByvalParameter(
cir::FuncOp funcOp, unsigned argNo, const ArgClassification &ac,
mlir::BlockArgument blockArg, mlir::DominanceInfo &dom,
SmallPtrSetImpl<mlir::Operation *> &claimedSlots) {
+ cir::PointerType paramPtrTy = getIndirectPtrTy(blockArg.getType(), ac);
+
// The spill is the store that writes the parameter itself. Any other use
// consumes the record value. At -O1 and above such a use comes from
// cir-simplify: CIRGen marks a const-qualified parameter's slot const, so
@@ -1772,21 +1798,21 @@ static mlir::LogicalResult prepareOneNonByvalParameter(
mlir::OpBuilder builder(funcOp.getContext());
builder.setInsertionPointToStart(blockArg.getOwner());
auto synthesized = cir::AllocaOp::create(
- builder, funcOp.getLoc(), cir::PointerType::get(blockArg.getType()),
+ builder, funcOp.getLoc(), paramPtrTy,
builder.getStringAttr("nonbyval.param"),
builder.getI64IntegerAttr(ac.indirectAlign.value()));
spill =
cir::StoreOp::create(builder, funcOp.getLoc(), blockArg, synthesized);
}
- // The incoming pointer replaces the slot itself, in the default address
- // space. Retargeting a cast of it would leave the allocation's other views
- // reading storage nothing writes. Storage in another address space, or
- // storage that is not a local alloca at all, is not something the incoming
- // pointer can be substituted for.
- cir::AllocaOp slot = spill.getAddr().getDefiningOp<cir::AllocaOp>();
- if (!slot ||
- spill.getAddr().getType() != cir::PointerType::get(blockArg.getType()))
+ // The incoming pointer replaces the slot itself, which also covers a spill
+ // through an address_space cast of it. Retargeting only the cast would
+ // leave the allocation's other views reading storage nothing writes.
+ // Storage in another address space than the incoming pointer, or storage
+ // that is not a local alloca at all, is not something the incoming pointer
+ // can be substituted for.
+ cir::AllocaOp slot = getSpillSlot(spill.getAddr());
+ if (!slot || slot.getType() != paramPtrTy)
return spill->emitOpError()
<< "non-byval parameter " << argNo
<< " spilled to storage that cannot take the incoming pointer is "
@@ -1917,8 +1943,11 @@ mlir::LogicalResult CIRABIRewriteContext::rewriteFunctionDefinition(
// below.
bool hasSRet =
fc.returnInfo.kind == ArgKind::Indirect && !oldResultTypes.empty();
- if (hasSRet)
- newArgTypes.insert(newArgTypes.begin(), cir::PointerType::get(origRetTy));
+ cir::PointerType sretPtrTy;
+ if (hasSRet) {
+ sretPtrTy = getIndirectPtrTy(origRetTy, fc.returnInfo);
+ newArgTypes.insert(newArgTypes.begin(), sretPtrTy);
+ }
if (funcOp.isDefinition()) {
mlir::Region &body = funcOp->getRegion(0);
@@ -1927,8 +1956,7 @@ mlir::LogicalResult CIRABIRewriteContext::rewriteFunctionDefinition(
// through it before any index-based argument handling below (which
// then accounts for the +1 offset).
if (hasSRet) {
- body.front().insertArgument(0u, cir::PointerType::get(origRetTy),
- funcOp.getLoc());
+ body.front().insertArgument(0u, sretPtrTy, funcOp.getLoc());
insertSRetStores(funcOp, origRetTy, builder);
}
@@ -2204,7 +2232,7 @@ CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp,
bool byRef = ac.kind == ArgKind::IndirectAliased;
bool byCopy = byRef || ac.byVal;
cir::PointerType ptrTy = byRef ? getByRefPtrTy(arg.getType(), ac)
- : cir::PointerType::get(arg.getType());
+ : getIndirectPtrTy(arg.getType(), ac);
// A record operand that names storage is passed from that storage
// rather than from the loaded value: a record value carries only the
diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h
index f22c80dec2d64..671be6c185a83 100644
--- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h
+++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h
@@ -81,11 +81,12 @@ class CIRABIRewriteContext : public mlir::abi::ABIRewriteContext {
/// Bring each non-byval indirect parameter of \p funcOp into the shape the
/// rest of the rewrite assumes: the parameter's only use, if it has one, is
- /// a single store into an alloca of the matching pointer type that no other
- /// non-byval indirect parameter spills to, and that alloca states the
- /// alignment the ABI promises rather than the one CIRGen picked for a local
- /// copy. A use of the parameter as a call argument is routed through a load
- /// of that slot, so that it names the storage an argument has to name.
+ /// a single store into an alloca of the incoming pointer's type, directly or
+ /// through an address_space cast of it. No other non-byval indirect
+ /// parameter spills to that alloca, and it states the alignment the ABI
+ /// promises rather than the one CIRGen picked for a local copy. A use of
+ /// the parameter as a call argument is routed through a load of that slot,
+ /// so that it names the storage an argument has to name.
///
/// Call for every function before any definition or call site is rewritten.
/// findParamSpill asserts this shape while the enclosing definition is
diff --git a/clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect-nyi.cir b/clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect-nyi.cir
deleted file mode 100644
index c0521d18a08bd..0000000000000
--- a/clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect-nyi.cir
+++ /dev/null
@@ -1,22 +0,0 @@
-// RUN: not cir-opt %s -cir-call-conv-lowering=target=amdgpu 2>&1 | FileCheck %s
-
-// AMDGPU Indirect classifications whose pointer is not in the default address
-// space are not lowered yet, so they are reported rather than misclassified.
-
-!s32i = !cir.int<s, 32>
-!rec_Big = !cir.struct<"Big" {data !cir.array<!s32i x 17>}>
-
-module attributes {
- dlti.dl_spec = #dlti.dl_spec<
- #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
- #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
-} {
-
- // A return past the register budget goes through an sret in addrspace(5).
- cir.func @ret_big(%arg0: !cir.ptr<!rec_Big>) -> !rec_Big requires_call_conv_lowering {
- %0 = cir.load %arg0 : !cir.ptr<!rec_Big>, !rec_Big
- cir.return %0 : !rec_Big
- }
-
- // CHECK: not yet implemented for the ABI coercion of type '!cir.struct<"Big"
-}
diff --git a/clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect.cir b/clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect.cir
new file mode 100644
index 0000000000000..1699f504ec083
--- /dev/null
+++ b/clang/test/CIR/Transforms/abi-lowering/amdgpu-indirect.cir
@@ -0,0 +1,93 @@
+// RUN: cir-opt %s -cir-call-conv-lowering=target=amdgpu | FileCheck %s
+
+// Big returns and non-trivial arguments are passed through a pointer in
+// addrspace(5) on AMDGPU, as in classic CodeGen, without extra copies.
+
+!s32i = !cir.int<s, 32>
+!rec_Big = !cir.struct<"Big" {data !cir.array<!s32i x 17>}>
+!rec_NT = !cir.struct<"NT" {data !s32i}>
+
+module attributes {
+ cir.record_layouts = {NT = #cir.record_layout<
+ arg_passing_kind = cannot_pass_in_regs, has_trivial_dtor = false,
+ record_align = 4>},
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>,
+ #dlti.dl_entry<"dlti.alloca_memory_space", 5 : ui64>>
+} {
+
+ cir.func private @fill(!cir.ptr<!rec_Big>)
+ cir.func private @make_big() -> !rec_Big requires_call_conv_lowering
+ cir.func private @copy_nt(!cir.ptr<!rec_NT>, !cir.ptr<!rec_NT>)
+ cir.func private @use_nt(!cir.ptr<!rec_NT>)
+
+ // CHECK: cir.func private @make_big(!cir.ptr<!rec_Big, target_address_space(5)> {llvm.align = 4 : i64, llvm.dead_on_unwind, llvm.sret = !rec_Big, llvm.writable})
+
+ // The callee builds its result directly in the caller's sret memory.
+ cir.func @ret_big() -> !rec_Big requires_call_conv_lowering {
+ %0 = cir.alloca "__retval" align(4) : !cir.ptr<!rec_Big, target_address_space(5)>
+ %1 = cir.cast address_space %0 : !cir.ptr<!rec_Big, target_address_space(5)> -> !cir.ptr<!rec_Big>
+ cir.call @fill(%1) : (!cir.ptr<!rec_Big>) -> ()
+ %2 = cir.load %0 : !cir.ptr<!rec_Big, target_address_space(5)>, !rec_Big
+ cir.return %2 : !rec_Big
+ }
+
+ // CHECK: cir.func @ret_big(%[[SRET:.*]]: !cir.ptr<!rec_Big, target_address_space(5)> {llvm.align = 4 : i64, llvm.dead_on_unwind, llvm.noalias, llvm.sret = !rec_Big, llvm.writable}) {
+ // CHECK-NEXT: %[[SRET_VIEW:.*]] = cir.cast address_space %[[SRET]] : !cir.ptr<!rec_Big, target_address_space(5)> -> !cir.ptr<!rec_Big>
+ // CHECK-NEXT: cir.call @fill(%[[SRET_VIEW]]) : (!cir.ptr<!rec_Big>) -> ()
+ // CHECK-NEXT: cir.return
+ // CHECK-NEXT: }
+
+ // The callee writes straight into `b`, with no temporary copy.
+ cir.func @use_big() requires_call_conv_lowering {
+ %0 = cir.alloca "b" align(4) init : !cir.ptr<!rec_Big, target_address_space(5)>
+ %1 = cir.cast address_space %0 : !cir.ptr<!rec_Big, target_address_space(5)> -> !cir.ptr<!rec_Big>
+ %2 = cir.call @make_big() : () -> !rec_Big
+ cir.store %2, %1 : !rec_Big, !cir.ptr<!rec_Big>
+ cir.call @fill(%1) : (!cir.ptr<!rec_Big>) -> ()
+ cir.return
+ }
+
+ // CHECK: cir.func @use_big() {
+ // CHECK-NEXT: %[[B_SLOT:.*]] = cir.alloca "b" align(4) init : !cir.ptr<!rec_Big, target_address_space(5)>
+ // CHECK-NEXT: %[[B_VIEW:.*]] = cir.cast address_space %[[B_SLOT]] : !cir.ptr<!rec_Big, target_address_space(5)> -> !cir.ptr<!rec_Big>
+ // CHECK-NEXT: cir.call @make_big(%[[B_SLOT]]) : (!cir.ptr<!rec_Big, target_address_space(5)> {llvm.align = 4 : i64, llvm.dead_on_unwind, llvm.sret = !rec_Big, llvm.writable}) -> ()
+ // CHECK-NEXT: cir.call @fill(%[[B_VIEW]]) : (!cir.ptr<!rec_Big>) -> ()
+ // CHECK-NEXT: cir.return
+ // CHECK-NEXT: }
+
+ // The callee works on the caller's object in place, not on a copy.
+ cir.func @take_nt(%arg0: !rec_NT) requires_call_conv_lowering {
+ %0 = cir.alloca "n" align(4) init : !cir.ptr<!rec_NT, target_address_space(5)>
+ %1 = cir.cast address_space %0 : !cir.ptr<!rec_NT, target_address_space(5)> -> !cir.ptr<!rec_NT>
+ cir.store %arg0, %1 : !rec_NT, !cir.ptr<!rec_NT>
+ cir.call @use_nt(%1) : (!cir.ptr<!rec_NT>) -> ()
+ cir.return
+ }
+
+ // CHECK: cir.func @take_nt(%[[N_PTR:.*]]: !cir.ptr<!rec_NT, target_address_space(5)> {llvm.align = 4 : i64, llvm.dereferenceable = 4 : i64, llvm.nofreeobj, llvm.noundef}) {
+ // CHECK-NEXT: %[[N_VIEW:.*]] = cir.cast address_space %[[N_PTR]] : !cir.ptr<!rec_NT, target_address_space(5)> -> !cir.ptr<!rec_NT>
+ // CHECK-NEXT: cir.call @use_nt(%[[N_VIEW]]) : (!cir.ptr<!rec_NT>) -> ()
+ // CHECK-NEXT: cir.return
+ // CHECK-NEXT: }
+
+ // The callee gets the copy-constructed temporary itself, not a byte copy.
+ cir.func @call_nt(%arg0: !cir.ptr<!rec_NT>) requires_call_conv_lowering {
+ %0 = cir.alloca "agg.tmp0" align(4) : !cir.ptr<!rec_NT, target_address_space(5)>
+ %1 = cir.cast address_space %0 : !cir.ptr<!rec_NT, target_address_space(5)> -> !cir.ptr<!rec_NT>
+ cir.call @copy_nt(%1, %arg0) : (!cir.ptr<!rec_NT>, !cir.ptr<!rec_NT>) -> ()
+ %2 = cir.load %1 : !cir.ptr<!rec_NT>, !rec_NT
+ cir.call @take_nt(%2) : (!rec_NT) -> ()
+ cir.return
+ }
+
+ // CHECK: cir.func @call_nt(%[[SRC:.*]]: !cir.ptr<!rec_NT>) {
+ // CHECK-NEXT: %[[TMP:.*]] = cir.alloca "agg.tmp0" align(4) : !cir.ptr<!rec_NT, target_address_space(5)>
+ // CHECK-NEXT: %[[TMP_VIEW:.*]] = cir.cast address_space %[[TMP]] : !cir.ptr<!rec_NT, target_address_space(5)> -> !cir.ptr<!rec_NT>
+ // CHECK-NEXT: cir.call @copy_nt(%[[TMP_VIEW]], %[[SRC]]) : (!cir.ptr<!rec_NT>, !cir.ptr<!rec_NT>) -> ()
+ // CHECK-NEXT: cir.call @take_nt(%[[TMP]]) : (!cir.ptr<!rec_NT, target_address_space(5)> {llvm.align = 4 : i64, llvm.dereferenceable = 4 : i64, llvm.nofreeobj, llvm.noundef}) -> ()
+ // CHECK-NEXT: cir.return
+ // CHECK-NEXT: }
+
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/239501
More information about the llvm-branch-commits
mailing list