[Mlir-commits] [mlir] [mlir][LLVM] Add operand bundle support (PR #108933)
Sirui Mu
llvmlistbot at llvm.org
Thu Sep 19 06:52:51 PDT 2024
https://github.com/Lancern updated https://github.com/llvm/llvm-project/pull/108933
>From 7a7bc80287ed9c063dd4e12b767052f314d77bfc Mon Sep 17 00:00:00 2001
From: Sirui Mu <msrlancern at gmail.com>
Date: Thu, 19 Sep 2024 21:52:21 +0800
Subject: [PATCH] [mlir][LLVM] Add operand bundle support
---
mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 39 ++-
mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp | 6 +
.../Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp | 16 +-
mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 235 ++++++++++++++++--
.../LLVMIR/LLVMToLLVMIRTranslation.cpp | 68 ++++-
mlir/test/Dialect/LLVMIR/invalid.mlir | 40 ++-
mlir/test/Target/LLVMIR/llvmir.mlir | 49 ++++
7 files changed, 414 insertions(+), 39 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index d956d7f27f784d..556b39c67c75a1 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -551,7 +551,13 @@ def LLVM_InvokeOp : LLVM_Op<"invoke", [
Variadic<LLVM_Type>:$normalDestOperands,
Variadic<LLVM_Type>:$unwindDestOperands,
OptionalAttr<DenseI32ArrayAttr>:$branch_weights,
- DefaultValuedAttr<CConv, "CConv::C">:$CConv);
+ DefaultValuedAttr<CConv, "CConv::C">:$CConv,
+ VariadicOfVariadic<LLVM_Type,
+ "op_bundle_sizes">:$op_bundle_operands,
+ DenseI32ArrayAttr:$op_bundle_sizes,
+ OptionalProperty<
+ ArrayProperty<StringProperty, "operand bundle tags">
+ >:$op_bundle_tags);
let results = (outs Optional<LLVM_Type>:$result);
let successors = (successor AnySuccessor:$normalDest,
AnySuccessor:$unwindDest);
@@ -587,7 +593,8 @@ def LLVM_LandingpadOp : LLVM_Op<"landingpad"> {
//===----------------------------------------------------------------------===//
def LLVM_CallOp : LLVM_MemAccessOpBase<"call",
- [DeclareOpInterfaceMethods<FastmathFlagsInterface>,
+ [AttrSizedOperandSegments,
+ DeclareOpInterfaceMethods<FastmathFlagsInterface>,
DeclareOpInterfaceMethods<CallOpInterface>,
DeclareOpInterfaceMethods<SymbolUserOpInterface>,
DeclareOpInterfaceMethods<BranchWeightOpInterface>]> {
@@ -641,8 +648,13 @@ def LLVM_CallOp : LLVM_MemAccessOpBase<"call",
OptionalAttr<LLVM_MemoryEffectsAttr>:$memory_effects,
OptionalAttr<UnitAttr>:$convergent,
OptionalAttr<UnitAttr>:$no_unwind,
- OptionalAttr<UnitAttr>:$will_return
- );
+ OptionalAttr<UnitAttr>:$will_return,
+ VariadicOfVariadic<LLVM_Type,
+ "op_bundle_sizes">:$op_bundle_operands,
+ DenseI32ArrayAttr:$op_bundle_sizes,
+ OptionalProperty<
+ ArrayProperty<StringProperty, "operand bundle tags">
+ >:$op_bundle_tags);
// Append the aliasing related attributes defined in LLVM_MemAccessOpBase.
let arguments = !con(args, aliasAttrs);
let results = (outs Optional<LLVM_Type>:$result);
@@ -662,6 +674,7 @@ def LLVM_CallOp : LLVM_MemAccessOpBase<"call",
OpBuilder<(ins "LLVMFunctionType":$calleeType, "StringRef":$callee,
CArg<"ValueRange", "{}">:$args)>
];
+ let hasVerifier = 1;
let hasCustomAssemblyFormat = 1;
let extraClassDeclaration = [{
/// Returns the callee function type.
@@ -1875,7 +1888,8 @@ def LLVM_InlineAsmOp : LLVM_Op<"inline_asm", [DeclareOpInterfaceMethods<MemoryEf
def LLVM_CallIntrinsicOp
: LLVM_Op<"call_intrinsic",
- [DeclareOpInterfaceMethods<FastmathFlagsInterface>]> {
+ [AttrSizedOperandSegments,
+ DeclareOpInterfaceMethods<FastmathFlagsInterface>]> {
let summary = "Call to an LLVM intrinsic function.";
let description = [{
Call the specified llvm intrinsic. If the intrinsic is overloaded, use
@@ -1883,13 +1897,24 @@ def LLVM_CallIntrinsicOp
}];
let arguments = (ins StrAttr:$intrin, Variadic<LLVM_Type>:$args,
DefaultValuedAttr<LLVM_FastmathFlagsAttr,
- "{}">:$fastmathFlags);
+ "{}">:$fastmathFlags,
+ VariadicOfVariadic<LLVM_Type,
+ "op_bundle_sizes">:$op_bundle_operands,
+ DenseI32ArrayAttr:$op_bundle_sizes,
+ OptionalProperty<
+ ArrayProperty<StringProperty, "operand bundle tags">
+ >:$op_bundle_tags);
let results = (outs Optional<LLVM_Type>:$results);
let llvmBuilder = [{
return convertCallLLVMIntrinsicOp(op, builder, moduleTranslation);
}];
let assemblyFormat = [{
- $intrin `(` $args `)` `:` functional-type($args, $results) attr-dict
+ $intrin `(` $args `)`
+ ( `bundlearg` `(` $op_bundle_operands^ `)` )?
+ ( `bundletag` `(` $op_bundle_tags^ `)` )?
+ `:` functional-type($args, $results)
+ ( `bundletype` `(` type($op_bundle_operands)^ `)` )?
+ attr-dict
}];
let hasVerifier = 1;
diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 4c2e8682285c52..d7c05514e9f1b8 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -544,6 +544,12 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
callOp.getLoc(), packedResult ? TypeRange(packedResult) : TypeRange(),
promoted, callOp->getAttrs());
+ newOp->setAttr(newOp.getOperandSegmentSizesAttrName(),
+ rewriter.getDenseI32ArrayAttr(
+ {static_cast<int32_t>(promoted.size()), 0}));
+ newOp->setAttr(newOp.getOpBundleSizesAttrName(),
+ rewriter.getDenseI32ArrayAttr({}));
+
SmallVector<Value, 4> results;
if (numResults < 2) {
// If < 2 results, packing did not do anything and we can just return.
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index ca786316324198..e00ac13e7e821d 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -837,8 +837,14 @@ class FunctionCallPattern
matchAndRewrite(spirv::FunctionCallOp callOp, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
if (callOp.getNumResults() == 0) {
- rewriter.replaceOpWithNewOp<LLVM::CallOp>(
+ auto newOp = rewriter.replaceOpWithNewOp<LLVM::CallOp>(
callOp, std::nullopt, adaptor.getOperands(), callOp->getAttrs());
+ newOp->setAttr(
+ newOp.getOperandSegmentSizesAttrName(),
+ rewriter.getDenseI32ArrayAttr(
+ {static_cast<int32_t>(adaptor.getOperands().size()), 0}));
+ newOp->setAttr(newOp.getOpBundleSizesAttrName(),
+ rewriter.getDenseI32ArrayAttr({}));
return success();
}
@@ -846,8 +852,14 @@ class FunctionCallPattern
auto dstType = typeConverter.convertType(callOp.getType(0));
if (!dstType)
return rewriter.notifyMatchFailure(callOp, "type conversion failed");
- rewriter.replaceOpWithNewOp<LLVM::CallOp>(
+ auto newOp = rewriter.replaceOpWithNewOp<LLVM::CallOp>(
callOp, dstType, adaptor.getOperands(), callOp->getAttrs());
+ newOp->setAttr(
+ newOp.getOperandSegmentSizesAttrName(),
+ rewriter.getDenseI32ArrayAttr(
+ {static_cast<int32_t>(adaptor.getOperands().size()), 0}));
+ newOp->setAttr(newOp.getOpBundleSizesAttrName(),
+ rewriter.getDenseI32ArrayAttr({}));
return success();
}
};
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 205d7494d4378c..40f199512d2b99 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -954,6 +954,7 @@ void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
/*CConv=*/nullptr, /*TailCallKind=*/nullptr,
/*memory_effects=*/nullptr,
/*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
+ /*op_bundle_operands=*/{}, /*op_bundle_tags=*/std::nullopt,
/*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
/*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
}
@@ -980,6 +981,7 @@ void CallOp::build(OpBuilder &builder, OperationState &state,
/*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
/*convergent=*/nullptr,
/*no_unwind=*/nullptr, /*will_return=*/nullptr,
+ /*op_bundle_operands=*/{}, /*op_bundle_tags=*/std::nullopt,
/*access_groups=*/nullptr,
/*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
}
@@ -992,6 +994,7 @@ void CallOp::build(OpBuilder &builder, OperationState &state,
/*fastmathFlags=*/nullptr, /*branch_weights=*/nullptr,
/*CConv=*/nullptr, /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
/*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
+ /*op_bundle_operands=*/{}, /*op_bundle_tags=*/std::nullopt,
/*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
/*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
}
@@ -1004,6 +1007,7 @@ void CallOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
/*fastmathFlags=*/nullptr, /*branch_weights=*/nullptr,
/*CConv=*/nullptr, /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
/*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
+ /*op_bundle_operands=*/{}, /*op_bundle_tags=*/std::nullopt,
/*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
/*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
}
@@ -1027,7 +1031,7 @@ void CallOp::setCalleeFromCallable(CallInterfaceCallable callee) {
}
Operation::operand_range CallOp::getArgOperands() {
- return getOperands().drop_front(getCallee().has_value() ? 0 : 1);
+ return getCalleeOperands().drop_front(getCallee().has_value() ? 0 : 1);
}
MutableOperandRange CallOp::getArgOperandsMutable() {
@@ -1100,6 +1104,27 @@ LogicalResult verifyCallOpVarCalleeType(OpTy callOp) {
return success();
}
+template <typename OpType>
+static LogicalResult verifyOperandBundles(OpType &op) {
+ OperandRangeRange opBundleOperands = op.getOpBundleOperands();
+ std::optional<ArrayRef<std::string>> opBundleTags = op.getOpBundleTags();
+
+ if (!opBundleTags.has_value()) {
+ if (!opBundleOperands.empty())
+ return op.emitError("expected operand bundle tags");
+ return success();
+ }
+
+ if (opBundleTags->size() != opBundleOperands.size())
+ return op.emitError("expected ")
+ << opBundleOperands.size()
+ << " operand bundle tags, but actually got " << opBundleTags->size();
+
+ return success();
+}
+
+LogicalResult CallOp::verify() { return verifyOperandBundles(*this); }
+
LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
if (failed(verifyCallOpVarCalleeType(*this)))
return failure();
@@ -1150,15 +1175,15 @@ LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
// Verify that the operand and result types match the callee.
if (!funcType.isVarArg() &&
- funcType.getNumParams() != (getNumOperands() - isIndirect))
+ funcType.getNumParams() != (getCalleeOperands().size() - isIndirect))
return emitOpError() << "incorrect number of operands ("
- << (getNumOperands() - isIndirect)
+ << (getCalleeOperands().size() - isIndirect)
<< ") for callee (expecting: "
<< funcType.getNumParams() << ")";
- if (funcType.getNumParams() > (getNumOperands() - isIndirect))
+ if (funcType.getNumParams() > (getCalleeOperands().size() - isIndirect))
return emitOpError() << "incorrect number of operands ("
- << (getNumOperands() - isIndirect)
+ << (getCalleeOperands().size() - isIndirect)
<< ") for varargs callee (expecting at least: "
<< funcType.getNumParams() << ")";
@@ -1188,6 +1213,33 @@ LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
return success();
}
+template <typename OpType>
+static void printOpBundleArgsAndTags(OpType &op, OpAsmPrinter &p) {
+ if (!op.getOpBundleOperands().empty())
+ return;
+
+ p << "bundlearg(";
+ llvm::interleaveComma(op.getOpBundleOperands(), p, [&](ValueRange operands) {
+ p << "(" << operands << ")";
+ });
+
+ p << ") bundletag([";
+ llvm::interleaveComma(*op.getOpBundleTags(), p,
+ [&](const std::string &tag) { p.printString(tag); });
+ p << "])";
+}
+
+template <typename OpType>
+static void printOpBundleTypes(OpType &op, OpAsmPrinter &p) {
+ if (op.getOpBundleOperands().empty())
+ return;
+
+ p << "bundletype(";
+ llvm::interleaveComma(op.getOpBundleOperands().getTypes(), p,
+ [&](TypeRange types) { p << "(" << types << ")"; });
+ p << ")";
+}
+
void CallOp::print(OpAsmPrinter &p) {
auto callee = getCallee();
bool isDirect = callee.has_value();
@@ -1208,16 +1260,23 @@ void CallOp::print(OpAsmPrinter &p) {
else
p << getOperand(0);
- auto args = getOperands().drop_front(isDirect ? 0 : 1);
+ auto args = getCalleeOperands().drop_front(isDirect ? 0 : 1);
p << '(' << args << ')';
// Print the variadic callee type if the call is variadic.
if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
p << " vararg(" << *varCalleeType << ")";
+ if (!getOpBundleOperands().empty()) {
+ p << " ";
+ printOpBundleArgsAndTags(*this, p);
+ }
+
p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()),
{getCalleeAttrName(), getTailCallKindAttrName(),
- getVarCalleeTypeAttrName(), getCConvAttrName()});
+ getVarCalleeTypeAttrName(), getCConvAttrName(),
+ getOperandSegmentSizesAttrName(),
+ getOpBundleSizesAttrName()});
p << " : ";
if (!isDirect)
@@ -1225,6 +1284,11 @@ void CallOp::print(OpAsmPrinter &p) {
// Reconstruct the function MLIR function type from operand and result types.
p.printFunctionalType(args.getTypes(), getResultTypes());
+
+ if (!getOpBundleOperands().empty()) {
+ p << ", ";
+ printOpBundleTypes(*this, p);
+ }
}
/// Parses the type of a call operation and resolves the operands if the parsing
@@ -1285,14 +1349,102 @@ static ParseResult parseOptionalCallFuncPtr(
return success();
}
+static ParseResult parseOpBundleArgsAndTags(
+ OpAsmParser &parser,
+ SmallVector<OpAsmParser::UnresolvedOperand> &opBundleOperands,
+ SmallVector<int32_t> &opBundleSizes,
+ SmallVector<std::string> &opBundleTags) {
+ if (succeeded(parser.parseOptionalKeyword("bundlearg"))) {
+ if (parser.parseLParen())
+ return failure();
+
+ int32_t curSize = 0;
+ do {
+ if (parser.parseOptionalLParen())
+ break;
+ if (parser.parseOperandList(opBundleOperands) || parser.parseRParen())
+ return failure();
+ opBundleSizes.push_back(opBundleOperands.size() - curSize);
+ curSize = opBundleOperands.size();
+ } while (succeeded(parser.parseOptionalComma()));
+
+ if (parser.parseRParen())
+ return failure();
+ }
+
+ if (succeeded(parser.parseOptionalKeyword("bundletag"))) {
+ if (parser.parseLParen())
+ return failure();
+ if (parser.parseLSquare())
+ return failure();
+
+ do {
+ std::string tag;
+ if (parser.parseString(&tag))
+ break;
+ opBundleTags.push_back(std::move(tag));
+ } while (succeeded(parser.parseOptionalComma()));
+
+ if (parser.parseRSquare())
+ return failure();
+ if (parser.parseRParen())
+ return failure();
+ }
+
+ return success();
+}
+
+static ParseResult parseOpBundleTypeAndResolveOperands(
+ OpAsmParser &parser, OperationState &state,
+ ArrayRef<OpAsmParser::UnresolvedOperand> opBundleOperands) {
+ SmallVector<Type> opBundleTypes;
+
+ auto currentLoc = parser.getCurrentLocation();
+ if (succeeded(parser.parseOptionalKeyword("bundletype"))) {
+ if (parser.parseLParen())
+ return failure();
+
+ do {
+ if (parser.parseOptionalLParen())
+ break;
+ if (parser.parseOptionalRParen() &&
+ (parser.parseTypeList(opBundleTypes) || parser.parseRParen()))
+ return failure();
+ } while (succeeded(parser.parseOptionalComma()));
+
+ if (parser.parseRParen())
+ return failure();
+ }
+
+ if (opBundleTypes.size() != opBundleOperands.size())
+ return parser.emitError(currentLoc, "expected ")
+ << opBundleOperands.size()
+ << " types for operand bundle operands, but actually got "
+ << opBundleTypes.size();
+
+ if (!opBundleOperands.empty()) {
+ if (parser.resolveOperands(opBundleOperands, opBundleTypes, currentLoc,
+ state.operands))
+ return failure();
+ }
+
+ return success();
+}
+
// <operation> ::= `llvm.call` (cconv)? (tailcallkind)? (function-id | ssa-use)
// `(` ssa-use-list `)`
// ( `vararg(` var-callee-type `)` )?
+// ( `bundlearg(` ssa-use-list-list `)` )?
+// ( `bundletags(` str-elements-attr `) )
// attribute-dict? `:` (type `,`)? function-type
+// (`,` `bundletype(` type-list-list `)`)?
ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) {
SymbolRefAttr funcAttr;
TypeAttr varCalleeType;
SmallVector<OpAsmParser::UnresolvedOperand> operands;
+ SmallVector<OpAsmParser::UnresolvedOperand> opBundleOperands;
+ SmallVector<int32_t> opBundleSizes;
+ SmallVector<std::string> opBundleTags;
// Default to C Calling Convention if no keyword is provided.
result.addAttribute(
@@ -1333,11 +1485,30 @@ ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) {
return failure();
}
+ if (parseOpBundleArgsAndTags(parser, opBundleOperands, opBundleSizes,
+ opBundleTags))
+ return failure();
+ result.addAttribute(
+ getOpBundleSizesAttrName(result.name),
+ DenseI32ArrayAttr::get(parser.getContext(), opBundleSizes));
+ if (!opBundleTags.empty())
+ result.getOrAddProperties<CallOp::Properties>().op_bundle_tags.emplace(
+ std::move(opBundleTags));
+
if (parser.parseOptionalAttrDict(result.attributes))
return failure();
// Parse the trailing type list and resolve the operands.
- return parseCallTypeAndResolveOperands(parser, result, isDirect, operands);
+ if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands))
+ return failure();
+ if (parseOpBundleTypeAndResolveOperands(parser, result, opBundleOperands))
+ return failure();
+
+ result.addAttribute(CallOp::getOperandSegmentSizeAttr(),
+ parser.getBuilder().getDenseI32ArrayAttr(
+ {static_cast<int32_t>(operands.size()),
+ static_cast<int32_t>(opBundleOperands.size())}));
+ return success();
}
LLVMFunctionType CallOp::getCalleeFunctionType() {
@@ -1356,7 +1527,8 @@ void InvokeOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
auto calleeType = func.getFunctionType();
build(builder, state, getCallOpResultTypes(calleeType),
getCallOpVarCalleeType(calleeType), SymbolRefAttr::get(func), ops,
- normalOps, unwindOps, nullptr, nullptr, normal, unwind);
+ normalOps, unwindOps, nullptr, nullptr, {}, std::nullopt, normal,
+ unwind);
}
void InvokeOp::build(OpBuilder &builder, OperationState &state, TypeRange tys,
@@ -1365,7 +1537,7 @@ void InvokeOp::build(OpBuilder &builder, OperationState &state, TypeRange tys,
ValueRange unwindOps) {
build(builder, state, tys,
/*var_callee_type=*/nullptr, callee, ops, normalOps, unwindOps, nullptr,
- nullptr, normal, unwind);
+ nullptr, {}, std::nullopt, normal, unwind);
}
void InvokeOp::build(OpBuilder &builder, OperationState &state,
@@ -1374,7 +1546,7 @@ void InvokeOp::build(OpBuilder &builder, OperationState &state,
Block *unwind, ValueRange unwindOps) {
build(builder, state, getCallOpResultTypes(calleeType),
getCallOpVarCalleeType(calleeType), callee, ops, normalOps, unwindOps,
- nullptr, nullptr, normal, unwind);
+ nullptr, nullptr, {}, std::nullopt, normal, unwind);
}
SuccessorOperands InvokeOp::getSuccessorOperands(unsigned index) {
@@ -1402,7 +1574,7 @@ void InvokeOp::setCalleeFromCallable(CallInterfaceCallable callee) {
}
Operation::operand_range InvokeOp::getArgOperands() {
- return getOperands().drop_front(getCallee().has_value() ? 0 : 1);
+ return getCalleeOperands().drop_front(getCallee().has_value() ? 0 : 1);
}
MutableOperandRange InvokeOp::getArgOperandsMutable() {
@@ -1423,6 +1595,9 @@ LogicalResult InvokeOp::verify() {
return emitError("first operation in unwind destination should be a "
"llvm.landingpad operation");
+ if (failed(verifyOperandBundles(*this)))
+ return failure();
+
return success();
}
@@ -1452,15 +1627,26 @@ void InvokeOp::print(OpAsmPrinter &p) {
if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
p << " vararg(" << *varCalleeType << ")";
+ if (!getOpBundleOperands().empty()) {
+ p << " ";
+ printOpBundleArgsAndTags(*this, p);
+ }
+
p.printOptionalAttrDict((*this)->getAttrs(),
{getCalleeAttrName(), getOperandSegmentSizeAttr(),
- getCConvAttrName(), getVarCalleeTypeAttrName()});
+ getCConvAttrName(), getVarCalleeTypeAttrName(),
+ getOpBundleSizesAttrName()});
p << " : ";
if (!isDirect)
p << getOperand(0).getType() << ", ";
p.printFunctionalType(llvm::drop_begin(getOperandTypes(), isDirect ? 0 : 1),
getResultTypes());
+
+ if (!getOpBundleOperands().empty()) {
+ p << ", ";
+ printOpBundleTypes(*this, p);
+ }
}
// <operation> ::= `llvm.invoke` (cconv)? (function-id | ssa-use)
@@ -1468,11 +1654,17 @@ void InvokeOp::print(OpAsmPrinter &p) {
// `to` bb-id (`[` ssa-use-and-type-list `]`)?
// `unwind` bb-id (`[` ssa-use-and-type-list `]`)?
// ( `vararg(` var-callee-type `)` )?
+// ( `bundlearg(` ssa-use-list-list `)` )?
+// ( `bundletags(` str-elements-attr `) )
// attribute-dict? `:` (type `,`)? function-type
+// (`,` `bundletype(` type-list-list `)`)?
ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 8> operands;
SymbolRefAttr funcAttr;
TypeAttr varCalleeType;
+ SmallVector<OpAsmParser::UnresolvedOperand> opBundleOperands;
+ SmallVector<int32_t> opBundleSizes;
+ SmallVector<std::string> opBundleTags;
Block *normalDest, *unwindDest;
SmallVector<Value, 4> normalOperands, unwindOperands;
Builder &builder = parser.getBuilder();
@@ -1513,12 +1705,24 @@ ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) {
return failure();
}
+ if (parseOpBundleArgsAndTags(parser, opBundleOperands, opBundleSizes,
+ opBundleTags))
+ return failure();
+ result.addAttribute(
+ getOpBundleSizesAttrName(result.name),
+ DenseI32ArrayAttr::get(parser.getContext(), opBundleSizes));
+ if (!opBundleTags.empty())
+ result.getOrAddProperties<InvokeOp::Properties>().op_bundle_tags.emplace(
+ std::move(opBundleTags));
+
if (parser.parseOptionalAttrDict(result.attributes))
return failure();
// Parse the trailing type list and resolve the function operands.
if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands))
return failure();
+ if (parseOpBundleTypeAndResolveOperands(parser, result, opBundleOperands))
+ return failure();
result.addSuccessors({normalDest, unwindDest});
result.addOperands(normalOperands);
@@ -1528,7 +1732,8 @@ ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) {
builder.getDenseI32ArrayAttr(
{static_cast<int32_t>(operands.size()),
static_cast<int32_t>(normalOperands.size()),
- static_cast<int32_t>(unwindOperands.size())}));
+ static_cast<int32_t>(unwindOperands.size()),
+ static_cast<int32_t>(opBundleOperands.size())}));
return success();
}
@@ -3108,6 +3313,8 @@ OpFoldResult LLVM::OrOp::fold(FoldAdaptor adaptor) {
LogicalResult CallIntrinsicOp::verify() {
if (!getIntrin().starts_with("llvm."))
return emitOpError() << "intrinsic name must start with 'llvm.'";
+ if (failed(verifyOperandBundles(*this)))
+ return failure();
return success();
}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
index d948ff5eaf1769..53ca302518a90e 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
@@ -102,6 +102,40 @@ getOverloadedDeclaration(CallIntrinsicOp op, llvm::Intrinsic::ID id,
return llvm::Intrinsic::getDeclaration(module, id, overloadedArgTysRef);
}
+static llvm::OperandBundleDef
+convertOperandBundle(OperandRange bundleOperands, StringRef bundleTag,
+ LLVM::ModuleTranslation &moduleTranslation) {
+ std::vector<llvm::Value *> operands;
+ operands.reserve(bundleOperands.size());
+ for (auto bundleArg : bundleOperands)
+ operands.push_back(moduleTranslation.lookupValue(bundleArg));
+ return llvm::OperandBundleDef(bundleTag.str(), std::move(operands));
+}
+
+static SmallVector<llvm::OperandBundleDef>
+convertOperandBundles(OperandRangeRange bundleOperands,
+ ArrayRef<std::string> bundleTags,
+ LLVM::ModuleTranslation &moduleTranslation) {
+ assert(bundleOperands.size() == bundleTags.size() &&
+ "operand bundles and tags do not match");
+
+ SmallVector<llvm::OperandBundleDef> bundles;
+ bundles.reserve(bundleOperands.size());
+
+ for (auto [operands, tag] : llvm::zip(bundleOperands, bundleTags))
+ bundles.push_back(convertOperandBundle(operands, tag, moduleTranslation));
+ return bundles;
+}
+
+static SmallVector<llvm::OperandBundleDef>
+convertOperandBundles(OperandRangeRange bundleOperands,
+ std::optional<ArrayRef<std::string>> bundleTags,
+ LLVM::ModuleTranslation &moduleTranslation) {
+ if (!bundleTags.has_value())
+ bundleTags.emplace();
+ return convertOperandBundles(bundleOperands, *bundleTags, moduleTranslation);
+}
+
/// Builder for LLVM_CallIntrinsicOp
static LogicalResult
convertCallLLVMIntrinsicOp(CallIntrinsicOp op, llvm::IRBuilderBase &builder,
@@ -138,15 +172,15 @@ convertCallLLVMIntrinsicOp(CallIntrinsicOp op, llvm::IRBuilderBase &builder,
// Check the argument types of the call. If the function is variadic, check
// the subrange of required arguments.
if (!fn->getFunctionType()->isVarArg() &&
- op.getNumOperands() != fn->arg_size()) {
+ op.getArgs().size() != fn->arg_size()) {
return mlir::emitError(op.getLoc(), "intrinsic call has ")
- << op.getNumOperands() << " operands but " << op.getIntrinAttr()
+ << op.getArgs().size() << " operands but " << op.getIntrinAttr()
<< " expects " << fn->arg_size();
}
if (fn->getFunctionType()->isVarArg() &&
- op.getNumOperands() < fn->arg_size()) {
+ op.getArgs().size() < fn->arg_size()) {
return mlir::emitError(op.getLoc(), "intrinsic call has ")
- << op.getNumOperands() << " operands but variadic "
+ << op.getArgs().size() << " operands but variadic "
<< op.getIntrinAttr() << " expects at least " << fn->arg_size();
}
// Check the arguments up to the number the function requires.
@@ -164,8 +198,10 @@ convertCallLLVMIntrinsicOp(CallIntrinsicOp op, llvm::IRBuilderBase &builder,
FastmathFlagsInterface itf = op;
builder.setFastMathFlags(getFastmathFlags(itf));
- auto *inst =
- builder.CreateCall(fn, moduleTranslation.lookupValues(op.getOperands()));
+ auto *inst = builder.CreateCall(
+ fn, moduleTranslation.lookupValues(op.getArgs()),
+ convertOperandBundles(op.getOpBundleOperands(), op.getOpBundleTags(),
+ moduleTranslation));
if (op.getNumResults() == 1)
moduleTranslation.mapValue(op->getResults().front()) = inst;
return success();
@@ -205,17 +241,21 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder,
// itself. Otherwise, this is an indirect call and the callee is the first
// operand, look it up as a normal value.
if (auto callOp = dyn_cast<LLVM::CallOp>(opInst)) {
- auto operands = moduleTranslation.lookupValues(callOp.getOperands());
+ auto operands = moduleTranslation.lookupValues(callOp.getCalleeOperands());
+ SmallVector<llvm::OperandBundleDef> opBundles =
+ convertOperandBundles(callOp.getOpBundleOperands(),
+ callOp.getOpBundleTags(), moduleTranslation);
ArrayRef<llvm::Value *> operandsRef(operands);
llvm::CallInst *call;
if (auto attr = callOp.getCalleeAttr()) {
- call = builder.CreateCall(
- moduleTranslation.lookupFunction(attr.getValue()), operandsRef);
+ call =
+ builder.CreateCall(moduleTranslation.lookupFunction(attr.getValue()),
+ operandsRef, opBundles);
} else {
llvm::FunctionType *calleeType = llvm::cast<llvm::FunctionType>(
moduleTranslation.convertType(callOp.getCalleeFunctionType()));
call = builder.CreateCall(calleeType, operandsRef.front(),
- operandsRef.drop_front());
+ operandsRef.drop_front(), opBundles);
}
call->setCallingConv(convertCConvToLLVM(callOp.getCConv()));
call->setTailCallKind(convertTailCallKindToLLVM(callOp.getTailCallKind()));
@@ -312,13 +352,17 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder,
if (auto invOp = dyn_cast<LLVM::InvokeOp>(opInst)) {
auto operands = moduleTranslation.lookupValues(invOp.getCalleeOperands());
+ SmallVector<llvm::OperandBundleDef> opBundles =
+ convertOperandBundles(invOp.getOpBundleOperands(),
+ invOp.getOpBundleTags(), moduleTranslation);
ArrayRef<llvm::Value *> operandsRef(operands);
llvm::InvokeInst *result;
if (auto attr = opInst.getAttrOfType<FlatSymbolRefAttr>("callee")) {
result = builder.CreateInvoke(
moduleTranslation.lookupFunction(attr.getValue()),
moduleTranslation.lookupBlock(invOp.getSuccessor(0)),
- moduleTranslation.lookupBlock(invOp.getSuccessor(1)), operandsRef);
+ moduleTranslation.lookupBlock(invOp.getSuccessor(1)), operandsRef,
+ opBundles);
} else {
llvm::FunctionType *calleeType = llvm::cast<llvm::FunctionType>(
moduleTranslation.convertType(invOp.getCalleeFunctionType()));
@@ -326,7 +370,7 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder,
calleeType, operandsRef.front(),
moduleTranslation.lookupBlock(invOp.getSuccessor(0)),
moduleTranslation.lookupBlock(invOp.getSuccessor(1)),
- operandsRef.drop_front());
+ operandsRef.drop_front(), opBundles);
}
result->setCallingConv(convertCConvToLLVM(invOp.getCConv()));
moduleTranslation.mapBranch(invOp, result);
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index 6670e4b186c397..82dccaa37aa634 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -218,7 +218,7 @@ func.func @store_unaligned_atomic(%val : f32, %ptr : !llvm.ptr) {
func.func @invalid_call() {
// expected-error at +1 {{'llvm.call' op must have either a `callee` attribute or at least an operand}}
- "llvm.call"() : () -> ()
+ "llvm.call"() {op_bundle_sizes = array<i32>} : () -> ()
llvm.return
}
@@ -286,7 +286,7 @@ func.func @call_non_llvm() {
func.func @call_non_llvm_arg(%arg0 : tensor<*xi32>) {
// expected-error at +1 {{'llvm.call' op operand #0 must be variadic of LLVM dialect-compatible type}}
- "llvm.call"(%arg0) : (tensor<*xi32>) -> ()
+ "llvm.call"(%arg0) {operandSegmentSizes = array<i32: 1, 0>, op_bundle_sizes = array<i32>} : (tensor<*xi32>) -> ()
llvm.return
}
@@ -1588,7 +1588,7 @@ llvm.func @variadic(...)
llvm.func @invalid_variadic_call(%arg: i32) {
// expected-error at +1 {{missing var_callee_type attribute for vararg call}}
- "llvm.call"(%arg) <{callee = @variadic}> : (i32) -> ()
+ "llvm.call"(%arg) <{callee = @variadic}> {operandSegmentSizes = array<i32: 1, 0>, op_bundle_sizes = array<i32>} : (i32) -> ()
llvm.return
}
@@ -1598,7 +1598,7 @@ llvm.func @variadic(...)
llvm.func @invalid_variadic_call(%arg: i32) {
// expected-error at +1 {{missing var_callee_type attribute for vararg call}}
- "llvm.call"(%arg) <{callee = @variadic}> : (i32) -> ()
+ "llvm.call"(%arg) <{callee = @variadic}> {operandSegmentSizes = array<i32: 1, 0>, op_bundle_sizes = array<i32>} : (i32) -> ()
llvm.return
}
@@ -1655,3 +1655,35 @@ llvm.func @alwaysinline_noinline() attributes { always_inline, no_inline } {
llvm.func @optnone_requires_noinline() attributes { optimize_none } {
llvm.return
}
+
+// -----
+
+llvm.func @foo()
+llvm.func @no_bundle_tags() {
+ %0 = llvm.mlir.constant(0 : i32) : i32
+ // expected-error at +1 {{expected operand bundle tags}}
+ llvm.call @foo() bundlearg((%0)) : () -> () bundletype((i32))
+ llvm.return
+}
+
+// -----
+
+llvm.func @foo()
+llvm.func @wrong_number_of_bundle_tags() {
+ %0 = llvm.mlir.constant(0 : i32) : i32
+ %1 = llvm.mlir.constant(0 : i32) : i32
+ %2 = llvm.mlir.constant(0 : i32) : i32
+ // expected-error at +1 {{expected 2 operand bundle tags, but actually got 1}}
+ llvm.call @foo() bundlearg((%0, %1), (%2)) bundletag(["tag"]) : () -> () bundletype((i32, i32), (i32))
+ llvm.return
+}
+
+// -----
+
+llvm.func @foo()
+llvm.func @wrong_number_of_bundle_types() {
+ %0 = llvm.mlir.constant(0 : i32) : i32
+ // expected-error at +1 {{expected 1 types for operand bundle operands, but actually got 2}}
+ llvm.call @foo() bundlearg((%0)) : () -> () bundletype((i32, i32))
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index 7eca1a40373054..4b3525367b2ffa 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -2621,3 +2621,52 @@ llvm.func @reqd_work_group_size() attributes {reqd_work_group_size = array<i32:
llvm.func @intel_reqd_sub_group_size() attributes {intel_reqd_sub_group_size = 32 : i32}
// CHECK: ![[#INTEL_REQD_SUB_GROUP_SIZE]] = !{i32 32}
+
+// -----
+
+llvm.func @foo()
+
+llvm.func @call_with_opbundle() {
+ %0 = llvm.mlir.constant(1 : i32) : i32
+ %1 = llvm.mlir.constant(2 : i32) : i32
+ %2 = llvm.mlir.constant(3 : i32) : i32
+ llvm.call @foo() bundlearg((%0, %1), (%2)) bundletag(["tag1", "tag2"]) : () -> () bundletype((i32, i32), (i32))
+ llvm.return
+}
+
+// CHECK: define void @call_with_opbundle() {
+// CHECK-NEXT: call void @foo() [ "tag1"(i32 1, i32 2), "tag2"(i32 3) ]
+// CHECK-NEXT: ret void
+// CHECK-NEXT: }
+
+llvm.func @__gxx_personality_v0(...) -> i32
+llvm.func @invoke_with_opbundle() attributes { personality = @__gxx_personality_v0 } {
+ %0 = llvm.mlir.constant(1 : i32) : i32
+ %1 = llvm.mlir.constant(2 : i32) : i32
+ %2 = llvm.mlir.constant(3 : i32) : i32
+ llvm.invoke @foo() to ^bb2 unwind ^bb1 bundlearg((%0, %1), (%2)) bundletag(["tag1", "tag2"]) : () -> () bundletype((i32, i32), (i32))
+
+^bb1:
+ %3 = llvm.landingpad cleanup : !llvm.struct<(ptr, i32)>
+ llvm.return
+
+^bb2:
+ llvm.return
+}
+
+// CHECK: define void @invoke_with_opbundle() personality ptr @__gxx_personality_v0 {
+// CHECK-NEXT: invoke void @foo() [ "tag1"(i32 1, i32 2), "tag2"(i32 3) ]
+// CHECK-NEXT: to label %{{.+}} unwind label %{{.+}}
+// CHECK: }
+
+llvm.func @call_intrin_with_opbundle(%arg0 : !llvm.ptr) {
+ %0 = llvm.mlir.constant(1 : i1) : i1
+ %1 = llvm.mlir.constant(16 : i32) : i32
+ llvm.call_intrinsic "llvm.assume"(%0) bundlearg((%arg0, %1)) bundletag(["align"]) : (i1) -> () bundletype((!llvm.ptr, i32))
+ llvm.return
+}
+
+// CHECK: define void @call_intrin_with_opbundle(ptr %0) {
+// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr %0, i32 16) ]
+// CHECK-NEXT: ret void
+// CHECK-NEXT: }
More information about the Mlir-commits
mailing list