[Mlir-commits] [mlir] 48e9ef4 - [mlir] Give each OpResult its own use list
River Riddle
llvmlistbot at llvm.org
Thu Apr 23 16:32:34 PDT 2020
Author: River Riddle
Date: 2020-04-23T16:28:55-07:00
New Revision: 48e9ef4320a315ba1f2358db7295d53917a14f4b
URL: https://github.com/llvm/llvm-project/commit/48e9ef4320a315ba1f2358db7295d53917a14f4b
DIFF: https://github.com/llvm/llvm-project/commit/48e9ef4320a315ba1f2358db7295d53917a14f4b.diff
LOG: [mlir] Give each OpResult its own use list
This revision removes the multi use-list to ensure that each result gets its own. This decision was made by doing some extensive benchmarking of programs that actually use multiple results. This results in a size increase of 1-word per result >1, but the common case of 1-result remains unaffected. A side benefit is that 0-result operations now shrink by 1-word.
Differential Revision: https://reviews.llvm.org/D78701
Added:
Modified:
mlir/include/mlir/IR/Operation.h
mlir/include/mlir/IR/OperationSupport.h
mlir/include/mlir/IR/UseDefLists.h
mlir/include/mlir/IR/Value.h
mlir/lib/IR/Operation.cpp
mlir/lib/IR/OperationSupport.cpp
mlir/lib/IR/Value.cpp
Removed:
################################################################################
diff --git a/mlir/include/mlir/IR/Operation.h b/mlir/include/mlir/IR/Operation.h
index f0be03a5cced..f715ac52b129 100644
--- a/mlir/include/mlir/IR/Operation.h
+++ b/mlir/include/mlir/IR/Operation.h
@@ -25,11 +25,10 @@ namespace mlir {
/// operations are organized into operation blocks represented by a 'Block'
/// class.
class Operation final
- : public IRMultiObjectWithUseList<OpOperand>,
- public llvm::ilist_node_with_parent<Operation, Block>,
- private llvm::TrailingObjects<Operation, detail::TrailingOpResult,
- BlockOperand, Region,
- detail::OperandStorage> {
+ : public llvm::ilist_node_with_parent<Operation, Block>,
+ private llvm::TrailingObjects<Operation, detail::InLineOpResult,
+ detail::TrailingOpResult, BlockOperand,
+ Region, detail::OperandStorage> {
public:
/// Create a new Operation with the specific fields.
static Operation *create(Location location, OperationName name,
@@ -489,6 +488,74 @@ class Operation final
return detail::walkOperations(this, std::forward<FnT>(callback));
}
+ //===--------------------------------------------------------------------===//
+ // Uses
+ //===--------------------------------------------------------------------===//
+
+ /// Drop all uses of results of this operation.
+ void dropAllUses() {
+ for (OpResult result : getOpResults())
+ result.dropAllUses();
+ }
+
+ /// This class implements a use iterator for the Operation. This iterates over
+ /// all uses of all results.
+ class UseIterator final
+ : public llvm::iterator_facade_base<
+ UseIterator, std::forward_iterator_tag, OpOperand> {
+ public:
+ /// Initialize UseIterator for op, specify end to return iterator to last
+ /// use.
+ explicit UseIterator(Operation *op, bool end = false);
+
+ UseIterator &operator++();
+ OpOperand *operator->() const { return use.getOperand(); }
+ OpOperand &operator*() const { return *use.getOperand(); }
+
+ bool operator==(const UseIterator &rhs) const { return use == rhs.use; }
+ bool operator!=(const UseIterator &rhs) const { return !(*this == rhs); }
+
+ private:
+ void skipOverResultsWithNoUsers();
+
+ /// The operation whose uses are being iterated over.
+ Operation *op;
+ /// The result of op who's uses are being iterated over.
+ Operation::result_iterator res;
+ /// The use of the result.
+ Value::use_iterator use;
+ };
+ using use_iterator = UseIterator;
+ using use_range = iterator_range<use_iterator>;
+
+ use_iterator use_begin() { return use_iterator(this); }
+ use_iterator use_end() { return use_iterator(this, /*end=*/true); }
+
+ /// Returns a range of all uses, which is useful for iterating over all uses.
+ use_range getUses() { return {use_begin(), use_end()}; }
+
+ /// Returns true if this operation has exactly one use.
+ bool hasOneUse() { return llvm::hasSingleElement(getUses()); }
+
+ /// Returns true if this operation has no uses.
+ bool use_empty() {
+ return llvm::all_of(getOpResults(),
+ [](OpResult result) { return result.use_empty(); });
+ }
+
+ //===--------------------------------------------------------------------===//
+ // Users
+ //===--------------------------------------------------------------------===//
+
+ using user_iterator = ValueUserIterator<use_iterator, OpOperand>;
+ using user_range = iterator_range<user_iterator>;
+
+ user_iterator user_begin() { return user_iterator(use_begin()); }
+ user_iterator user_end() { return user_iterator(use_end()); }
+
+ /// Returns a range of all users.
+ user_range getUsers() { return {user_begin(), user_end()}; }
+
//===--------------------------------------------------------------------===//
// Other
//===--------------------------------------------------------------------===//
@@ -543,13 +610,14 @@ class Operation final
return *getTrailingObjects<detail::OperandStorage>();
}
- /// Returns a raw pointer to the storage for the given trailing result. The
- /// given result number should be 0-based relative to the trailing results,
- /// and not all of the results of the operation. This method should generally
- /// only be used by the 'Value' classes.
- detail::TrailingOpResult *getTrailingResult(unsigned trailingResultNumber) {
- return getTrailingObjects<detail::TrailingOpResult>() +
- trailingResultNumber;
+ /// Returns a pointer to the use list for the given trailing result.
+ detail::TrailingOpResult *getTrailingResult(unsigned resultNumber) {
+ return getTrailingObjects<detail::TrailingOpResult>() + resultNumber;
+ }
+
+ /// Returns a pointer to the use list for the given inline result.
+ detail::InLineOpResult *getInlineResult(unsigned resultNumber) {
+ return getTrailingObjects<detail::InLineOpResult>() + resultNumber;
}
/// Provide a 'getParent' method for ilist_node_with_parent methods.
@@ -595,15 +663,20 @@ class Operation final
// allow block to access the 'orderIndex' field.
friend class Block;
- // allow value to access the 'getTrailingResult' method.
+ // allow value to access the 'ResultStorage' methods.
friend class Value;
// allow ilist_node_with_parent to access the 'getParent' method.
friend class llvm::ilist_node_with_parent<Operation, Block>;
// This stuff is used by the TrailingObjects template.
- friend llvm::TrailingObjects<Operation, detail::TrailingOpResult,
- BlockOperand, Region, detail::OperandStorage>;
+ friend llvm::TrailingObjects<Operation, detail::InLineOpResult,
+ detail::TrailingOpResult, BlockOperand, Region,
+ detail::OperandStorage>;
+ size_t numTrailingObjects(OverloadToken<detail::InLineOpResult>) const {
+ return OpResult::getNumInline(
+ const_cast<Operation *>(this)->getNumResults());
+ }
size_t numTrailingObjects(OverloadToken<detail::TrailingOpResult>) const {
return OpResult::getNumTrailing(
const_cast<Operation *>(this)->getNumResults());
diff --git a/mlir/include/mlir/IR/OperationSupport.h b/mlir/include/mlir/IR/OperationSupport.h
index 35c5b017ab89..e5dd2827595b 100644
--- a/mlir/include/mlir/IR/OperationSupport.h
+++ b/mlir/include/mlir/IR/OperationSupport.h
@@ -467,14 +467,31 @@ class OperandStorage final
} // end namespace detail
//===----------------------------------------------------------------------===//
-// TrailingOpResult
+// ResultStorage
//===----------------------------------------------------------------------===//
namespace detail {
-/// This class provides the implementation for a trailing operation result.
-struct TrailingOpResult {
- /// The only element is the trailing result number, or the offset from the
- /// beginning of the trailing array.
+/// This class provides the implementation for an in-line operation result. This
+/// is an operation result whose number can be stored inline inside of the bits
+/// of an Operation*.
+struct InLineOpResult : public IRObjectWithUseList<OpOperand> {};
+/// This class provides the implementation for an out-of-line operation result.
+/// This is an operation result whose number cannot be stored inline inside of
+/// the bits of an Operation*.
+struct TrailingOpResult : public IRObjectWithUseList<OpOperand> {
+ TrailingOpResult(uint64_t trailingResultNumber)
+ : trailingResultNumber(trailingResultNumber) {}
+
+ /// Returns the parent operation of this trailing result.
+ Operation *getOwner();
+
+ /// Return the proper result number of this op result.
+ unsigned getResultNumber() {
+ return trailingResultNumber + OpResult::getMaxInlineResults();
+ }
+
+ /// The trailing result number, or the offset from the beginning of the
+ /// trailing array.
uint64_t trailingResultNumber;
};
} // end namespace detail
diff --git a/mlir/include/mlir/IR/UseDefLists.h b/mlir/include/mlir/IR/UseDefLists.h
index c07c6c5e40a5..06df9edbfa8b 100644
--- a/mlir/include/mlir/IR/UseDefLists.h
+++ b/mlir/include/mlir/IR/UseDefLists.h
@@ -99,86 +99,6 @@ template <typename OperandType> class IRObjectWithUseList {
OperandType *firstUse = nullptr;
};
-//===----------------------------------------------------------------------===//
-// IRMultiObjectWithUseList
-//===----------------------------------------------------------------------===//
-
-/// This class represents multiple IR objects with a single use list. This class
-/// provides wrapper functionality for manipulating the uses of a single object.
-template <typename OperandType>
-class IRMultiObjectWithUseList : public IRObjectWithUseList<OperandType> {
-public:
- using BaseType = IRObjectWithUseList<OperandType>;
- using ValueType = typename OperandType::ValueType;
-
- /// Drop all uses of `value` from their respective owners.
- void dropAllUses(ValueType value) {
- assert(this == OperandType::getUseList(value) &&
- "value not attached to this use list");
- for (OperandType &use : llvm::make_early_inc_range(getUses(value)))
- use.drop();
- }
- using BaseType::dropAllUses;
-
- /// Replace all uses of `oldValue` with the new value, updating anything in
- /// the IR that uses 'this' to use the other value instead. When this returns
- /// there are zero uses of 'this'.
- void replaceAllUsesWith(ValueType oldValue, ValueType newValue) {
- assert(this == OperandType::getUseList(oldValue) &&
- "value not attached to this use list");
- assert((!newValue || this != OperandType::getUseList(newValue)) &&
- "cannot RAUW a value with itself");
- for (OperandType &use : llvm::make_early_inc_range(getUses(oldValue)))
- use.set(newValue);
- }
- using BaseType::replaceAllUsesWith;
-
- //===--------------------------------------------------------------------===//
- // Uses
- //===--------------------------------------------------------------------===//
-
- using filtered_use_iterator = FilteredValueUseIterator<OperandType>;
- using filtered_use_range = iterator_range<filtered_use_iterator>;
-
- filtered_use_iterator use_begin(ValueType value) const {
- return filtered_use_iterator(this->getFirstUse(), value);
- }
- filtered_use_iterator use_end(ValueType) const { return use_end(); }
- filtered_use_range getUses(ValueType value) const {
- return {use_begin(value), use_end(value)};
- }
- bool hasOneUse(ValueType value) const {
- return llvm::hasSingleElement(getUses(value));
- }
- bool use_empty(ValueType value) const {
- return use_begin(value) == use_end(value);
- }
- using BaseType::getUses;
- using BaseType::hasOneUse;
- using BaseType::use_begin;
- using BaseType::use_empty;
- using BaseType::use_end;
-
- //===--------------------------------------------------------------------===//
- // Users
- //===--------------------------------------------------------------------===//
-
- using filtered_user_iterator =
- ValueUserIterator<filtered_use_iterator, OperandType>;
- using filtered_user_range = iterator_range<filtered_user_iterator>;
-
- filtered_user_iterator user_begin(ValueType value) const {
- return {use_begin(value)};
- }
- filtered_user_iterator user_end(ValueType value) const { return {use_end()}; }
- filtered_user_range getUsers(ValueType value) const {
- return {user_begin(value), user_end(value)};
- }
- using BaseType::getUsers;
- using BaseType::user_begin;
- using BaseType::user_end;
-};
-
//===----------------------------------------------------------------------===//
// IROperand
//===----------------------------------------------------------------------===//
@@ -209,6 +129,9 @@ template <typename DerivedT, typename IRValueTy> class IROperand {
insertIntoCurrent();
}
+ /// Returns true if this operand contains the given value.
+ bool is(ValueType other) const { return value == other; }
+
/// Return the owner of this operand.
Operation *getOwner() { return owner; }
Operation *getOwner() const { return owner; }
@@ -310,6 +233,7 @@ class OpaqueValue {
OpaqueValue(std::nullptr_t = nullptr) : impl(nullptr) {}
OpaqueValue(const OpaqueValue &) = default;
OpaqueValue &operator=(const OpaqueValue &) = default;
+ bool operator==(const OpaqueValue &other) const { return impl == other.impl; }
operator bool() const { return impl; }
/// Implicit conversion back to 'Value'.
@@ -320,7 +244,8 @@ class OpaqueValue {
};
} // namespace detail
-/// A reference to a value, suitable for use as an operand of an operation.
+/// This class represents an operand of an operation. Instances of this class
+/// contain a reference to a specific `Value`.
class OpOperand : public IROperand<OpOperand, detail::OpaqueValue> {
public:
using IROperand<OpOperand, detail::OpaqueValue>::IROperand;
@@ -342,34 +267,33 @@ class OpOperand : public IROperand<OpOperand, detail::OpaqueValue> {
// ValueUseIterator
//===----------------------------------------------------------------------===//
-namespace detail {
-/// A base iterator class that allows for iterating over the uses of a value.
-/// This is templated to allow for derived iterators to override specific
-/// iterator methods.
-template <typename DerivedT, typename OperandType>
-class ValueUseIteratorImpl
- : public llvm::iterator_facade_base<DerivedT, std::forward_iterator_tag,
+/// An iterator class that allows for iterating over the uses of an IR operand
+/// type.
+template <typename OperandType>
+class ValueUseIterator
+ : public llvm::iterator_facade_base<ValueUseIterator<OperandType>,
+ std::forward_iterator_tag,
OperandType> {
public:
- template <typename T>
- ValueUseIteratorImpl(const ValueUseIteratorImpl<T, OperandType> &other)
- : current(other.getOperand()) {}
- ValueUseIteratorImpl(OperandType *current = nullptr) : current(current) {}
+ ValueUseIterator(OperandType *current = nullptr) : current(current) {}
+ /// Returns the user that owns this use.
Operation *getUser() const { return current->getOwner(); }
- OperandType *getOperand() const { return current; }
+ /// Returns the current operands.
+ OperandType *getOperand() const { return current; }
OperandType &operator*() const { return *current; }
- using llvm::iterator_facade_base<DerivedT, std::forward_iterator_tag,
+ using llvm::iterator_facade_base<ValueUseIterator<OperandType>,
+ std::forward_iterator_tag,
OperandType>::operator++;
- ValueUseIteratorImpl &operator++() {
+ ValueUseIterator &operator++() {
assert(current && "incrementing past end()!");
current = (OperandType *)current->getNextOperandUsingThisValue();
return *this;
}
- bool operator==(const ValueUseIteratorImpl &rhs) const {
+ bool operator==(const ValueUseIterator &rhs) const {
return current == rhs.current;
}
@@ -377,63 +301,12 @@ class ValueUseIteratorImpl
OperandType *current;
};
-} // end namespace detail
-
-/// An iterator over all of the uses of an IR object.
-template <typename OperandType>
-class ValueUseIterator
- : public detail::ValueUseIteratorImpl<ValueUseIterator<OperandType>,
- OperandType> {
-public:
- using detail::ValueUseIteratorImpl<ValueUseIterator<OperandType>,
- OperandType>::ValueUseIteratorImpl;
-};
-
-/// This class represents an iterator of the uses of a IR object that optionally
-/// filters on a specific sub-value. This allows for filtering the uses of an
-/// IRMultiObjectWithUseList.
-template <typename OperandType>
-class FilteredValueUseIterator
- : public detail::ValueUseIteratorImpl<FilteredValueUseIterator<OperandType>,
- OperandType> {
-public:
- using BaseT =
- detail::ValueUseIteratorImpl<FilteredValueUseIterator<OperandType>,
- OperandType>;
-
- FilteredValueUseIterator() = default;
- FilteredValueUseIterator(const ValueUseIterator<OperandType> &it)
- : BaseT(it), filterVal(nullptr) {}
- FilteredValueUseIterator(OperandType *current,
- typename OperandType::ValueType filterVal)
- : BaseT(current), filterVal(filterVal) {
- findNextValid();
- }
-
- using BaseT::operator++;
- FilteredValueUseIterator<OperandType> &operator++() {
- BaseT::operator++();
- findNextValid();
- return *this;
- }
-
-private:
- void findNextValid() {
- if (!filterVal)
- return;
- while (this->current && ((OperandType *)this->current)->get() != filterVal)
- BaseT::operator++();
- }
-
- /// An optional value to use to filter specific uses.
- typename OperandType::ValueType filterVal;
-};
-
//===----------------------------------------------------------------------===//
// ValueUserIterator
//===----------------------------------------------------------------------===//
-/// An iterator over all users of a ValueBase.
+/// An iterator over the users of an IRObject. This is a wrapper iterator around
+/// a specific use iterator.
template <typename UseIteratorT, typename OperandType>
class ValueUserIterator final
: public llvm::mapped_iterator<UseIteratorT,
@@ -444,7 +317,7 @@ class ValueUserIterator final
using pointer = Operation *;
using reference = Operation *;
- /// Initializes the result type iterator to the specified result iterator.
+ /// Initializes the user iterator to the specified use iterator.
ValueUserIterator(UseIteratorT it)
: llvm::mapped_iterator<UseIteratorT, Operation *(*)(OperandType &)>(
it, &unwrap) {}
diff --git a/mlir/include/mlir/IR/Value.h b/mlir/include/mlir/IR/Value.h
index b779287577b0..42001ae7db87 100644
--- a/mlir/include/mlir/IR/Value.h
+++ b/mlir/include/mlir/IR/Value.h
@@ -149,7 +149,7 @@ class Value {
// Uses
/// This class implements an iterator over the uses of a value.
- using use_iterator = FilteredValueUseIterator<OpOperand>;
+ using use_iterator = ValueUseIterator<OpOperand>;
using use_range = iterator_range<use_iterator>;
use_iterator use_begin() const;
@@ -300,12 +300,21 @@ class OpResult : public Value {
/// Returns the number of this result.
unsigned getResultNumber() const;
+ /// Returns the maximum number of results that can be stored inline.
+ static unsigned getMaxInlineResults() {
+ return static_cast<unsigned>(Kind::TrailingOpResult);
+ }
+
private:
+ /// Given a number of operation results, returns the number that need to be
+ /// stored inline.
+ static unsigned getNumInline(unsigned numResults);
+
/// Given a number of operation results, returns the number that need to be
/// stored as trailing.
static unsigned getNumTrailing(unsigned numResults);
- /// Allow access to `create` and `destroy`.
+ /// Allow access to constructor.
friend Operation;
};
diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp
index 0d8b28185eb5..a2293af0ffdc 100644
--- a/mlir/lib/IR/Operation.cpp
+++ b/mlir/lib/IR/Operation.cpp
@@ -103,14 +103,16 @@ Operation *Operation::create(Location location, OperationName name,
bool resizableOperandList) {
// We only need to allocate additional memory for a subset of results.
unsigned numTrailingResults = OpResult::getNumTrailing(resultTypes.size());
+ unsigned numInlineResults = OpResult::getNumInline(resultTypes.size());
unsigned numSuccessors = successors.size();
unsigned numOperands = operands.size();
// Compute the byte size for the operation and the operand storage.
- auto byteSize = totalSizeToAlloc<detail::TrailingOpResult, BlockOperand,
- Region, detail::OperandStorage>(
- numTrailingResults, numSuccessors, numRegions,
- /*detail::OperandStorage*/ 1);
+ auto byteSize =
+ totalSizeToAlloc<detail::InLineOpResult, detail::TrailingOpResult,
+ BlockOperand, Region, detail::OperandStorage>(
+ numInlineResults, numTrailingResults, numSuccessors, numRegions,
+ /*detail::OperandStorage*/ 1);
byteSize += llvm::alignTo(detail::OperandStorage::additionalAllocSize(
numOperands, resizableOperandList),
alignof(Operation));
@@ -123,16 +125,11 @@ Operation *Operation::create(Location location, OperationName name,
assert((numSuccessors == 0 || !op->isKnownNonTerminator()) &&
"unexpected successors in a non-terminator operation");
- // Initialize the trailing results.
- if (LLVM_UNLIKELY(numTrailingResults > 0)) {
- // We initialize the trailing results with their result number. This makes
- // 'getResultNumber' checks much more efficient. The main purpose for these
- // results is to give an anchor to the main operation anyways, so this is
- // purely an optimization.
- auto *trailingResultIt = op->getTrailingObjects<detail::TrailingOpResult>();
- for (unsigned i = 0; i != numTrailingResults; ++i, ++trailingResultIt)
- trailingResultIt->trailingResultNumber = i;
- }
+ // Initialize the results.
+ for (unsigned i = 0; i < numInlineResults; ++i)
+ new (op->getInlineResult(i)) detail::InLineOpResult();
+ for (unsigned i = 0; i < numTrailingResults; ++i)
+ new (op->getTrailingResult(i)) detail::TrailingOpResult(i);
// Initialize the regions.
for (unsigned i = 0; i != numRegions; ++i)
@@ -1072,3 +1069,38 @@ void impl::ensureRegionTerminator(
block.push_back(buildTerminatorOp());
}
+
+//===----------------------------------------------------------------------===//
+// UseIterator
+//===----------------------------------------------------------------------===//
+
+Operation::UseIterator::UseIterator(Operation *op, bool end)
+ : op(op), res(end ? op->result_end() : op->result_begin()) {
+ // Only initialize current use if there are results/can be uses.
+ if (op->getNumResults())
+ skipOverResultsWithNoUsers();
+}
+
+Operation::UseIterator &Operation::UseIterator::operator++() {
+ // We increment over uses, if we reach the last use then move to next
+ // result.
+ if (use != (*res).use_end())
+ ++use;
+ if (use == (*res).use_end()) {
+ ++res;
+ skipOverResultsWithNoUsers();
+ }
+ return *this;
+}
+
+void Operation::UseIterator::skipOverResultsWithNoUsers() {
+ while (res != op->result_end() && (*res).use_empty())
+ ++res;
+
+ // If we are at the last result, then set use to first use of
+ // first result (sentinel value used for end).
+ if (res == op->result_end())
+ use = {};
+ else
+ use = (*res).use_begin();
+}
diff --git a/mlir/lib/IR/OperationSupport.cpp b/mlir/lib/IR/OperationSupport.cpp
index 9994a4f03379..70f75f6338bf 100644
--- a/mlir/lib/IR/OperationSupport.cpp
+++ b/mlir/lib/IR/OperationSupport.cpp
@@ -132,6 +132,22 @@ void detail::OperandStorage::grow(ResizableStorage &resizeUtil,
resizeUtil.setDynamicStorage(newStorage);
}
+//===----------------------------------------------------------------------===//
+// ResultStorage
+//===----------------------------------------------------------------------===//
+
+/// Returns the parent operation of this trailing result.
+Operation *detail::TrailingOpResult::getOwner() {
+ // We need to do some arithmetic to get the operation pointer. Move the
+ // trailing owner to the start of the array.
+ TrailingOpResult *trailingIt = this - trailingResultNumber;
+
+ // Move the owner past the inline op results to get to the operation.
+ auto *inlineResultIt = reinterpret_cast<InLineOpResult *>(trailingIt) -
+ OpResult::getMaxInlineResults();
+ return reinterpret_cast<Operation *>(inlineResultIt) - 1;
+}
+
//===----------------------------------------------------------------------===//
// Operation Value-Iterators
//===----------------------------------------------------------------------===//
diff --git a/mlir/lib/IR/Value.cpp b/mlir/lib/IR/Value.cpp
index af60ce1dd2bb..f011c1341769 100644
--- a/mlir/lib/IR/Value.cpp
+++ b/mlir/lib/IR/Value.cpp
@@ -11,10 +11,12 @@
#include "mlir/IR/Operation.h"
#include "mlir/IR/StandardTypes.h"
#include "llvm/ADT/SmallPtrSet.h"
+
using namespace mlir;
+using namespace mlir::detail;
/// Construct a value.
-Value::Value(detail::BlockArgumentImpl *impl)
+Value::Value(BlockArgumentImpl *impl)
: ownerAndKind(impl, Kind::BlockArgument) {}
Value::Value(Operation *op, unsigned resultNo) {
assert(op->getNumResults() > resultNo && "invalid result number");
@@ -23,12 +25,9 @@ Value::Value(Operation *op, unsigned resultNo) {
return;
}
- // If we can't pack the result directly, we need to represent this as a
- // trailing result.
- unsigned trailingResultNo =
- resultNo - static_cast<unsigned>(Kind::TrailingOpResult);
- ownerAndKind = {op->getTrailingResult(trailingResultNo),
- Kind::TrailingOpResult};
+ // If we can't pack the result directly, grab the use list from the parent op.
+ unsigned trailingNo = resultNo - OpResult::getMaxInlineResults();
+ ownerAndKind = {op->getTrailingResult(trailingNo), Kind::TrailingOpResult};
}
/// Return the type of this value.
@@ -96,30 +95,23 @@ Region *Value::getParentRegion() {
IRObjectWithUseList<OpOperand> *Value::getUseList() const {
if (BlockArgument arg = dyn_cast<BlockArgument>())
return arg.getImpl();
- return cast<OpResult>().getOwner();
+ if (getKind() != Kind::TrailingOpResult) {
+ OpResult result = cast<OpResult>();
+ return result.getOwner()->getInlineResult(result.getResultNumber());
+ }
+
+ // Otherwise this is a trailing operation result, which contains a use list.
+ return reinterpret_cast<TrailingOpResult *>(ownerAndKind.getPointer());
}
/// Drop all uses of this object from their respective owners.
-void Value::dropAllUses() const {
- if (BlockArgument arg = dyn_cast<BlockArgument>())
- return arg.getImpl()->dropAllUses();
- Operation *owner = cast<OpResult>().getOwner();
- if (owner->hasSingleResult)
- return owner->dropAllUses();
- return owner->dropAllUses(*this);
-}
+void Value::dropAllUses() const { return getUseList()->dropAllUses(); }
/// Replace all uses of 'this' value with the new value, updating anything in
/// the IR that uses 'this' to use the other value instead. When this returns
/// there are zero uses of 'this'.
void Value::replaceAllUsesWith(Value newValue) const {
- if (BlockArgument arg = dyn_cast<BlockArgument>())
- return arg.getImpl()->replaceAllUsesWith(newValue);
- Operation *owner = cast<OpResult>().getOwner();
- IRMultiObjectWithUseList<OpOperand> *useList = owner;
- if (owner->hasSingleResult)
- return useList->replaceAllUsesWith(newValue);
- useList->replaceAllUsesWith(*this, newValue);
+ return getUseList()->replaceAllUsesWith(newValue);
}
/// Replace all uses of 'this' value with the new value, updating anything in
@@ -137,28 +129,14 @@ void Value::replaceAllUsesExcept(
// Uses
auto Value::use_begin() const -> use_iterator {
- if (BlockArgument arg = dyn_cast<BlockArgument>())
- return arg.getImpl()->use_begin();
- Operation *owner = cast<OpResult>().getOwner();
- return owner->hasSingleResult ? use_iterator(owner->use_begin())
- : owner->use_begin(*this);
+ return getUseList()->use_begin();
}
/// Returns true if this value has exactly one use.
-bool Value::hasOneUse() const {
- if (BlockArgument arg = dyn_cast<BlockArgument>())
- return arg.getImpl()->hasOneUse();
- Operation *owner = cast<OpResult>().getOwner();
- return owner->hasSingleResult ? owner->hasOneUse() : owner->hasOneUse(*this);
-}
+bool Value::hasOneUse() const { return getUseList()->hasOneUse(); }
/// Returns true if this value has no uses.
-bool Value::use_empty() const {
- if (BlockArgument arg = dyn_cast<BlockArgument>())
- return arg.getImpl()->use_empty();
- Operation *owner = cast<OpResult>().getOwner();
- return owner->hasSingleResult ? owner->use_empty() : owner->use_empty(*this);
-}
+bool Value::use_empty() const { return getUseList()->use_empty(); }
//===----------------------------------------------------------------------===//
// OpResult
@@ -167,18 +145,12 @@ bool Value::use_empty() const {
/// Returns the operation that owns this result.
Operation *OpResult::getOwner() const {
// If the result is in-place, the `owner` is the operation.
+ void *owner = ownerAndKind.getPointer();
if (LLVM_LIKELY(getKind() != Kind::TrailingOpResult))
- return reinterpret_cast<Operation *>(ownerAndKind.getPointer());
+ return static_cast<Operation *>(owner);
- // Otherwise, we need to do some arithmetic to get the operation pointer.
- // Move the trailing owner to the start of the array.
- auto *trailingIt =
- static_cast<detail::TrailingOpResult *>(ownerAndKind.getPointer());
- trailingIt -= trailingIt->trailingResultNumber;
-
- // This point is the first trailing object after the operation. So all we need
- // to do here is adjust for the operation size.
- return reinterpret_cast<Operation *>(trailingIt) - 1;
+ // Otherwise, query the trailing result for the owner.
+ return static_cast<TrailingOpResult *>(owner)->getOwner();
}
/// Return the result number of this result.
@@ -186,20 +158,23 @@ unsigned OpResult::getResultNumber() const {
// If the result is in-place, we can use the kind directly.
if (LLVM_LIKELY(getKind() != Kind::TrailingOpResult))
return static_cast<unsigned>(ownerAndKind.getInt());
- // Otherwise, we add the number of inline results to the trailing owner.
- auto *trailingIt =
- static_cast<detail::TrailingOpResult *>(ownerAndKind.getPointer());
- unsigned trailingNumber = trailingIt->trailingResultNumber;
- return trailingNumber + static_cast<unsigned>(Kind::TrailingOpResult);
+ // Otherwise, query the trailing result.
+ auto *result = static_cast<TrailingOpResult *>(ownerAndKind.getPointer());
+ return result->getResultNumber();
+}
+
+/// Given a number of operation results, returns the number that need to be
+/// stored inline.
+unsigned OpResult::getNumInline(unsigned numResults) {
+ return std::min(numResults, getMaxInlineResults());
}
/// Given a number of operation results, returns the number that need to be
/// stored as trailing.
unsigned OpResult::getNumTrailing(unsigned numResults) {
// If we can pack all of the results, there is no need for additional storage.
- if (numResults <= static_cast<unsigned>(Kind::TrailingOpResult))
- return 0;
- return numResults - static_cast<unsigned>(Kind::TrailingOpResult);
+ unsigned maxInline = getMaxInlineResults();
+ return numResults <= maxInline ? 0 : numResults - maxInline;
}
//===----------------------------------------------------------------------===//
@@ -227,12 +202,12 @@ IRObjectWithUseList<OpOperand> *OpOperand::getUseList(Value value) {
/// Return the current value being used by this operand.
Value OpOperand::get() const {
- return IROperand<OpOperand, detail::OpaqueValue>::get();
+ return IROperand<OpOperand, OpaqueValue>::get();
}
/// Set the operand to the given value.
void OpOperand::set(Value value) {
- IROperand<OpOperand, detail::OpaqueValue>::set(value);
+ IROperand<OpOperand, OpaqueValue>::set(value);
}
/// Return which operand this is in the operand list.
@@ -241,14 +216,13 @@ unsigned OpOperand::getOperandNumber() {
}
//===----------------------------------------------------------------------===//
-// detail::OpaqueValue
+// OpaqueValue
//===----------------------------------------------------------------------===//
/// Implicit conversion from 'Value'.
-detail::OpaqueValue::OpaqueValue(Value value)
- : impl(value.getAsOpaquePointer()) {}
+OpaqueValue::OpaqueValue(Value value) : impl(value.getAsOpaquePointer()) {}
/// Implicit conversion back to 'Value'.
-detail::OpaqueValue::operator Value() const {
+OpaqueValue::operator Value() const {
return Value::getFromOpaquePointer(impl);
}
More information about the Mlir-commits
mailing list