[llvm] r262892 - Revert "SelectionDAG: Store SDNode operands in an ArrayRecycler"
Justin Bogner via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 7 17:07:03 PST 2016
Author: bogner
Date: Mon Mar 7 19:07:03 2016
New Revision: 262892
URL: http://llvm.org/viewvc/llvm-project?rev=262892&view=rev
Log:
Revert "SelectionDAG: Store SDNode operands in an ArrayRecycler"
Looks like the largest SDNode is different between 32 and 64 bit now,
so this is breaking 32 bit bots. Reverting while I figure out a fix.
This reverts r262886.
Modified:
llvm/trunk/include/llvm/CodeGen/SelectionDAG.h
llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h
llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
Modified: llvm/trunk/include/llvm/CodeGen/SelectionDAG.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/SelectionDAG.h?rev=262892&r1=262891&r2=262892&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/SelectionDAG.h (original)
+++ llvm/trunk/include/llvm/CodeGen/SelectionDAG.h Mon Mar 7 19:07:03 2016
@@ -23,7 +23,6 @@
#include "llvm/CodeGen/DAGCombine.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
-#include "llvm/Support/ArrayRecycler.h"
#include "llvm/Support/RecyclingAllocator.h"
#include "llvm/Target/TargetMachine.h"
#include <cassert>
@@ -209,7 +208,6 @@ class SelectionDAG {
/// Pool allocation for machine-opcode SDNode operands.
BumpPtrAllocator OperandAllocator;
- ArrayRecycler<SDUse> OperandRecycler;
/// Pool allocation for misc. objects that are created once per SelectionDAG.
BumpPtrAllocator Allocator;
@@ -276,30 +274,6 @@ private:
SDNodeT(std::forward<ArgTypes>(Args)...);
}
- void createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
- assert(!Node->OperandList && "Node already has operands");
- SDUse *Ops = OperandRecycler.allocate(
- ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
-
- for (unsigned I = 0; I != Vals.size(); ++I) {
- Ops[I].setUser(Node);
- Ops[I].setInitial(Vals[I]);
- }
- Node->NumOperands = Vals.size();
- Node->OperandList = Ops;
- checkForCycles(Node);
- }
-
- void removeOperands(SDNode *Node) {
- if (!Node->OperandList)
- return;
- OperandRecycler.deallocate(
- ArrayRecycler<SDUse>::Capacity::get(Node->NumOperands),
- Node->OperandList);
- Node->NumOperands = 0;
- Node->OperandList = nullptr;
- }
-
void operator=(const SelectionDAG&) = delete;
SelectionDAG(const SelectionDAG&) = delete;
@@ -1342,8 +1316,9 @@ private:
void allnodes_clear();
- SDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1,
- SDValue N2, const SDNodeFlags *Flags = nullptr);
+ BinarySDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
+ SDValue N1, SDValue N2,
+ const SDNodeFlags *Flags = nullptr);
/// Look up the node specified by ID in CSEMap. If it exists, return it. If
/// not, return the insertion token that will make insertion faster. This
Modified: llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h?rev=262892&r1=262891&r2=262892&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h (original)
+++ llvm/trunk/include/llvm/CodeGen/SelectionDAGNodes.h Mon Mar 7 19:07:03 2016
@@ -44,7 +44,7 @@ class GlobalValue;
class MachineBasicBlock;
class MachineConstantPoolValue;
class SDNode;
-class HandleSDNode;
+class BinaryWithFlagsSDNode;
class Value;
class MCSymbol;
template <typename T> struct DenseMapInfo;
@@ -280,8 +280,6 @@ public:
private:
friend class SelectionDAG;
friend class SDNode;
- // TODO: unfriend HandleSDNode once we fix its operand handling.
- friend class HandleSDNode;
void setUser(SDNode *p) { User = p; }
@@ -396,6 +394,10 @@ private:
/// The operation that this node performs.
int16_t NodeType;
+ /// This is true if OperandList was new[]'d. If true,
+ /// then they will be delete[]'d when the node is destroyed.
+ uint16_t OperandsNeedDelete : 1;
+
/// This tracks whether this node has one or more dbg_value
/// nodes corresponding to it.
uint16_t HasDebugValue : 1;
@@ -404,7 +406,7 @@ protected:
/// This member is defined by this class, but is not used for
/// anything. Subclasses can use it to hold whatever state they find useful.
/// This field is initialized to zero by the ctor.
- uint16_t SubclassData : 15;
+ uint16_t SubclassData : 14;
private:
/// Unique id per SDNode in the DAG.
@@ -438,8 +440,6 @@ private:
friend class SelectionDAG;
friend struct ilist_traits<SDNode>;
- // TODO: unfriend HandleSDNode once we fix its operand handling.
- friend class HandleSDNode;
public:
/// Unique and persistent id per SDNode in the DAG.
@@ -792,20 +792,101 @@ protected:
return Ret;
}
- /// Create an SDNode.
- ///
- /// SDNodes are created without any operands, and never own the operand
- /// storage. To add operands, see SelectionDAG::createOperands.
+ SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
+ ArrayRef<SDValue> Ops)
+ : NodeType(Opc), OperandsNeedDelete(true), HasDebugValue(false),
+ SubclassData(0), NodeId(-1),
+ OperandList(Ops.size() ? new SDUse[Ops.size()] : nullptr),
+ ValueList(VTs.VTs), UseList(nullptr), NumOperands(Ops.size()),
+ NumValues(VTs.NumVTs), IROrder(Order), debugLoc(std::move(dl)) {
+ assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
+ assert(NumOperands == Ops.size() &&
+ "NumOperands wasn't wide enough for its operands!");
+ assert(NumValues == VTs.NumVTs &&
+ "NumValues wasn't wide enough for its operands!");
+ for (unsigned i = 0; i != Ops.size(); ++i) {
+ assert(OperandList && "no operands available");
+ OperandList[i].setUser(this);
+ OperandList[i].setInitial(Ops[i]);
+ }
+ checkForCycles(this);
+ }
+
+ /// This constructor adds no operands itself; operands can be
+ /// set later with InitOperands.
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
- : NodeType(Opc), HasDebugValue(false), SubclassData(0), NodeId(-1),
- OperandList(nullptr), ValueList(VTs.VTs), UseList(nullptr),
- NumOperands(0), NumValues(VTs.NumVTs), IROrder(Order),
- debugLoc(std::move(dl)) {
+ : NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false),
+ SubclassData(0), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs),
+ UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs),
+ IROrder(Order), debugLoc(std::move(dl)) {
assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
assert(NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!");
}
+ /// Initialize the operands list of this with 1 operand.
+ void InitOperands(SDUse *Ops, const SDValue &Op0) {
+ Ops[0].setUser(this);
+ Ops[0].setInitial(Op0);
+ NumOperands = 1;
+ OperandList = Ops;
+ checkForCycles(this);
+ }
+
+ /// Initialize the operands list of this with 2 operands.
+ void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1) {
+ Ops[0].setUser(this);
+ Ops[0].setInitial(Op0);
+ Ops[1].setUser(this);
+ Ops[1].setInitial(Op1);
+ NumOperands = 2;
+ OperandList = Ops;
+ checkForCycles(this);
+ }
+
+ /// Initialize the operands list of this with 3 operands.
+ void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1,
+ const SDValue &Op2) {
+ Ops[0].setUser(this);
+ Ops[0].setInitial(Op0);
+ Ops[1].setUser(this);
+ Ops[1].setInitial(Op1);
+ Ops[2].setUser(this);
+ Ops[2].setInitial(Op2);
+ NumOperands = 3;
+ OperandList = Ops;
+ checkForCycles(this);
+ }
+
+ /// Initialize the operands list of this with 4 operands.
+ void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1,
+ const SDValue &Op2, const SDValue &Op3) {
+ Ops[0].setUser(this);
+ Ops[0].setInitial(Op0);
+ Ops[1].setUser(this);
+ Ops[1].setInitial(Op1);
+ Ops[2].setUser(this);
+ Ops[2].setInitial(Op2);
+ Ops[3].setUser(this);
+ Ops[3].setInitial(Op3);
+ NumOperands = 4;
+ OperandList = Ops;
+ checkForCycles(this);
+ }
+
+ /// Initialize the operands list of this with N operands.
+ void InitOperands(SDUse *Ops, const SDValue *Vals, unsigned N) {
+ for (unsigned i = 0; i != N; ++i) {
+ Ops[i].setUser(this);
+ Ops[i].setInitial(Vals[i]);
+ }
+ NumOperands = N;
+ assert(NumOperands == N &&
+ "NumOperands wasn't wide enough for its operands!");
+ OperandList = Ops;
+ checkForCycles(this);
+ }
+
/// Release the operands and set this node to have zero operands.
void DropOperands();
};
@@ -931,6 +1012,30 @@ inline void SDUse::setNode(SDNode *N) {
if (N) N->addUse(*this);
}
+/// This class is used for single-operand SDNodes. This is solely
+/// to allow co-allocation of node operands with the node itself.
+class UnarySDNode : public SDNode {
+ SDUse Op;
+public:
+ UnarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
+ SDValue X)
+ : SDNode(Opc, Order, dl, VTs) {
+ InitOperands(&Op, X);
+ }
+};
+
+/// This class is used for two-operand SDNodes. This is solely
+/// to allow co-allocation of node operands with the node itself.
+class BinarySDNode : public SDNode {
+ SDUse Ops[2];
+public:
+ BinarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
+ SDValue X, SDValue Y)
+ : SDNode(Opc, Order, dl, VTs) {
+ InitOperands(Ops, X, Y);
+ }
+};
+
/// Returns true if the opcode is a binary operation with flags.
static bool isBinOpWithFlags(unsigned Opcode) {
switch (Opcode) {
@@ -955,17 +1060,30 @@ static bool isBinOpWithFlags(unsigned Op
/// This class is an extension of BinarySDNode
/// used from those opcodes that have associated extra flags.
-class BinaryWithFlagsSDNode : public SDNode {
+class BinaryWithFlagsSDNode : public BinarySDNode {
public:
SDNodeFlags Flags;
BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
- const SDNodeFlags &NodeFlags)
- : SDNode(Opc, Order, dl, VTs), Flags(NodeFlags) {}
+ SDValue X, SDValue Y, const SDNodeFlags &NodeFlags)
+ : BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags(NodeFlags) {}
static bool classof(const SDNode *N) {
return isBinOpWithFlags(N->getOpcode());
}
};
+/// This class is used for three-operand SDNodes. This is solely
+/// to allow co-allocation of node operands with the node itself.
+class TernarySDNode : public SDNode {
+ SDUse Ops[3];
+public:
+ TernarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
+ SDValue X, SDValue Y, SDValue Z)
+ : SDNode(Opc, Order, dl, VTs) {
+ InitOperands(Ops, X, Y, Z);
+ }
+};
+
+
/// This class is used to form a handle around another node that
/// is persistent and is updated across invocations of replaceAllUsesWith on its
/// operand. This node should be directly created by end-users and not added to
@@ -978,28 +1096,20 @@ public:
// HandleSDNodes are never inserted into the DAG, so they won't be
// auto-numbered. Use ID 65535 as a sentinel.
PersistentId = 0xffff;
-
- // Manually set up the operand list. This node type is special in that it's
- // always stack allocated and SelectionDAG does not manage its operands.
- // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
- // be so special.
- Op.setUser(this);
- Op.setInitial(X);
- NumOperands = 1;
- OperandList = &Op;
+ InitOperands(&Op, X);
}
~HandleSDNode();
const SDValue &getValue() const { return Op; }
};
-class AddrSpaceCastSDNode : public SDNode {
+class AddrSpaceCastSDNode : public UnarySDNode {
private:
unsigned SrcAddrSpace;
unsigned DestAddrSpace;
public:
- AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, unsigned SrcAS,
- unsigned DestAS);
+ AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, SDValue X,
+ unsigned SrcAS, unsigned DestAS);
unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
unsigned getDestAddressSpace() const { return DestAddrSpace; }
@@ -1023,6 +1133,9 @@ public:
MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
EVT MemoryVT, MachineMemOperand *MMO);
+ MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
+ ArrayRef<SDValue> Ops, EVT MemoryVT, MachineMemOperand *MMO);
+
bool readMem() const { return MMO->isLoad(); }
bool writeMem() const { return MMO->isStore(); }
@@ -1125,6 +1238,8 @@ public:
/// This is an SDNode representing atomic operations.
class AtomicSDNode : public MemSDNode {
+ SDUse Ops[4];
+
/// For cmpxchg instructions, the ordering requirements when a store does not
/// occur.
AtomicOrdering FailureOrdering;
@@ -1150,12 +1265,16 @@ class AtomicSDNode : public MemSDNode {
}
public:
- AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL,
- EVT MemVT, MachineMemOperand *MMO,
+ AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, EVT MemVT,
+ const SDValue* AllOps, SDUse *DynOps, unsigned NumOps,
+ MachineMemOperand *MMO,
AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
SynchronizationScope SynchScope)
- : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
+ : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
InitAtomic(SuccessOrdering, FailureOrdering, SynchScope);
+ assert((DynOps || NumOps <= array_lengthof(Ops)) &&
+ "Too many ops for internal storage!");
+ InitOperands(DynOps ? DynOps : Ops, AllOps, NumOps);
}
const SDValue &getBasePtr() const { return getOperand(1); }
@@ -1203,8 +1322,9 @@ public:
class MemIntrinsicSDNode : public MemSDNode {
public:
MemIntrinsicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
- EVT MemoryVT, MachineMemOperand *MMO)
- : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
+ ArrayRef<SDValue> Ops, EVT MemoryVT,
+ MachineMemOperand *MMO)
+ : MemSDNode(Opc, Order, dl, VTs, Ops, MemoryVT, MMO) {
SubclassData |= 1u << 13;
}
@@ -1227,15 +1347,20 @@ public:
/// An index of -1 is treated as undef, such that the code generator may put
/// any value in the corresponding element of the result.
class ShuffleVectorSDNode : public SDNode {
+ SDUse Ops[2];
+
// The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
// is freed when the SelectionDAG object is destroyed.
const int *Mask;
protected:
friend class SelectionDAG;
- ShuffleVectorSDNode(EVT VT, unsigned Order, DebugLoc dl, const int *M)
- : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
-
+ ShuffleVectorSDNode(EVT VT, unsigned Order, DebugLoc dl, SDValue N1,
+ SDValue N2, const int *M)
+ : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {
+ InitOperands(Ops, N1, N2);
+ }
public:
+
ArrayRef<int> getMask() const {
EVT VT = getValueType(0);
return makeArrayRef(Mask, VT.getVectorNumElements());
@@ -1666,11 +1791,13 @@ public:
};
class EHLabelSDNode : public SDNode {
+ SDUse Chain;
MCSymbol *Label;
friend class SelectionDAG;
- EHLabelSDNode(unsigned Order, DebugLoc dl, MCSymbol *L)
- : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) {}
-
+ EHLabelSDNode(unsigned Order, DebugLoc dl, SDValue ch, MCSymbol *L)
+ : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) {
+ InitOperands(&Chain, ch);
+ }
public:
MCSymbol *getLabel() const { return Label; }
@@ -1736,10 +1863,11 @@ class CvtRndSatSDNode : public SDNode {
ISD::CvtCode CvtCode;
friend class SelectionDAG;
explicit CvtRndSatSDNode(EVT VT, unsigned Order, DebugLoc dl,
- ISD::CvtCode Code)
- : SDNode(ISD::CONVERT_RNDSAT, Order, dl, getSDVTList(VT)), CvtCode(Code) {
+ ArrayRef<SDValue> Ops, ISD::CvtCode Code)
+ : SDNode(ISD::CONVERT_RNDSAT, Order, dl, getSDVTList(VT), Ops),
+ CvtCode(Code) {
+ assert(Ops.size() == 5 && "wrong number of operations");
}
-
public:
ISD::CvtCode getCvtCode() const { return CvtCode; }
@@ -1768,13 +1896,24 @@ public:
/// Base class for LoadSDNode and StoreSDNode
class LSBaseSDNode : public MemSDNode {
+ //! Operand array for load and store
+ /*!
+ \note Moving this array to the base class captures more
+ common functionality shared between LoadSDNode and
+ StoreSDNode
+ */
+ SDUse Ops[4];
public:
LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl,
+ SDValue *Operands, unsigned numOperands,
SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
MachineMemOperand *MMO)
: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
SubclassData |= AM << 2;
assert(getAddressingMode() == AM && "MemIndexedMode encoding error!");
+ InitOperands(Ops, Operands, numOperands);
+ assert((getOffset().getOpcode() == ISD::UNDEF || isIndexed()) &&
+ "Only indexed loads and stores have a non-undef offset operand");
}
const SDValue &getOffset() const {
@@ -1802,9 +1941,10 @@ public:
/// This class is used to represent ISD::LOAD nodes.
class LoadSDNode : public LSBaseSDNode {
friend class SelectionDAG;
- LoadSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM,
- ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO)
- : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
+ LoadSDNode(SDValue *ChainPtrOff, unsigned Order, DebugLoc dl, SDVTList VTs,
+ ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
+ MachineMemOperand *MMO)
+ : LSBaseSDNode(ISD::LOAD, Order, dl, ChainPtrOff, 3, VTs, AM, MemVT, MMO) {
SubclassData |= (unsigned short)ETy;
assert(getExtensionType() == ETy && "LoadExtType encoding error!");
assert(readMem() && "Load MachineMemOperand is not a load!");
@@ -1829,9 +1969,11 @@ public:
/// This class is used to represent ISD::STORE nodes.
class StoreSDNode : public LSBaseSDNode {
friend class SelectionDAG;
- StoreSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM,
- bool isTrunc, EVT MemVT, MachineMemOperand *MMO)
- : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
+ StoreSDNode(SDValue *ChainValuePtrOff, unsigned Order, DebugLoc dl,
+ SDVTList VTs, ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
+ MachineMemOperand *MMO)
+ : LSBaseSDNode(ISD::STORE, Order, dl, ChainValuePtrOff, 4,
+ VTs, AM, MemVT, MMO) {
SubclassData |= (unsigned short)isTrunc;
assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!");
assert(!readMem() && "Store MachineMemOperand is a load!");
@@ -1855,11 +1997,16 @@ public:
/// This base class is used to represent MLOAD and MSTORE nodes
class MaskedLoadStoreSDNode : public MemSDNode {
+ // Operands
+ SDUse Ops[4];
public:
friend class SelectionDAG;
MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl,
- SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
- : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {}
+ SDValue *Operands, unsigned numOperands, SDVTList VTs,
+ EVT MemVT, MachineMemOperand *MMO)
+ : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
+ InitOperands(Ops, Operands, numOperands);
+ }
// In the both nodes address is Op1, mask is Op2:
// MaskedLoadSDNode (Chain, ptr, mask, src0), src0 is a passthru value
@@ -1878,9 +2025,11 @@ public:
class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
public:
friend class SelectionDAG;
- MaskedLoadSDNode(unsigned Order, DebugLoc dl, SDVTList VTs,
- ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO)
- : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, MemVT, MMO) {
+ MaskedLoadSDNode(unsigned Order, DebugLoc dl, SDValue *Operands,
+ unsigned numOperands, SDVTList VTs, ISD::LoadExtType ETy,
+ EVT MemVT, MachineMemOperand *MMO)
+ : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, Operands, numOperands,
+ VTs, MemVT, MMO) {
SubclassData |= (unsigned short)ETy;
}
@@ -1898,10 +2047,12 @@ class MaskedStoreSDNode : public MaskedL
public:
friend class SelectionDAG;
- MaskedStoreSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, bool isTrunc,
- EVT MemVT, MachineMemOperand *MMO)
- : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) {
- SubclassData |= (unsigned short)isTrunc;
+ MaskedStoreSDNode(unsigned Order, DebugLoc dl, SDValue *Operands,
+ unsigned numOperands, SDVTList VTs, bool isTrunc, EVT MemVT,
+ MachineMemOperand *MMO)
+ : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, Operands, numOperands,
+ VTs, MemVT, MMO) {
+ SubclassData |= (unsigned short)isTrunc;
}
/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
@@ -1919,11 +2070,17 @@ public:
/// MGATHER and MSCATTER nodes
///
class MaskedGatherScatterSDNode : public MemSDNode {
+ // Operands
+ SDUse Ops[5];
public:
friend class SelectionDAG;
MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl,
- SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
- : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {}
+ ArrayRef<SDValue> Operands, SDVTList VTs, EVT MemVT,
+ MachineMemOperand *MMO)
+ : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
+ assert(Operands.size() == 5 && "Incompatible number of operands");
+ InitOperands(Ops, Operands.data(), Operands.size());
+ }
// In the both nodes address is Op1, mask is Op2:
// MaskedGatherSDNode (Chain, src0, mask, base, index), src0 is a passthru value
@@ -1945,9 +2102,19 @@ public:
class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
public:
friend class SelectionDAG;
- MaskedGatherSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, EVT MemVT,
- MachineMemOperand *MMO)
- : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO) {}
+ MaskedGatherSDNode(unsigned Order, DebugLoc dl, ArrayRef<SDValue> Operands,
+ SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
+ : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, Operands, VTs, MemVT,
+ MMO) {
+ assert(getValue().getValueType() == getValueType(0) &&
+ "Incompatible type of the PassThru value in MaskedGatherSDNode");
+ assert(getMask().getValueType().getVectorNumElements() ==
+ getValueType(0).getVectorNumElements() &&
+ "Vector width mismatch between mask and data");
+ assert(getIndex().getValueType().getVectorNumElements() ==
+ getValueType(0).getVectorNumElements() &&
+ "Vector width mismatch between index and data");
+ }
static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::MGATHER;
@@ -1960,9 +2127,17 @@ class MaskedScatterSDNode : public Maske
public:
friend class SelectionDAG;
- MaskedScatterSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, EVT MemVT,
- MachineMemOperand *MMO)
- : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO) {}
+ MaskedScatterSDNode(unsigned Order, DebugLoc dl,ArrayRef<SDValue> Operands,
+ SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
+ : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, Operands, VTs, MemVT,
+ MMO) {
+ assert(getMask().getValueType().getVectorNumElements() ==
+ getValue().getValueType().getVectorNumElements() &&
+ "Vector width mismatch between mask and data");
+ assert(getIndex().getValueType().getVectorNumElements() ==
+ getValue().getValueType().getVectorNumElements() &&
+ "Vector width mismatch between index and data");
+ }
static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::MSCATTER;
@@ -1981,6 +2156,10 @@ private:
MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc DL, SDVTList VTs)
: SDNode(Opc, Order, DL, VTs), MemRefs(nullptr), MemRefsEnd(nullptr) {}
+ /// Operands for this instruction, if they fit here. If
+ /// they don't, this field is unused.
+ SDUse LocalOperands[4];
+
/// Memory reference descriptions for this instruction.
mmo_iterator MemRefs;
mmo_iterator MemRefsEnd;
@@ -2056,7 +2235,7 @@ template <> struct GraphTraits<SDNode*>
};
/// The largest SDNode class.
-typedef AtomicSDNode LargestSDNode;
+typedef MaskedGatherScatterSDNode LargestSDNode;
/// The SDNode class with the greatest alignment requirement.
typedef GlobalAddressSDNode MostAlignedSDNode;
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp?rev=262892&r1=262891&r2=262892&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAG.cpp Mon Mar 7 19:07:03 2016
@@ -664,8 +664,8 @@ void SDDbgInfo::erase(const SDNode *Node
}
void SelectionDAG::DeallocateNode(SDNode *N) {
- // If we have operands, deallocate them.
- removeOperands(N);
+ if (N->OperandsNeedDelete)
+ delete[] N->OperandList;
// Set the opcode to DELETED_NODE to help catch bugs when node
// memory is reallocated.
@@ -912,7 +912,6 @@ void SelectionDAG::init(MachineFunction
SelectionDAG::~SelectionDAG() {
assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
allnodes_clear();
- OperandRecycler.clear(OperandAllocator);
delete DbgInfo;
}
@@ -926,26 +925,24 @@ void SelectionDAG::allnodes_clear() {
#endif
}
-SDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
- SDValue N1, SDValue N2,
- const SDNodeFlags *Flags) {
- SDValue Ops[] = {N1, N2};
-
+BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
+ SDVTList VTs, SDValue N1,
+ SDValue N2,
+ const SDNodeFlags *Flags) {
if (isBinOpWithFlags(Opcode)) {
// If no flags were passed in, use a default flags object.
SDNodeFlags F;
if (Flags == nullptr)
Flags = &F;
- auto *FN = newSDNode<BinaryWithFlagsSDNode>(Opcode, DL.getIROrder(),
- DL.getDebugLoc(), VTs, *Flags);
- createOperands(FN, Ops);
+ auto *FN = newSDNode<BinaryWithFlagsSDNode>(
+ Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, *Flags);
return FN;
}
- auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
- createOperands(N, Ops);
+ auto *N = newSDNode<BinarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
+ VTs, N1, N2);
return N;
}
@@ -985,7 +982,6 @@ SDNode *SelectionDAG::FindNodeOrInsertPo
void SelectionDAG::clear() {
allnodes_clear();
- OperandRecycler.clear(OperandAllocator);
OperandAllocator.Reset();
CSEMap.clear();
@@ -1642,9 +1638,7 @@ SDValue SelectionDAG::getVectorShuffle(E
memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
- dl.getDebugLoc(), MaskAlloc);
- createOperands(N, Ops);
-
+ dl.getDebugLoc(), N1, N2, MaskAlloc);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -1677,10 +1671,8 @@ SDValue SelectionDAG::getConvertRndSat(E
if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
return SDValue(E, 0);
- auto *N =
- newSDNode<CvtRndSatSDNode>(VT, dl.getIROrder(), dl.getDebugLoc(), Code);
- createOperands(N, Ops);
-
+ auto *N = newSDNode<CvtRndSatSDNode>(VT, dl.getIROrder(), dl.getDebugLoc(),
+ Ops, Code);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -1723,9 +1715,8 @@ SDValue SelectionDAG::getEHLabel(SDLoc d
if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
- auto *N = newSDNode<EHLabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Label);
- createOperands(N, Ops);
-
+ auto *N =
+ newSDNode<EHLabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Root, Label);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -1808,9 +1799,7 @@ SDValue SelectionDAG::getAddrSpaceCast(S
return SDValue(E, 0);
auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
- VT, SrcAS, DestAS);
- createOperands(N, Ops);
-
+ VT, Ptr, SrcAS, DestAS);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -3194,20 +3183,20 @@ SDValue SelectionDAG::getNode(unsigned O
SDNode *N;
SDVTList VTs = getVTList(VT);
- SDValue Ops[] = {Operand};
if (VT != MVT::Glue) { // Don't CSE flag producing nodes
FoldingSetNodeID ID;
+ SDValue Ops[1] = { Operand };
AddNodeIDNode(ID, Opcode, VTs, Ops);
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
- N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
- createOperands(N, Ops);
+ N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs,
+ Operand);
CSEMap.InsertNode(N, IP);
} else {
- N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
- createOperands(N, Ops);
+ N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs,
+ Operand);
}
InsertNode(N);
@@ -3906,7 +3895,7 @@ SDValue SelectionDAG::getNode(unsigned O
}
// Memoize this node if possible.
- SDNode *N;
+ BinarySDNode *N;
SDVTList VTs = getVTList(VT);
if (VT != MVT::Glue) {
SDValue Ops[] = {N1, N2};
@@ -3920,6 +3909,7 @@ SDValue SelectionDAG::getNode(unsigned O
}
N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
+
CSEMap.InsertNode(N, IP);
} else {
N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
@@ -4010,20 +4000,20 @@ SDValue SelectionDAG::getNode(unsigned O
// Memoize node if it doesn't produce a flag.
SDNode *N;
SDVTList VTs = getVTList(VT);
- SDValue Ops[] = {N1, N2, N3};
if (VT != MVT::Glue) {
+ SDValue Ops[] = { N1, N2, N3 };
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTs, Ops);
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
- N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
- createOperands(N, Ops);
+ N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs,
+ N1, N2, N3);
CSEMap.InsertNode(N, IP);
} else {
- N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
- createOperands(N, Ops);
+ N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs,
+ N1, N2, N3);
}
InsertNode(N);
@@ -4815,11 +4805,18 @@ SDValue SelectionDAG::getAtomic(unsigned
return SDValue(E, 0);
}
- auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
- VTList, MemVT, MMO, SuccessOrdering,
- FailureOrdering, SynchScope);
- createOperands(N, Ops);
+ // Allocate the operands array for the node out of the BumpPtrAllocator, since
+ // SDNode doesn't have access to it. This memory will be "leaked" when
+ // the node is deallocated, but recovered when the allocator is released.
+ // If the number of operands is less than 5 we use AtomicSDNode's internal
+ // storage.
+ unsigned NumOps = Ops.size();
+ SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
+ : nullptr;
+ auto *N = newSDNode<AtomicSDNode>(
+ Opcode, dl.getIROrder(), dl.getDebugLoc(), VTList, MemVT, Ops.data(),
+ DynOps, NumOps, MMO, SuccessOrdering, FailureOrdering, SynchScope);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -5012,14 +5009,11 @@ SelectionDAG::getMemIntrinsicNode(unsign
}
N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
- VTList, MemVT, MMO);
- createOperands(N, Ops);
-
- CSEMap.InsertNode(N, IP);
+ VTList, Ops, MemVT, MMO);
+ CSEMap.InsertNode(N, IP);
} else {
N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
- VTList, MemVT, MMO);
- createOperands(N, Ops);
+ VTList, Ops, MemVT, MMO);
}
InsertNode(N);
return SDValue(N, 0);
@@ -5137,10 +5131,8 @@ SelectionDAG::getLoad(ISD::MemIndexedMod
cast<LoadSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
- auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
- ExtType, MemVT, MMO);
- createOperands(N, Ops);
-
+ auto *N = newSDNode<LoadSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs,
+ AM, ExtType, MemVT, MMO);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -5246,10 +5238,8 @@ SDValue SelectionDAG::getStore(SDValue C
cast<StoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
- auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
+ auto *N = newSDNode<StoreSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs,
ISD::UNINDEXED, false, VT, MMO);
- createOperands(N, Ops);
-
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -5316,10 +5306,8 @@ SDValue SelectionDAG::getTruncStore(SDVa
cast<StoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
- auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
+ auto *N = newSDNode<StoreSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs,
ISD::UNINDEXED, true, SVT, MMO);
- createOperands(N, Ops);
-
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -5342,11 +5330,9 @@ SelectionDAG::getIndexedStore(SDValue Or
if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
return SDValue(E, 0);
- auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
- ST->isTruncatingStore(), ST->getMemoryVT(),
- ST->getMemOperand());
- createOperands(N, Ops);
-
+ auto *N = newSDNode<StoreSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs,
+ AM, ST->isTruncatingStore(),
+ ST->getMemoryVT(), ST->getMemOperand());
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -5372,10 +5358,8 @@ SelectionDAG::getMaskedLoad(EVT VT, SDLo
cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
- auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
- ExtTy, MemVT, MMO);
- createOperands(N, Ops);
-
+ auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), Ops,
+ 4, VTs, ExtTy, MemVT, MMO);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -5400,10 +5384,8 @@ SDValue SelectionDAG::getMaskedStore(SDV
cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
- auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
- isTrunc, MemVT, MMO);
- createOperands(N, Ops);
-
+ auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), Ops,
+ 4, VTs, isTrunc, MemVT, MMO);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -5413,7 +5395,6 @@ SDValue
SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl,
ArrayRef<SDValue> Ops,
MachineMemOperand *MMO) {
- assert(Ops.size() == 5 && "Incompatible number of operands");
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
@@ -5428,20 +5409,8 @@ SelectionDAG::getMaskedGather(SDVTList V
cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
-
auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
- VTs, VT, MMO);
- createOperands(N, Ops);
-
- assert(N->getValue().getValueType() == N->getValueType(0) &&
- "Incompatible type of the PassThru value in MaskedGatherSDNode");
- assert(N->getMask().getValueType().getVectorNumElements() ==
- N->getValueType(0).getVectorNumElements() &&
- "Vector width mismatch between mask and data");
- assert(N->getIndex().getValueType().getVectorNumElements() ==
- N->getValueType(0).getVectorNumElements() &&
- "Vector width mismatch between index and data");
-
+ Ops, VTs, VT, MMO);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -5450,8 +5419,6 @@ SelectionDAG::getMaskedGather(SDVTList V
SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl,
ArrayRef<SDValue> Ops,
MachineMemOperand *MMO) {
- assert(Ops.size() == 5 && "Incompatible number of operands");
-
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
ID.AddInteger(VT.getRawBits());
@@ -5465,16 +5432,7 @@ SDValue SelectionDAG::getMaskedScatter(S
return SDValue(E, 0);
}
auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
- VTs, VT, MMO);
- createOperands(N, Ops);
-
- assert(N->getMask().getValueType().getVectorNumElements() ==
- N->getValue().getValueType().getVectorNumElements() &&
- "Vector width mismatch between mask and data");
- assert(N->getIndex().getValueType().getVectorNumElements() ==
- N->getValue().getValueType().getVectorNumElements() &&
- "Vector width mismatch between index and data");
-
+ Ops, VTs, VT, MMO);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -5553,13 +5511,10 @@ SDValue SelectionDAG::getNode(unsigned O
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
- N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
- createOperands(N, Ops);
-
+ N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops);
CSEMap.InsertNode(N, IP);
} else {
- N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
- createOperands(N, Ops);
+ N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops);
}
InsertNode(N);
@@ -5601,6 +5556,7 @@ SDValue SelectionDAG::getNode(unsigned O
// Memoize the node unless it returns a flag.
SDNode *N;
+ unsigned NumOps = Ops.size();
if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTList, Ops);
@@ -5608,12 +5564,34 @@ SDValue SelectionDAG::getNode(unsigned O
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
- N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
- createOperands(N, Ops);
+ if (NumOps == 1) {
+ N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
+ VTList, Ops[0]);
+ } else if (NumOps == 2) {
+ N = newSDNode<BinarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
+ VTList, Ops[0], Ops[1]);
+ } else if (NumOps == 3) {
+ N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
+ VTList, Ops[0], Ops[1], Ops[2]);
+ } else {
+ N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList,
+ Ops);
+ }
CSEMap.InsertNode(N, IP);
} else {
- N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
- createOperands(N, Ops);
+ if (NumOps == 1) {
+ N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
+ VTList, Ops[0]);
+ } else if (NumOps == 2) {
+ N = newSDNode<BinarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
+ VTList, Ops[0], Ops[1]);
+ } else if (NumOps == 3) {
+ N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
+ VTList, Ops[0], Ops[1], Ops[2]);
+ } else {
+ N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList,
+ Ops);
+ }
}
InsertNode(N);
return SDValue(N, 0);
@@ -5999,6 +5977,7 @@ SDNode *SelectionDAG::UpdadeSDLocOnMerge
/// deleting things.
SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
SDVTList VTs, ArrayRef<SDValue> Ops) {
+ unsigned NumOps = Ops.size();
// If an identical node already exists, use it.
void *IP = nullptr;
if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
@@ -6027,13 +6006,36 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode
DeadNodeSet.insert(Used);
}
- // For MachineNode, initialize the memory references information.
- if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
+ if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
+ // Initialize the memory references information.
MN->setMemRefs(nullptr, nullptr);
-
- // Swap for an appropriately sized array from the recycler.
- removeOperands(N);
- createOperands(N, Ops);
+ // If NumOps is larger than the # of operands we can have in a
+ // MachineSDNode, reallocate the operand list.
+ if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
+ if (MN->OperandsNeedDelete)
+ delete[] MN->OperandList;
+ if (NumOps > array_lengthof(MN->LocalOperands))
+ // We're creating a final node that will live unmorphed for the
+ // remainder of the current SelectionDAG iteration, so we can allocate
+ // the operands directly out of a pool with no recycling metadata.
+ MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
+ Ops.data(), NumOps);
+ else
+ MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
+ MN->OperandsNeedDelete = false;
+ } else
+ MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
+ } else {
+ // If NumOps is larger than the # of operands we currently have, reallocate
+ // the operand list.
+ if (NumOps > N->NumOperands) {
+ if (N->OperandsNeedDelete)
+ delete[] N->OperandList;
+ N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
+ N->OperandsNeedDelete = true;
+ } else
+ N->InitOperands(N->OperandList, Ops.data(), NumOps);
+ }
// Delete any nodes that are still dead after adding the uses for the
// new operands.
@@ -6176,14 +6178,16 @@ SelectionDAG::getMachineNode(unsigned Op
MachineSDNode *
SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
- ArrayRef<SDValue> Ops) {
+ ArrayRef<SDValue> OpsArray) {
bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
MachineSDNode *N;
void *IP = nullptr;
+ const SDValue *Ops = OpsArray.data();
+ unsigned NumOps = OpsArray.size();
if (DoCSE) {
FoldingSetNodeID ID;
- AddNodeIDNode(ID, ~Opcode, VTs, Ops);
+ AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) {
return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
@@ -6192,7 +6196,17 @@ SelectionDAG::getMachineNode(unsigned Op
// Allocate a new MachineSDNode.
N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
- createOperands(N, Ops);
+
+ // Initialize the operands list.
+ if (NumOps > array_lengthof(N->LocalOperands))
+ // We're creating a final node that will live unmorphed for the
+ // remainder of the current SelectionDAG iteration, so we can allocate
+ // the operands directly out of a pool with no recycling metadata.
+ N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
+ Ops, NumOps);
+ else
+ N->InitOperands(N->LocalOperands, Ops, NumOps);
+ N->OperandsNeedDelete = false;
if (DoCSE)
CSEMap.InsertNode(N, IP);
@@ -6725,9 +6739,10 @@ GlobalAddressSDNode::GlobalAddressSDNode
}
AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT,
- unsigned SrcAS, unsigned DestAS)
- : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
- SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
+ SDValue X, unsigned SrcAS,
+ unsigned DestAS)
+ : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X),
+ SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
EVT memvt, MachineMemOperand *mmo)
@@ -6743,6 +6758,16 @@ MemSDNode::MemSDNode(unsigned Opc, unsig
assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
}
+MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
+ ArrayRef<SDValue> Ops, EVT memvt, MachineMemOperand *mmo)
+ : SDNode(Opc, Order, dl, VTs, Ops),
+ MemoryVT(memvt), MMO(mmo) {
+ SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
+ MMO->isNonTemporal(), MMO->isInvariant());
+ assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
+ assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
+}
+
/// Profile - Gather unique data for the node.
///
void SDNode::Profile(FoldingSetNodeID &ID) const {
More information about the llvm-commits
mailing list