[llvm-commits] [llvm] r142060 - in /llvm/trunk/lib/CodeGen/SelectionDAG: LegalizeDAG.cpp LegalizeVectorOps.cpp
Nadav Rotem
nadav.rotem at intel.com
Sat Oct 15 00:41:10 PDT 2011
Author: nadav
Date: Sat Oct 15 02:41:10 2011
New Revision: 142060
URL: http://llvm.org/viewvc/llvm-project?rev=142060&view=rev
Log:
Move the legalization of vector loads and stores into LegalizeVectorOps. In some
cases we need the second type-legalization pass in order to support all cases.
Modified:
llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp?rev=142060&r1=142059&r2=142060&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp Sat Oct 15 02:41:10 2011
@@ -1369,89 +1369,8 @@
break;
}
- // If this is a promoted vector load, and the vector element types are
- // legal, then scalarize it.
- if (ExtType == ISD::EXTLOAD && SrcVT.isVector() &&
- TLI.isTypeLegal(Node->getValueType(0).getScalarType())) {
- SmallVector<SDValue, 8> LoadVals;
- SmallVector<SDValue, 8> LoadChains;
- unsigned NumElem = SrcVT.getVectorNumElements();
- unsigned Stride = SrcVT.getScalarType().getSizeInBits()/8;
-
- for (unsigned Idx=0; Idx<NumElem; Idx++) {
- Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
- DAG.getIntPtrConstant(Stride));
- SDValue ScalarLoad = DAG.getExtLoad(ISD::EXTLOAD, dl,
- Node->getValueType(0).getScalarType(),
- Tmp1, Tmp2, LD->getPointerInfo().getWithOffset(Idx * Stride),
- SrcVT.getScalarType(),
- LD->isVolatile(), LD->isNonTemporal(),
- LD->getAlignment());
-
- LoadVals.push_back(ScalarLoad.getValue(0));
- LoadChains.push_back(ScalarLoad.getValue(1));
- }
- Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
- &LoadChains[0], LoadChains.size());
- SDValue ValRes = DAG.getNode(ISD::BUILD_VECTOR, dl,
- Node->getValueType(0), &LoadVals[0], LoadVals.size());
-
- Tmp1 = LegalizeOp(ValRes); // Relegalize new nodes.
- Tmp2 = LegalizeOp(Result.getValue(0)); // Relegalize new nodes.
- break;
- }
-
- // If this is a promoted vector load, and the vector element types are
- // illegal, create the promoted vector from bitcasted segments.
- if (ExtType == ISD::EXTLOAD && SrcVT.isVector()) {
- EVT MemElemTy = Node->getValueType(0).getScalarType();
- EVT SrcSclrTy = SrcVT.getScalarType();
- unsigned SizeRatio =
- (MemElemTy.getSizeInBits() / SrcSclrTy.getSizeInBits());
-
- SmallVector<SDValue, 8> LoadVals;
- SmallVector<SDValue, 8> LoadChains;
- unsigned NumElem = SrcVT.getVectorNumElements();
- unsigned Stride = SrcVT.getScalarType().getSizeInBits()/8;
-
- for (unsigned Idx=0; Idx<NumElem; Idx++) {
- Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
- DAG.getIntPtrConstant(Stride));
- SDValue ScalarLoad = DAG.getExtLoad(ISD::EXTLOAD, dl,
- SrcVT.getScalarType(),
- Tmp1, Tmp2, LD->getPointerInfo().getWithOffset(Idx * Stride),
- SrcVT.getScalarType(),
- LD->isVolatile(), LD->isNonTemporal(),
- LD->getAlignment());
- if (TLI.isBigEndian()) {
- // MSB (which is garbage, comes first)
- LoadVals.push_back(ScalarLoad.getValue(0));
- for (unsigned i = 0; i<SizeRatio-1; ++i)
- LoadVals.push_back(DAG.getUNDEF(SrcVT.getScalarType()));
- } else {
- // LSB (which is data, comes first)
- for (unsigned i = 0; i<SizeRatio-1; ++i)
- LoadVals.push_back(DAG.getUNDEF(SrcVT.getScalarType()));
- LoadVals.push_back(ScalarLoad.getValue(0));
- }
- LoadChains.push_back(ScalarLoad.getValue(1));
- }
-
- Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
- &LoadChains[0], LoadChains.size());
- EVT TempWideVector = EVT::getVectorVT(*DAG.getContext(),
- SrcVT.getScalarType(), NumElem*SizeRatio);
- SDValue ValRes = DAG.getNode(ISD::BUILD_VECTOR, dl,
- TempWideVector, &LoadVals[0], LoadVals.size());
-
- // Cast to the correct type
- ValRes = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), ValRes);
-
- Tmp1 = LegalizeOp(ValRes); // Relegalize new nodes.
- Tmp2 = LegalizeOp(Result.getValue(0)); // Relegalize new nodes.
- break;
-
- }
+ assert(!SrcVT.isVector() &&
+ "Vector Loads are handled in LegalizeVectorOps");
// FIXME: This does not work for vectors on most targets. Sign- and
// zero-extend operations are currently folded into extending loads,
@@ -1628,106 +1547,8 @@
Result = TLI.LowerOperation(Result, DAG);
break;
case TargetLowering::Expand:
-
- EVT WideScalarVT = Tmp3.getValueType().getScalarType();
- EVT NarrowScalarVT = StVT.getScalarType();
-
- if (StVT.isVector()) {
- unsigned NumElem = StVT.getVectorNumElements();
- // The type of the data we want to save
- EVT RegVT = Tmp3.getValueType();
- EVT RegSclVT = RegVT.getScalarType();
- // The type of data as saved in memory.
- EVT MemSclVT = StVT.getScalarType();
-
- bool RegScalarLegal = TLI.isTypeLegal(RegSclVT);
- bool MemScalarLegal = TLI.isTypeLegal(MemSclVT);
-
- // We need to expand this store. If the register element type
- // is legal then we can scalarize the vector and use
- // truncating stores.
- if (RegScalarLegal) {
- // Cast floats into integers
- unsigned ScalarSize = MemSclVT.getSizeInBits();
- EVT EltVT = EVT::getIntegerVT(*DAG.getContext(), ScalarSize);
-
- // Round odd types to the next pow of two.
- if (!isPowerOf2_32(ScalarSize))
- ScalarSize = NextPowerOf2(ScalarSize);
-
- // Store Stride in bytes
- unsigned Stride = ScalarSize/8;
- // Extract each of the elements from the original vector
- // and save them into memory individually.
- SmallVector<SDValue, 8> Stores;
- for (unsigned Idx = 0; Idx < NumElem; Idx++) {
- SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
- RegSclVT, Tmp3, DAG.getIntPtrConstant(Idx));
-
- Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
- DAG.getIntPtrConstant(Stride));
-
- // This scalar TruncStore may be illegal, but we lehalize it
- // later.
- SDValue Store = DAG.getTruncStore(Tmp1, dl, Ex, Tmp2,
- ST->getPointerInfo().getWithOffset(Idx*Stride), MemSclVT,
- isVolatile, isNonTemporal, Alignment);
-
- Stores.push_back(Store);
- }
-
- Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
- &Stores[0], Stores.size());
- break;
- }
-
- // The scalar register type is illegal.
- // For example saving <2 x i64> -> <2 x i32> on a x86.
- // In here we bitcast the value into a vector of smaller parts and
- // save it using smaller scalars.
- if (!RegScalarLegal && MemScalarLegal) {
- // Store Stride in bytes
- unsigned Stride = MemSclVT.getSizeInBits()/8;
-
- unsigned SizeRatio =
- (RegSclVT.getSizeInBits() / MemSclVT.getSizeInBits());
-
- EVT CastValueVT = EVT::getVectorVT(*DAG.getContext(),
- MemSclVT,
- SizeRatio * NumElem);
-
- // Cast the wide elem vector to wider vec with smaller elem type.
- // Example <2 x i64> -> <4 x i32>
- Tmp3 = DAG.getNode(ISD::BITCAST, dl, CastValueVT, Tmp3);
-
- SmallVector<SDValue, 8> Stores;
- for (unsigned Idx=0; Idx < NumElem * SizeRatio; Idx++) {
- // Extract the Ith element.
- SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
- NarrowScalarVT, Tmp3, DAG.getIntPtrConstant(Idx));
- // Bump pointer.
- Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2,
- DAG.getIntPtrConstant(Stride));
-
- // Store if, this element is:
- // - First element on big endian, or
- // - Last element on little endian
- if (( TLI.isBigEndian() && (Idx % SizeRatio == 0)) ||
- ((!TLI.isBigEndian() && (Idx % SizeRatio == SizeRatio-1)))) {
- SDValue Store = DAG.getStore(Tmp1, dl, Ex, Tmp2,
- ST->getPointerInfo().getWithOffset(Idx*Stride),
- isVolatile, isNonTemporal, Alignment);
- Stores.push_back(Store);
- }
- }
- Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
- &Stores[0], Stores.size());
- break;
- }
-
- assert(false && "Unable to legalize the vector trunc store!");
- }// is vector
-
+ assert(!StVT.isVector() &&
+ "Vector Stores are handled in LegalizeVectorOps");
// TRUNCSTORE:i16 i32 -> STORE i16
assert(TLI.isTypeLegal(StVT) && "Do not know how to expand this store!");
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp?rev=142060&r1=142059&r2=142060&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp Sat Oct 15 02:41:10 2011
@@ -64,6 +64,8 @@
// Implement vselect in terms of XOR, AND, OR when blend is not supported
// by the target.
SDValue ExpandVSELECT(SDValue Op);
+ SDValue ExpandLoad(SDValue Op);
+ SDValue ExpandStore(SDValue Op);
SDValue ExpandFNEG(SDValue Op);
// Implements vector promotion; this is essentially just bitcasting the
// operands to a different type and bitcasting the result back to the
@@ -124,6 +126,33 @@
SDValue Result =
SDValue(DAG.UpdateNodeOperands(Op.getNode(), Ops.data(), Ops.size()), 0);
+ if (Op.getOpcode() == ISD::LOAD) {
+ LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
+ ISD::LoadExtType ExtType = LD->getExtensionType();
+ if (LD->getMemoryVT().isVector() && ExtType != ISD::NON_EXTLOAD) {
+ if (TLI.isLoadExtLegal(LD->getExtensionType(), LD->getMemoryVT()))
+ return TranslateLegalizeResults(Op, Result);
+ Changed = true;
+ return LegalizeOp(ExpandLoad(Op));
+ }
+ } else if (Op.getOpcode() == ISD::STORE) {
+ StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
+ EVT StVT = ST->getMemoryVT();
+ EVT ValVT = ST->getValue().getValueType();
+ if (StVT.isVector() && ST->isTruncatingStore())
+ switch (TLI.getTruncStoreAction(ValVT, StVT)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal:
+ return TranslateLegalizeResults(Op, Result);
+ case TargetLowering::Custom:
+ Changed = true;
+ return LegalizeOp(TLI.LowerOperation(Result, DAG));
+ case TargetLowering::Expand:
+ Changed = true;
+ return LegalizeOp(ExpandStore(Op));
+ }
+ }
+
bool HasVectorValue = false;
for (SDNode::value_iterator J = Node->value_begin(), E = Node->value_end();
J != E;
@@ -262,6 +291,96 @@
return DAG.getNode(ISD::BITCAST, dl, VT, Op);
}
+
+SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
+ DebugLoc dl = Op.getDebugLoc();
+ LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
+ SDValue Chain = LD->getChain();
+ SDValue BasePTR = LD->getBasePtr();
+ EVT SrcVT = LD->getMemoryVT();
+
+ SmallVector<SDValue, 8> LoadVals;
+ SmallVector<SDValue, 8> LoadChains;
+ unsigned NumElem = SrcVT.getVectorNumElements();
+ unsigned Stride = SrcVT.getScalarType().getSizeInBits()/8;
+
+ for (unsigned Idx=0; Idx<NumElem; Idx++) {
+ BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR,
+ DAG.getIntPtrConstant(Stride));
+ SDValue ScalarLoad = DAG.getExtLoad(ISD::EXTLOAD, dl,
+ Op.getNode()->getValueType(0).getScalarType(),
+ Chain, BasePTR, LD->getPointerInfo().getWithOffset(Idx * Stride),
+ SrcVT.getScalarType(),
+ LD->isVolatile(), LD->isNonTemporal(),
+ LD->getAlignment());
+
+ LoadVals.push_back(ScalarLoad.getValue(0));
+ LoadChains.push_back(ScalarLoad.getValue(1));
+ }
+
+ SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
+ &LoadChains[0], LoadChains.size());
+ SDValue Value = DAG.getNode(ISD::BUILD_VECTOR, dl,
+ Op.getNode()->getValueType(0), &LoadVals[0], LoadVals.size());
+
+ AddLegalizedOperand(Op.getValue(0), Value);
+ AddLegalizedOperand(Op.getValue(1), NewChain);
+
+ return (Op.getResNo() ? NewChain : Value);
+}
+
+SDValue VectorLegalizer::ExpandStore(SDValue Op) {
+ DebugLoc dl = Op.getDebugLoc();
+ StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
+ SDValue Chain = ST->getChain();
+ SDValue BasePTR = ST->getBasePtr();
+ SDValue Value = ST->getValue();
+ EVT StVT = ST->getMemoryVT();
+
+ unsigned Alignment = ST->getAlignment();
+ bool isVolatile = ST->isVolatile();
+ bool isNonTemporal = ST->isNonTemporal();
+
+ unsigned NumElem = StVT.getVectorNumElements();
+ // The type of the data we want to save
+ EVT RegVT = Value.getValueType();
+ EVT RegSclVT = RegVT.getScalarType();
+ // The type of data as saved in memory.
+ EVT MemSclVT = StVT.getScalarType();
+
+ // Cast floats into integers
+ unsigned ScalarSize = MemSclVT.getSizeInBits();
+ EVT EltVT = EVT::getIntegerVT(*DAG.getContext(), ScalarSize);
+
+ // Round odd types to the next pow of two.
+ if (!isPowerOf2_32(ScalarSize))
+ ScalarSize = NextPowerOf2(ScalarSize);
+
+ // Store Stride in bytes
+ unsigned Stride = ScalarSize/8;
+ // Extract each of the elements from the original vector
+ // and save them into memory individually.
+ SmallVector<SDValue, 8> Stores;
+ for (unsigned Idx = 0; Idx < NumElem; Idx++) {
+ SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
+ RegSclVT, Value, DAG.getIntPtrConstant(Idx));
+
+ BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR,
+ DAG.getIntPtrConstant(Stride));
+
+ // This scalar TruncStore may be illegal, but we legalize it later.
+ SDValue Store = DAG.getTruncStore(Chain, dl, Ex, BasePTR,
+ ST->getPointerInfo().getWithOffset(Idx*Stride), MemSclVT,
+ isVolatile, isNonTemporal, Alignment);
+
+ Stores.push_back(Store);
+ }
+ SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
+ &Stores[0], Stores.size());
+ AddLegalizedOperand(Op, TF);
+ return TF;
+}
+
SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) {
// Implement VSELECT in terms of XOR, AND, OR
// on platforms which do not support blend natively.
More information about the llvm-commits
mailing list