[llvm] 154daf1 - [Hexagon] Widen short vector stores to HVX vectors using masked stores
Krzysztof Parzyszek via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 27 07:25:21 PDT 2020
Author: Krzysztof Parzyszek
Date: 2020-08-27T09:25:08-05:00
New Revision: 154daf1f94fd44b4ddc863ac865b82eec56f2c89
URL: https://github.com/llvm/llvm-project/commit/154daf1f94fd44b4ddc863ac865b82eec56f2c89
DIFF: https://github.com/llvm/llvm-project/commit/154daf1f94fd44b4ddc863ac865b82eec56f2c89.diff
LOG: [Hexagon] Widen short vector stores to HVX vectors using masked stores
Also invent a flag -hexagon-hvx-widen=N to set the minimum threshold
for widening short vectors to HVX vectors.
Added:
llvm/test/CodeGen/Hexagon/autohvx/short-store-widen.ll
Modified:
llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
llvm/lib/Target/Hexagon/HexagonISelLowering.h
llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 89d732fe0ded..7b1c8b6079a0 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -2064,20 +2064,9 @@ HexagonTargetLowering::getPreferredVectorAction(MVT VT) const {
return TargetLoweringBase::TypeScalarizeVector;
if (Subtarget.useHVXOps()) {
- unsigned HwLen = Subtarget.getVectorLength();
- // If the size of VT is at least half of the vector length,
- // widen the vector. Note: the threshold was not selected in
- // any scientific way.
- ArrayRef<MVT> Tys = Subtarget.getHVXElementTypes();
- if (llvm::find(Tys, ElemTy) != Tys.end()) {
- unsigned HwWidth = 8*HwLen;
- unsigned VecWidth = VT.getSizeInBits();
- if (VecWidth >= HwWidth/2 && VecWidth < HwWidth)
- return TargetLoweringBase::TypeWidenVector;
- }
- // Split vectors of i1 that correspond to (byte) vector pairs.
- if (ElemTy == MVT::i1 && VecLen == 2*HwLen)
- return TargetLoweringBase::TypeSplitVector;
+ unsigned Action = getPreferredHvxVectorAction(VT);
+ if (Action != ~0u)
+ return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action);
}
// Always widen (remaining) vectors of i1.
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
index ccea0da46e0d..d7a960fde0a2 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
@@ -338,6 +338,8 @@ namespace HexagonISD {
private:
void initializeHVXLowering();
+ unsigned getPreferredHvxVectorAction(MVT VecTy) const;
+
void validateConstPtrAlignment(SDValue Ptr, const SDLoc &dl,
unsigned NeedAlign) const;
@@ -473,6 +475,7 @@ namespace HexagonISD {
SDValue SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const;
SDValue SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const;
+ SDValue WidenHvxStore(SDValue Op, SelectionDAG &DAG) const;
std::pair<const TargetRegisterClass*, uint8_t>
findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT)
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index 6e0733775ec4..573316a5ce91 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -14,6 +14,10 @@
using namespace llvm;
+static cl::opt<unsigned> HvxWidenThreshold("hexagon-hvx-widen",
+ cl::Hidden, cl::init(16),
+ cl::desc("Lower threshold (in bytes) for widening to HVX vectors"));
+
static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 };
static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
@@ -218,9 +222,56 @@ HexagonTargetLowering::initializeHVXLowering() {
setOperationAction(ISD::SIGN_EXTEND_INREG, T, Legal);
}
+ // Handle store widening for short vectors.
+ std::vector<MVT> ShortTys;
+ unsigned HwLen = Subtarget.getVectorLength();
+ for (MVT ElemTy : Subtarget.getHVXElementTypes()) {
+ if (ElemTy == MVT::i1)
+ continue;
+ int ElemWidth = ElemTy.getSizeInBits().getFixedSize();
+ int MaxElems = (8*HwLen) / ElemWidth;
+ for (int N = 2; N < MaxElems; N *= 2) {
+ MVT VecTy = MVT::getVectorVT(ElemTy, N);
+ auto Action = getPreferredVectorAction(VecTy);
+ if (Action == TargetLoweringBase::TypeWidenVector)
+ setOperationAction(ISD::STORE, VecTy, Custom);
+ }
+ }
+
setTargetDAGCombine(ISD::VSELECT);
}
+unsigned
+HexagonTargetLowering::getPreferredHvxVectorAction(MVT VecTy) const {
+ MVT ElemTy = VecTy.getVectorElementType();
+ unsigned VecLen = VecTy.getVectorNumElements();
+ unsigned HwLen = Subtarget.getVectorLength();
+
+ // Split vectors of i1 that correspond to (byte) vector pairs.
+ if (ElemTy == MVT::i1 && VecLen == 2*HwLen)
+ return TargetLoweringBase::TypeSplitVector;
+ // Treat i1 as i8 from now on.
+ if (ElemTy == MVT::i1)
+ ElemTy = MVT::i8;
+
+ // If the size of VecTy is at least half of the vector length,
+ // widen the vector. Note: the threshold was not selected in
+ // any scientific way.
+ ArrayRef<MVT> Tys = Subtarget.getHVXElementTypes();
+ if (llvm::find(Tys, ElemTy) != Tys.end()) {
+ unsigned VecWidth = VecTy.getSizeInBits();
+ bool HaveThreshold = HvxWidenThreshold.getNumOccurrences() > 0;
+ if (HaveThreshold && 8*HvxWidenThreshold <= VecWidth)
+ return TargetLoweringBase::TypeWidenVector;
+ unsigned HwWidth = 8*HwLen;
+ if (VecWidth >= HwWidth/2 && VecWidth < HwWidth)
+ return TargetLoweringBase::TypeWidenVector;
+ }
+
+ // Defer to default.
+ return ~0u;
+}
+
SDValue
HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
const SDLoc &dl, SelectionDAG &DAG) const {
@@ -1801,6 +1852,39 @@ HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
llvm_unreachable(Name.c_str());
}
+SDValue
+HexagonTargetLowering::WidenHvxStore(SDValue Op, SelectionDAG &DAG) const {
+ const SDLoc &dl(Op);
+ auto *StoreN = cast<StoreSDNode>(Op.getNode());
+ assert(StoreN->isUnindexed() && "Not widening indexed stores yet");
+ assert(StoreN->getMemoryVT().getVectorElementType() != MVT::i1 &&
+ "Not widening stores of i1 yet");
+
+ SDValue Chain = StoreN->getChain();
+ SDValue Base = StoreN->getBasePtr();
+ SDValue Offset = DAG.getUNDEF(MVT::i32);
+
+ SDValue Value = opCastElem(StoreN->getValue(), MVT::i8, DAG);
+ MVT ValueTy = ty(Value);
+ unsigned ValueLen = ValueTy.getVectorNumElements();
+ unsigned HwLen = Subtarget.getVectorLength();
+ assert(isPowerOf2_32(ValueLen));
+
+ for (unsigned Len = ValueLen; Len < HwLen; ) {
+ Value = opJoin({DAG.getUNDEF(ty(Value)), Value}, dl, DAG);
+ Len = ty(Value).getVectorNumElements(); // This is Len *= 2
+ }
+ assert(ty(Value).getVectorNumElements() == HwLen); // Paranoia
+
+ MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
+ SDValue StoreQ = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
+ {DAG.getConstant(ValueLen, dl, MVT::i32)}, DAG);
+ MachineFunction &MF = DAG.getMachineFunction();
+ auto *MOp = MF.getMachineMemOperand(StoreN->getMemOperand(), 0, HwLen);
+ return DAG.getMaskedStore(Chain, dl, Value, Base, Offset, StoreQ, ty(Value),
+ MOp, ISD::UNINDEXED, false, false);
+}
+
SDValue
HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
unsigned Opc = Op.getOpcode();
@@ -1879,6 +1963,14 @@ HexagonTargetLowering::LowerHvxOperationWrapper(SDNode *N,
SDValue Op(N, 0);
switch (Opc) {
+ case ISD::STORE: {
+ SDValue Value = cast<StoreSDNode>(N)->getValue();
+ unsigned Action = getPreferredHvxVectorAction(ty(Value));
+ assert(Action == TargetLoweringBase::TypeWidenVector && "Not widening?");
+ SDValue Store = WidenHvxStore(SDValue(N, 0), DAG);
+ Results.push_back(Store);
+ break;
+ }
case ISD::MLOAD:
if (isHvxPairTy(ty(Op))) {
SDValue S = SplitHvxMemOp(Op, DAG);
@@ -1939,6 +2031,16 @@ HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
bool
HexagonTargetLowering::isHvxOperation(SDNode *N) const {
+ if (N->getOpcode() == ISD::STORE) {
+ // If it's a store-to-be-widened, treat it as an HVX operation.
+ SDValue Val = cast<StoreSDNode>(N)->getValue();
+ MVT ValTy = ty(Val);
+ if (ValTy.isVector()) {
+ auto Action = getPreferredVectorAction(ValTy);
+ if (Action == TargetLoweringBase::TypeWidenVector)
+ return true;
+ }
+ }
// If the type of any result, or any operand type are HVX vector types,
// this is an HVX operation.
auto IsHvxTy = [this] (EVT Ty) {
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/short-store-widen.ll b/llvm/test/CodeGen/Hexagon/autohvx/short-store-widen.ll
new file mode 100644
index 000000000000..311450502f24
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/autohvx/short-store-widen.ll
@@ -0,0 +1,16 @@
+; RUN: llc -march=hexagon -hexagon-hvx-widen=16 < %s | FileCheck %s
+
+; CHECK-LABEL: f0:
+; CHECK: q[[Q0:[0-3]]] = vsetq(r{{[0-9]+}})
+; CHECK: if (q[[Q0]]) vmem({{.*}}) = v
+define void @f0(<32 x i8>* %a0) #0 {
+ %v0 = load <32 x i8>, <32 x i8>* %a0, align 128
+ %v1 = insertelement <32 x i8> undef, i8 1, i32 0
+ %v2 = shufflevector <32 x i8> %v1, <32 x i8> undef, <32 x i32> zeroinitializer
+ %v3 = add <32 x i8> %v0, %v2
+ store <32 x i8> %v3, <32 x i8>* %a0, align 128
+ ret void
+}
+
+attributes #0 = { nounwind "target-cpu"="hexagonv65" "target-features"="+hvx,+hvx-length128b" }
+
More information about the llvm-commits
mailing list