[llvm] d056d5d - [RISCV] Use custom isel for vector indexed load/store intrinsics.
Craig Topper via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 19 10:18:26 PST 2021
Author: Craig Topper
Date: 2021-02-19T10:10:06-08:00
New Revision: d056d5decfb593cfc53b421a89075d2063f76ec9
URL: https://github.com/llvm/llvm-project/commit/d056d5decfb593cfc53b421a89075d2063f76ec9
DIFF: https://github.com/llvm/llvm-project/commit/d056d5decfb593cfc53b421a89075d2063f76ec9.diff
LOG: [RISCV] Use custom isel for vector indexed load/store intrinsics.
There are many legal combinations of index and data VTs supported
for these intrinsics. This results in a lot of isel patterns in
RISCVGenDAGISel.inc.
By adding a separate table similar to what we use for segment
load/stores, we can more efficiently manually select these
intrinsics. We should also be able to reuse this table scalable
vector gather/scatter.
This reduces the llc binary size by ~56K.
Reviewed By: khchen
Differential Revision: https://reviews.llvm.org/D97033
Added:
Modified:
llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
Removed:
################################################################################
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 0cd9b6cec768..1a979edfde03 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -31,6 +31,8 @@ namespace RISCV {
#define GET_RISCVVLSEGTable_IMPL
#define GET_RISCVVLXSEGTable_IMPL
#define GET_RISCVVSXSEGTable_IMPL
+#define GET_RISCVVLXTable_IMPL
+#define GET_RISCVVSXTable_IMPL
#include "RISCVGenSearchableTables.inc"
} // namespace RISCV
} // namespace llvm
@@ -666,6 +668,50 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
selectVLSEGFF(Node, /*IsMasked*/ true);
return;
}
+ case Intrinsic::riscv_vloxei:
+ case Intrinsic::riscv_vloxei_mask:
+ case Intrinsic::riscv_vluxei:
+ case Intrinsic::riscv_vluxei_mask: {
+ bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
+ IntNo == Intrinsic::riscv_vluxei_mask;
+ bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
+ IntNo == Intrinsic::riscv_vloxei_mask;
+
+ SDLoc DL(Node);
+ MVT VT = Node->getSimpleValueType(0);
+ unsigned ScalarSize = VT.getScalarSizeInBits();
+ MVT XLenVT = Subtarget->getXLenVT();
+ SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
+
+ unsigned CurOp = 2;
+ SmallVector<SDValue, 7> Operands;
+ if (IsMasked)
+ Operands.push_back(Node->getOperand(CurOp++));
+ Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
+ Operands.push_back(Node->getOperand(CurOp++)); // Index.
+ MVT IndexVT = Operands.back()->getSimpleValueType(0);
+ if (IsMasked)
+ Operands.push_back(Node->getOperand(CurOp++)); // Mask.
+ SDValue VL;
+ selectVLOp(Node->getOperand(CurOp++), VL);
+ Operands.push_back(VL);
+ Operands.push_back(SEW);
+ Operands.push_back(Node->getOperand(0)); // Chain.
+
+ assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
+ "Element count mismatch");
+
+ RISCVVLMUL LMUL = getLMUL(VT);
+ RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
+ unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
+ const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
+ IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
+ static_cast<unsigned>(IndexLMUL));
+ SDNode *Load =
+ CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
+ ReplaceNode(Node, Load);
+ return;
+ }
}
break;
}
@@ -748,6 +794,49 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
case Intrinsic::riscv_vsuxseg8_mask:
selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
return;
+ case Intrinsic::riscv_vsoxei:
+ case Intrinsic::riscv_vsoxei_mask:
+ case Intrinsic::riscv_vsuxei:
+ case Intrinsic::riscv_vsuxei_mask: {
+ bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
+ IntNo == Intrinsic::riscv_vsuxei_mask;
+ bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
+ IntNo == Intrinsic::riscv_vsoxei_mask;
+
+ SDLoc DL(Node);
+ MVT VT = Node->getOperand(2)->getSimpleValueType(0);
+ unsigned ScalarSize = VT.getScalarSizeInBits();
+ MVT XLenVT = Subtarget->getXLenVT();
+ SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
+
+ unsigned CurOp = 2;
+ SmallVector<SDValue, 6> Operands;
+ Operands.push_back(Node->getOperand(CurOp++)); // Store value.
+ Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
+ Operands.push_back(Node->getOperand(CurOp++)); // Index.
+ MVT IndexVT = Operands.back()->getSimpleValueType(0);
+ if (IsMasked)
+ Operands.push_back(Node->getOperand(CurOp++)); // Mask.
+ SDValue VL;
+ selectVLOp(Node->getOperand(CurOp++), VL);
+ Operands.push_back(VL);
+ Operands.push_back(SEW);
+ Operands.push_back(Node->getOperand(0)); // Chain.
+
+ assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
+ "Element count mismatch");
+
+ RISCVVLMUL LMUL = getLMUL(VT);
+ RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
+ unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
+ const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
+ IsMasked, IsOrdered, IndexScalarSize, static_cast<unsigned>(LMUL),
+ static_cast<unsigned>(IndexLMUL));
+ SDNode *Store =
+ CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
+ ReplaceNode(Node, Store);
+ return;
+ }
}
break;
}
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
index f4ed99aa9304..258866d7dba8 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
@@ -126,10 +126,21 @@ struct VSXSEGPseudo {
uint16_t Pseudo;
};
+struct VLX_VSXPseudo {
+ uint8_t Masked;
+ uint8_t Ordered;
+ uint8_t SEW;
+ uint8_t LMUL;
+ uint8_t IndexLMUL;
+ uint16_t Pseudo;
+};
+
#define GET_RISCVVSSEGTable_DECL
#define GET_RISCVVLSEGTable_DECL
#define GET_RISCVVLXSEGTable_DECL
#define GET_RISCVVSXSEGTable_DECL
+#define GET_RISCVVLXTable_DECL
+#define GET_RISCVVSXTable_DECL
#include "RISCVGenSearchableTables.inc"
} // namespace RISCV
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index afbab16a232d..850e7b0bd545 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -413,6 +413,40 @@ def RISCVVIntrinsicsTable : GenericTable {
let PrimaryKeyName = "getRISCVVIntrinsicInfo";
}
+class RISCVVLX<bit M, bit O, bits<7> S, bits<3> L, bits<3> IL> {
+ bits<1> Masked = M;
+ bits<1> Ordered = O;
+ bits<7> SEW = S;
+ bits<3> LMUL = L;
+ bits<3> IndexLMUL = IL;
+ Pseudo Pseudo = !cast<Pseudo>(NAME);
+}
+
+def RISCVVLXTable : GenericTable {
+ let FilterClass = "RISCVVLX";
+ let CppTypeName = "VLX_VSXPseudo";
+ let Fields = ["Masked", "Ordered", "SEW", "LMUL", "IndexLMUL", "Pseudo"];
+ let PrimaryKey = ["Masked", "Ordered", "SEW", "LMUL", "IndexLMUL"];
+ let PrimaryKeyName = "getVLXPseudo";
+}
+
+class RISCVVSX<bit M, bit O, bits<7> S, bits<3> L, bits<3> IL> {
+ bits<1> Masked = M;
+ bits<1> Ordered = O;
+ bits<7> SEW = S;
+ bits<3> LMUL = L;
+ bits<3> IndexLMUL = IL;
+ Pseudo Pseudo = !cast<Pseudo>(NAME);
+}
+
+def RISCVVSXTable : GenericTable {
+ let FilterClass = "RISCVVSX";
+ let CppTypeName = "VLX_VSXPseudo";
+ let Fields = ["Masked", "Ordered", "SEW", "LMUL", "IndexLMUL", "Pseudo"];
+ let PrimaryKey = ["Masked", "Ordered", "SEW", "LMUL", "IndexLMUL"];
+ let PrimaryKeyName = "getVSXPseudo";
+}
+
class RISCVVLSEG<bits<4> N, bit M, bit Str, bit F, bits<7> S, bits<3> L> {
bits<4> NF = N;
bits<1> Masked = M;
@@ -616,10 +650,12 @@ class VPseudoSLoadMask<VReg RetClass>:
let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
}
-class VPseudoILoadNoMask<VReg RetClass, VReg IdxClass>:
+class VPseudoILoadNoMask<VReg RetClass, VReg IdxClass, bits<7> EEW, bits<3> LMUL,
+ bit Ordered>:
Pseudo<(outs RetClass:$rd),
(ins GPR:$rs1, IdxClass:$rs2, GPR:$vl, ixlenimm:$sew),[]>,
- RISCVVPseudo {
+ RISCVVPseudo,
+ RISCVVLX</*Masked*/0, Ordered, EEW, VLMul, LMUL> {
let mayLoad = 1;
let mayStore = 0;
let hasSideEffects = 0;
@@ -631,12 +667,14 @@ class VPseudoILoadNoMask<VReg RetClass, VReg IdxClass>:
let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
}
-class VPseudoILoadMask<VReg RetClass, VReg IdxClass>:
+class VPseudoILoadMask<VReg RetClass, VReg IdxClass, bits<7> EEW, bits<3> LMUL,
+ bit Ordered>:
Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
(ins GetVRegNoV0<RetClass>.R:$merge,
GPR:$rs1, IdxClass:$rs2,
VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>,
- RISCVVPseudo {
+ RISCVVPseudo,
+ RISCVVLX</*Masked*/1, Ordered, EEW, VLMul, LMUL> {
let mayLoad = 1;
let mayStore = 0;
let hasSideEffects = 0;
@@ -877,10 +915,12 @@ class VPseudoBinaryNoMask<VReg RetClass,
let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
}
-class VPseudoIStoreNoMask<VReg StClass, VReg IdxClass>:
+class VPseudoIStoreNoMask<VReg StClass, VReg IdxClass, bits<7> EEW, bits<3> LMUL,
+ bit Ordered>:
Pseudo<(outs),
(ins StClass:$rd, GPR:$rs1, IdxClass:$rs2, GPR:$vl, ixlenimm:$sew),[]>,
- RISCVVPseudo {
+ RISCVVPseudo,
+ RISCVVSX</*Masked*/0, Ordered, EEW, VLMul, LMUL> {
let mayLoad = 0;
let mayStore = 1;
let hasSideEffects = 0;
@@ -892,10 +932,12 @@ class VPseudoIStoreNoMask<VReg StClass, VReg IdxClass>:
let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
}
-class VPseudoIStoreMask<VReg StClass, VReg IdxClass>:
+class VPseudoIStoreMask<VReg StClass, VReg IdxClass, bits<7> EEW, bits<3> LMUL,
+ bit Ordered>:
Pseudo<(outs),
(ins StClass:$rd, GPR:$rs1, IdxClass:$rs2, VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>,
- RISCVVPseudo {
+ RISCVVPseudo,
+ RISCVVSX</*Masked*/1, Ordered, EEW, VLMul, LMUL> {
let mayLoad = 0;
let mayStore = 1;
let hasSideEffects = 0;
@@ -1284,7 +1326,7 @@ multiclass VPseudoSLoad {
}
}
-multiclass VPseudoILoad {
+multiclass VPseudoILoad<bit Ordered> {
foreach eew = EEWList in {
foreach sew = EEWList in {
foreach lmul = MxSet<sew>.m in {
@@ -1298,8 +1340,10 @@ multiclass VPseudoILoad {
defvar Vreg = lmul.vrclass;
defvar IdxVreg = idx_lmul.vrclass;
let VLMul = lmul.value in {
- def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo : VPseudoILoadNoMask<Vreg, IdxVreg>;
- def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo # "_MASK" : VPseudoILoadMask<Vreg, IdxVreg>;
+ def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo :
+ VPseudoILoadNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>;
+ def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo # "_MASK" :
+ VPseudoILoadMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>;
}
}
}
@@ -1341,7 +1385,7 @@ multiclass VPseudoSStore {
}
}
-multiclass VPseudoIStore {
+multiclass VPseudoIStore<bit Ordered> {
foreach eew = EEWList in {
foreach sew = EEWList in {
foreach lmul = MxSet<sew>.m in {
@@ -1356,9 +1400,9 @@ multiclass VPseudoIStore {
defvar IdxVreg = idx_lmul.vrclass;
let VLMul = lmul.value in {
def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo :
- VPseudoIStoreNoMask<Vreg, IdxVreg>;
+ VPseudoIStoreNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>;
def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo # "_MASK" :
- VPseudoIStoreMask<Vreg, IdxVreg>;
+ VPseudoIStoreMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>;
}
}
}
@@ -3263,10 +3307,10 @@ defm PseudoVSS : VPseudoSStore;
//===----------------------------------------------------------------------===//
// Vector Indexed Loads and Stores
-defm PseudoVLUX : VPseudoILoad;
-defm PseudoVLOX : VPseudoILoad;
-defm PseudoVSOX : VPseudoIStore;
-defm PseudoVSUX : VPseudoIStore;
+defm PseudoVLUX : VPseudoILoad</*Ordered=*/false>;
+defm PseudoVLOX : VPseudoILoad</*Ordered=*/true>;
+defm PseudoVSOX : VPseudoIStore</*Ordered=*/true>;
+defm PseudoVSUX : VPseudoIStore</*Ordered=*/false>;
//===----------------------------------------------------------------------===//
// 7.7. Unit-stride Fault-Only-First Loads
@@ -3844,45 +3888,6 @@ foreach vti = AllVectors in
vti.Vector, vti.Mask, vti.SEW, vti.LMul, vti.RegClass>;
}
-//===----------------------------------------------------------------------===//
-// 7.6 Vector Indexed Instructions
-//===----------------------------------------------------------------------===//
-
-foreach vti = AllVectors in
-foreach eew = EEWList in {
- defvar vlmul = vti.LMul;
- defvar octuple_lmul = octuple_from_str<vti.LMul.MX>.ret;
- defvar log_sew = shift_amount<vti.SEW>.val;
- // The data vector register group has EEW=SEW, EMUL=LMUL, while the offset
- // vector register group has EEW encoding in the instruction and EMUL=(EEW/SEW)*LMUL.
- // calculate octuple elmul which is (eew * octuple_lmul) >> log_sew
- defvar octuple_elmul = !srl(!mul(eew, octuple_lmul), log_sew);
- // legal octuple elmul should be more than 0 and less than equal 64
- if !gt(octuple_elmul, 0) then {
- if !le(octuple_elmul, 64) then {
- defvar elmul_str = octuple_to_str<octuple_elmul>.ret;
- defvar elmul =!cast<LMULInfo>("V_" # elmul_str);
- defvar idx_vti = !cast<VTypeInfo>("VI" # eew # elmul_str);
-
- defm : VPatILoad<"int_riscv_vluxei",
- "PseudoVLUXEI"#eew,
- vti.Vector, idx_vti.Vector, vti.Mask, vti.SEW,
- vlmul, elmul, vti.RegClass, idx_vti.RegClass>;
- defm : VPatILoad<"int_riscv_vloxei",
- "PseudoVLOXEI"#eew,
- vti.Vector, idx_vti.Vector, vti.Mask, vti.SEW,
- vlmul, elmul, vti.RegClass, idx_vti.RegClass>;
- defm : VPatIStore<"int_riscv_vsoxei",
- "PseudoVSOXEI"#eew,
- vti.Vector, idx_vti.Vector, vti.Mask, vti.SEW,
- vlmul, elmul, vti.RegClass, idx_vti.RegClass>;
- defm : VPatIStore<"int_riscv_vsuxei",
- "PseudoVSUXEI"#eew,
- vti.Vector, idx_vti.Vector, vti.Mask, vti.SEW,
- vlmul, elmul, vti.RegClass, idx_vti.RegClass>;
- }
- }
-}
} // Predicates = [HasStdExtV]
//===----------------------------------------------------------------------===//
More information about the llvm-commits
mailing list