[llvm] 4400018 - [RISCV] Model all 3 arithmetic sources of vector FMA at MC layer.
Craig Topper via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 4 09:09:26 PDT 2023
Author: Craig Topper
Date: 2023-08-04T09:08:16-07:00
New Revision: 44000181fd53c8e100acad55e99e89403d64daf8
URL: https://github.com/llvm/llvm-project/commit/44000181fd53c8e100acad55e99e89403d64daf8
DIFF: https://github.com/llvm/llvm-project/commit/44000181fd53c8e100acad55e99e89403d64daf8.diff
LOG: [RISCV] Model all 3 arithmetic sources of vector FMA at MC layer.
For the most part, MC version of vector instructions don't model when
the destination is also a source. This primarily occurs for mask/tail
undisturbed. The MC layer can't see the policy bits so this kind of
makes sense.
We also lumped FMA instructions into this, but the destination of
FMA is an arithmetic source not just an undisturbed value. This needs
to be correct for llvm-mca to understand the dependency for the FMA
instructions. Though every other instruction is still wrong for
tail/mask undisturbed.
This patch models the FMA instructions correctly at the MCA layer.
This necessitates changes to the assembler to offset operand numbers.
I've added the extra sched class operand and fixed the operand order
for the scalar read class.
Reviewed By: rogfer01
Differential Revision: https://reviews.llvm.org/D151850
Added:
Modified:
llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
llvm/lib/Target/RISCV/RISCVInstrInfoV.td
llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td
Removed:
################################################################################
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 046a208921ae64..6ef7f1078aa3ca 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -3352,16 +3352,21 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
}
unsigned DestReg = Inst.getOperand(0).getReg();
+ unsigned Offset = 0;
+ int TiedOp = MCID.getOperandConstraint(1, MCOI::TIED_TO);
+ if (TiedOp == 0)
+ Offset = 1;
+
// Operands[1] will be the first operand, DestReg.
SMLoc Loc = Operands[1]->getStartLoc();
if (MCID.TSFlags & RISCVII::VS2Constraint) {
- unsigned CheckReg = Inst.getOperand(1).getReg();
+ unsigned CheckReg = Inst.getOperand(Offset + 1).getReg();
if (DestReg == CheckReg)
return Error(Loc, "The destination vector register group cannot overlap"
" the source vector register group.");
}
- if ((MCID.TSFlags & RISCVII::VS1Constraint) && (Inst.getOperand(2).isReg())) {
- unsigned CheckReg = Inst.getOperand(2).getReg();
+ if ((MCID.TSFlags & RISCVII::VS1Constraint) && Inst.getOperand(Offset + 2).isReg()) {
+ unsigned CheckReg = Inst.getOperand(Offset + 2).getReg();
if (DestReg == CheckReg)
return Error(Loc, "The destination vector register group cannot overlap"
" the source vector register group.");
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index 9d22d5be829656..fce1309ccc0ff0 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -162,6 +162,9 @@ class SchedBinaryMC<string write, string read0, string read1,
class SchedTernary<string write, string read0, string read1, string read2,
string mx, int sew = 0, bit forceMasked = 0>:
SchedNary<write, [read0, read1, read2], mx, sew, forceMasked>;
+class SchedTernaryMC<string write, string read0, string read1, string read2,
+ int sew = 0, bit forceMasked = 1>:
+ SchedNary<write, [read0, read1, read2], "WorstCase", sew, forceMasked>;
// For reduction instructions.
class SchedReduction<string write, string read, string mx, int sew>:
@@ -438,10 +441,14 @@ class VALUmVV<bits<6> funct6, RISCVVFormat opv, string opcodestr>
}
// op vd, vs1, vs2, vm (reverse the order of vs1 and vs2)
-class VALUrVV<bits<6> funct6, RISCVVFormat opv, string opcodestr>
- : RVInstVV<funct6, opv, (outs VR:$vd),
- (ins VR:$vs1, VR:$vs2, VMaskOp:$vm),
- opcodestr, "$vd, $vs1, $vs2$vm">;
+class VALUrVV<bits<6> funct6, RISCVVFormat opv, string opcodestr,
+ bit EarlyClobber = 0>
+ : RVInstVV<funct6, opv, (outs VR:$vd_wb),
+ (ins VR:$vd, VR:$vs1, VR:$vs2, VMaskOp:$vm),
+ opcodestr, "$vd, $vs1, $vs2$vm"> {
+ let Constraints = !if(EarlyClobber, "@earlyclobber $vd_wb, $vd = $vd_wb",
+ "$vd = $vd_wb");
+}
// op vd, vs2, vs1
class VALUVVNoVm<bits<6> funct6, RISCVVFormat opv, string opcodestr>
@@ -466,10 +473,14 @@ class VALUmVX<bits<6> funct6, RISCVVFormat opv, string opcodestr>
}
// op vd, rs1, vs2, vm (reverse the order of rs1 and vs2)
-class VALUrVX<bits<6> funct6, RISCVVFormat opv, string opcodestr>
- : RVInstVX<funct6, opv, (outs VR:$vd),
- (ins GPR:$rs1, VR:$vs2, VMaskOp:$vm),
- opcodestr, "$vd, $rs1, $vs2$vm">;
+class VALUrVX<bits<6> funct6, RISCVVFormat opv, string opcodestr,
+ bit EarlyClobber = 0>
+ : RVInstVX<funct6, opv, (outs VR:$vd_wb),
+ (ins VR:$vd, GPR:$rs1, VR:$vs2, VMaskOp:$vm),
+ opcodestr, "$vd, $rs1, $vs2$vm"> {
+ let Constraints = !if(EarlyClobber, "@earlyclobber $vd_wb, $vd = $vd_wb",
+ "$vd = $vd_wb");
+}
// op vd, vs1, vs2
class VALUVXNoVm<bits<6> funct6, RISCVVFormat opv, string opcodestr>
@@ -508,10 +519,14 @@ class VALUVF<bits<6> funct6, RISCVVFormat opv, string opcodestr>
opcodestr, "$vd, $vs2, $rs1$vm">;
// op vd, rs1, vs2, vm (Float) (with mask, reverse the order of rs1 and vs2)
-class VALUrVF<bits<6> funct6, RISCVVFormat opv, string opcodestr>
- : RVInstVX<funct6, opv, (outs VR:$vd),
- (ins FPR32:$rs1, VR:$vs2, VMaskOp:$vm),
- opcodestr, "$vd, $rs1, $vs2$vm">;
+class VALUrVF<bits<6> funct6, RISCVVFormat opv, string opcodestr,
+ bit EarlyClobber = 0>
+ : RVInstVX<funct6, opv, (outs VR:$vd_wb),
+ (ins VR:$vd, FPR32:$rs1, VR:$vs2, VMaskOp:$vm),
+ opcodestr, "$vd, $rs1, $vs2$vm"> {
+ let Constraints = !if(EarlyClobber, "@earlyclobber $vd_wb, $vd = $vd_wb",
+ "$vd = $vd_wb");
+}
// op vd, vs2, vm (use vs1 as instruction encoding)
class VALUVs2<bits<6> funct6, bits<5> vs1, RISCVVFormat opv, string opcodestr>
@@ -590,20 +605,26 @@ multiclass VALU_MV_V_X<string opcodestr, bits<6> funct6, string vw> {
multiclass VMAC_MV_V_X<string opcodestr, bits<6> funct6> {
def V : VALUrVV<funct6, OPMVV, opcodestr # ".vv">,
- SchedBinaryMC<"WriteVIMulAddV", "ReadVIMulAddV", "ReadVIMulAddV">;
+ SchedTernaryMC<"WriteVIMulAddV", "ReadVIMulAddV", "ReadVIMulAddV",
+ "ReadVIMulAddV">;
def X : VALUrVX<funct6, OPMVX, opcodestr # ".vx">,
- SchedBinaryMC<"WriteVIMulAddX", "ReadVIMulAddV", "ReadVIMulAddX">;
+ SchedTernaryMC<"WriteVIMulAddX", "ReadVIMulAddV", "ReadVIMulAddX",
+ "ReadVIMulAddV">;
}
multiclass VWMAC_MV_X<string opcodestr, bits<6> funct6> {
+ let RVVConstraint = WidenV in
def X : VALUrVX<funct6, OPMVX, opcodestr # ".vx">,
- SchedBinaryMC<"WriteVIWMulAddX", "ReadVIWMulAddV", "ReadVIWMulAddX">;
+ SchedTernaryMC<"WriteVIWMulAddX", "ReadVIWMulAddV", "ReadVIWMulAddX",
+ "ReadVIWMulAddV">;
}
multiclass VWMAC_MV_V_X<string opcodestr, bits<6> funct6>
: VWMAC_MV_X<opcodestr, funct6> {
- def V : VALUrVV<funct6, OPMVV, opcodestr # ".vv">,
- SchedBinaryMC<"WriteVIWMulAddV", "ReadVIWMulAddV", "ReadVIWMulAddV">;
+ let RVVConstraint = WidenV in
+ def V : VALUrVV<funct6, OPMVV, opcodestr # ".vv", /*EarlyClobber*/1>,
+ SchedTernaryMC<"WriteVIWMulAddV", "ReadVIWMulAddV", "ReadVIWMulAddV",
+ "ReadVIWMulAddV">;
}
multiclass VALU_MV_VS2<string opcodestr, bits<6> funct6, bits<5> vs1> {
@@ -693,16 +714,22 @@ multiclass VWMUL_FV_V_F<string opcodestr, bits<6> funct6> {
multiclass VMAC_FV_V_F<string opcodestr, bits<6> funct6> {
def V : VALUrVV<funct6, OPFVV, opcodestr # ".vv">,
- SchedBinaryMC<"WriteVFMulAddV", "ReadVFMulAddV", "ReadVFMulAddV">;
+ SchedTernaryMC<"WriteVFMulAddV", "ReadVFMulAddV", "ReadVFMulAddV",
+ "ReadVFMulAddV">;
def F : VALUrVF<funct6, OPFVF, opcodestr # ".vf">,
- SchedBinaryMC<"WriteVFMulAddF", "ReadVFMulAddV", "ReadVFMulAddF">;
+ SchedTernaryMC<"WriteVFMulAddF", "ReadVFMulAddV", "ReadVFMulAddF",
+ "ReadVFMulAddV">;
}
multiclass VWMAC_FV_V_F<string opcodestr, bits<6> funct6> {
- def V : VALUrVV<funct6, OPFVV, opcodestr # ".vv">,
- SchedBinaryMC<"WriteVFWMulAddV", "ReadVFWMulAddV", "ReadVFWMulAddV">;
- def F : VALUrVF<funct6, OPFVF, opcodestr # ".vf">,
- SchedBinaryMC<"WriteVFWMulAddF", "ReadVFWMulAddV", "ReadVFWMulAddF">;
+ let RVVConstraint = WidenV in {
+ def V : VALUrVV<funct6, OPFVV, opcodestr # ".vv", /*EarlyClobber*/1>,
+ SchedTernaryMC<"WriteVFWMulAddV", "ReadVFWMulAddV", "ReadVFWMulAddV",
+ "ReadVFWMulAddV">;
+ def F : VALUrVF<funct6, OPFVF, opcodestr # ".vf", /*EarlyClobber*/1>,
+ SchedTernaryMC<"WriteVFWMulAddF", "ReadVFWMulAddV", "ReadVFWMulAddF",
+ "ReadVFWMulAddV">;
+ }
}
multiclass VSQR_FV_VS2<string opcodestr, bits<6> funct6, bits<5> vs1> {
@@ -1289,12 +1316,10 @@ defm VMADD_V : VMAC_MV_V_X<"vmadd", 0b101001>;
defm VNMSUB_V : VMAC_MV_V_X<"vnmsub", 0b101011>;
// Vector Widening Integer Multiply-Add Instructions
-let Constraints = "@earlyclobber $vd", RVVConstraint = WidenV in {
defm VWMACCU_V : VWMAC_MV_V_X<"vwmaccu", 0b111100>;
defm VWMACC_V : VWMAC_MV_V_X<"vwmacc", 0b111101>;
defm VWMACCSU_V : VWMAC_MV_V_X<"vwmaccsu", 0b111111>;
defm VWMACCUS_V : VWMAC_MV_X<"vwmaccus", 0b111110>;
-} // Constraints = "@earlyclobber $vd", RVVConstraint = WidenV
// Vector Integer Merge Instructions
defm VMERGE_V : VMRG_IV_V_X_I<"vmerge", 0b010111>;
@@ -1394,8 +1419,7 @@ defm VFNMSUB_V : VMAC_FV_V_F<"vfnmsub", 0b101011>;
}
// Vector Widening Floating-Point Fused Multiply-Add Instructions
-let Constraints = "@earlyclobber $vd", RVVConstraint = WidenV,
- Uses = [FRM], mayRaiseFPException = true in {
+let Uses = [FRM], mayRaiseFPException = true in {
defm VFWMACC_V : VWMAC_FV_V_F<"vfwmacc", 0b111100>;
defm VFWNMACC_V : VWMAC_FV_V_F<"vfwnmacc", 0b111101>;
defm VFWMSAC_V : VWMAC_FV_V_F<"vfwmsac", 0b111110>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 4e2ce84972cb8b..98f01a855385ab 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -3138,8 +3138,8 @@ multiclass VPseudoVMAC_VV_VX_AAXA<string Constraint = ""> {
SchedTernary<"WriteVIMulAddV", "ReadVIMulAddV", "ReadVIMulAddV",
"ReadVIMulAddV", mx>;
defm "" : VPseudoTernaryV_VX_AAXA<m, Constraint>,
- SchedTernary<"WriteVIMulAddX", "ReadVIMulAddV", "ReadVIMulAddV",
- "ReadVIMulAddX", mx>;
+ SchedTernary<"WriteVIMulAddX", "ReadVIMulAddV", "ReadVIMulAddX",
+ "ReadVIMulAddV", mx>;
}
}
@@ -3153,8 +3153,8 @@ multiclass VPseudoVMAC_VV_VF_AAXA<string Constraint = ""> {
foreach f = FPList in {
foreach m = f.MxList in {
defm "" : VPseudoTernaryV_VF_AAXA<m, f, Constraint>,
- SchedTernary<"WriteVFMulAddF", "ReadVFMulAddV", "ReadVFMulAddV",
- "ReadVFMulAddF", m.MX>;
+ SchedTernary<"WriteVFMulAddF", "ReadVFMulAddV", "ReadVFMulAddF",
+ "ReadVFMulAddV", m.MX>;
}
}
}
@@ -3169,8 +3169,8 @@ multiclass VPseudoVMAC_VV_VF_AAXA_RM<string Constraint = ""> {
foreach f = FPList in {
foreach m = f.MxList in {
defm "" : VPseudoTernaryV_VF_AAXA_RM<m, f, Constraint>,
- SchedTernary<"WriteVFMulAddF", "ReadVFMulAddV", "ReadVFMulAddV",
- "ReadVFMulAddF", m.MX>;
+ SchedTernary<"WriteVFMulAddF", "ReadVFMulAddV", "ReadVFMulAddF",
+ "ReadVFMulAddV", m.MX>;
}
}
}
@@ -3193,16 +3193,16 @@ multiclass VPseudoVWMAC_VV_VX {
SchedTernary<"WriteVIWMulAddV", "ReadVIWMulAddV", "ReadVIWMulAddV",
"ReadVIWMulAddV", mx>;
defm "" : VPseudoTernaryW_VX<m>,
- SchedTernary<"WriteVIWMulAddX", "ReadVIWMulAddV", "ReadVIWMulAddV",
- "ReadVIWMulAddX", mx>;
+ SchedTernary<"WriteVIWMulAddX", "ReadVIWMulAddV", "ReadVIWMulAddX",
+ "ReadVIWMulAddV", mx>;
}
}
multiclass VPseudoVWMAC_VX {
foreach m = MxListW in {
defm "" : VPseudoTernaryW_VX<m>,
- SchedTernary<"WriteVIWMulAddX", "ReadVIWMulAddV", "ReadVIWMulAddV",
- "ReadVIWMulAddX", m.MX>;
+ SchedTernary<"WriteVIWMulAddX", "ReadVIWMulAddV", "ReadVIWMulAddX",
+ "ReadVIWMulAddV", m.MX>;
}
}
@@ -3217,7 +3217,7 @@ multiclass VPseudoVWMAC_VV_VF_RM {
foreach m = f.MxListFW in {
defm "" : VPseudoTernaryW_VF_RM<m, f>,
SchedTernary<"WriteVFWMulAddF", "ReadVFWMulAddV",
- "ReadVFWMulAddV", "ReadVFWMulAddF", m.MX>;
+ "ReadVFWMulAddF", "ReadVFWMulAddV", m.MX>;
}
}
}
@@ -3242,7 +3242,7 @@ multiclass VPseudoVWMAC_VV_VF_BF_RM {
defm "" : VPseudoTernaryW_VF_BF_RM<m, f>,
Sched<[WriteVFWMulAddF_MX, ReadVFWMulAddV_MX,
- ReadVFWMulAddV_MX, ReadVFWMulAddF_MX, ReadVMask]>;
+ ReadVFWMulAddF_MX, ReadVFWMulAddV_MX, ReadVMask]>;
}
}
}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
index 17cb2a12f63a7b..ff4bc851cf210f 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
@@ -56,16 +56,24 @@ class THInstVdotVX<bits<6> funct6, RISCVVFormat opv, dag outs, dag ins,
let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
// op vd, vs1, vs2, vm (reverse the order of vs1 and vs2)
-class THVdotALUrVV<bits<6> funct6, RISCVVFormat opv, string opcodestr>
- : THInstVdotVV<funct6, opv, (outs VR:$vd),
- (ins VR:$vs1, VR:$vs2, VMaskOp:$vm),
- opcodestr, "$vd, $vs1, $vs2$vm">;
+class THVdotALUrVV<bits<6> funct6, RISCVVFormat opv, string opcodestr,
+ bit EarlyClobber>
+ : THInstVdotVV<funct6, opv, (outs VR:$vd_wb),
+ (ins VR:$vd, VR:$vs1, VR:$vs2, VMaskOp:$vm),
+ opcodestr, "$vd, $vs1, $vs2$vm"> {
+ let Constraints = !if(EarlyClobber, "@earlyclobber $vd_wb, $vd = $vd_wb",
+ "$vd = $vd_wb");
+}
// op vd, rs1, vs2, vm (reverse the order of rs1 and vs2)
-class THVdotALUrVX<bits<6> funct6, RISCVVFormat opv, string opcodestr>
- : THInstVdotVX<funct6, opv, (outs VR:$vd),
- (ins GPR:$rs1, VR:$vs2, VMaskOp:$vm),
- opcodestr, "$vd, $rs1, $vs2$vm">;
+class THVdotALUrVX<bits<6> funct6, RISCVVFormat opv, string opcodestr,
+ bit EarlyClobber>
+ : THInstVdotVX<funct6, opv, (outs VR:$vd_wb),
+ (ins VR:$vd, GPR:$rs1, VR:$vs2, VMaskOp:$vm),
+ opcodestr, "$vd, $rs1, $vs2$vm"> {
+ let Constraints = !if(EarlyClobber, "@earlyclobber $vd_wb, $vd = $vd_wb",
+ "$vd = $vd_wb");
+}
} // hasSideEffects = 0, mayLoad = 0, mayStore = 0
let Predicates = [HasVendorXTHeadBa], DecoderNamespace = "XTHeadBa",
@@ -227,12 +235,14 @@ class THStoreUpdate<bits<5> funct5, string opcodestr>
//===----------------------------------------------------------------------===//
multiclass THVdotVMAQA_VX<string opcodestr, bits<6> funct6> {
- def _VX : THVdotALUrVX<funct6, OPMVX, opcodestr # ".vx">;
+ let RVVConstraint = WidenV in
+ def _VX : THVdotALUrVX<funct6, OPMVX, opcodestr # ".vx", /*EarlyClobber*/1>;
}
-multiclass THVdotVMAQA<string opcodestr, bits<6> funct6> {
- def _VV : THVdotALUrVV<funct6, OPMVX, opcodestr # ".vv">;
- defm "" : THVdotVMAQA_VX<opcodestr, funct6>;
+multiclass THVdotVMAQA<string opcodestr, bits<6> funct6>
+ : THVdotVMAQA_VX<opcodestr, funct6> {
+ let RVVConstraint = WidenV in
+ def _VV : THVdotALUrVV<funct6, OPMVX, opcodestr # ".vv", /*EarlyClobber*/1>;
}
//===----------------------------------------------------------------------===//
@@ -448,9 +458,7 @@ def TH_FSURD : THStoreIndexed<FPR64, 0b01110, "th.fsurd">,
Sched<[WriteFST64, ReadFStoreData, ReadFMemBase]>;
}
-let Predicates = [HasVendorXTHeadVdot],
- Constraints = "@earlyclobber $vd",
- RVVConstraint = WidenV in {
+let Predicates = [HasVendorXTHeadVdot] in {
defm THVdotVMAQA : THVdotVMAQA<"th.vmaqa", 0b100000>;
defm THVdotVMAQAU : THVdotVMAQA<"th.vmaqau", 0b100010>;
defm THVdotVMAQASU : THVdotVMAQA<"th.vmaqasu", 0b100100>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td
index 046074d848f598..1b1f3b9b16e44f 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvfbf.td
@@ -25,7 +25,8 @@ let Uses = [FRM] in
defm VFNCVTBF16_F_F_W : VNCVTF_FV_VS2<"vfncvtbf16.f.f.w", 0b010010, 0b11101>;
}
-let Predicates = [HasStdExtZvfbfwma], Constraints = "@earlyclobber $vd",
+let Predicates = [HasStdExtZvfbfwma],
+ Constraints = "@earlyclobber $vd_wb, $vd = $vd_wb",
RVVConstraint = WidenV, Uses = [FRM], mayRaiseFPException = true in {
defm VFWMACCBF16_V : VWMAC_FV_V_F<"vfwmaccbf16", 0b111011>;
}
More information about the llvm-commits
mailing list