[llvm] dbbfc95 - [RISCV] Separate ActiveElementsAffectResult into VL and Mask flags (#106517)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 29 16:46:11 PDT 2024
Author: Luke Lau
Date: 2024-08-30T07:46:06+08:00
New Revision: dbbfc952f0d4703b89fa238e2aba98f1229fb972
URL: https://github.com/llvm/llvm-project/commit/dbbfc952f0d4703b89fa238e2aba98f1229fb972
DIFF: https://github.com/llvm/llvm-project/commit/dbbfc952f0d4703b89fa238e2aba98f1229fb972.diff
LOG: [RISCV] Separate ActiveElementsAffectResult into VL and Mask flags (#106517)
In #106110 we had to mark v[f]slide1down.vx as
ActiveElementsAffectResult since the elements in the body depend on VL.
However it doesn't depend on the mask, so this was overly conservative
and broke the vmerge peephole.
We can recover this by splitting up ActiveElementsAffectResult into VL
and Mask bits, so we can more accurately model v[f]slide1down.vx and
re-enable the peephole.
Added:
Modified:
llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
llvm/lib/Target/RISCV/RISCVInstrFormats.td
llvm/lib/Target/RISCV/RISCVInstrInfoV.td
llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index c65bd5b1d33631..7a0b35c1afce23 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -124,8 +124,11 @@ enum {
TargetOverlapConstraintTypeShift = UsesVXRMShift + 1,
TargetOverlapConstraintTypeMask = 3ULL << TargetOverlapConstraintTypeShift,
- ActiveElementsAffectResultShift = TargetOverlapConstraintTypeShift + 2,
- ActiveElementsAffectResultMask = 1ULL << ActiveElementsAffectResultShift,
+ ElementsDependOnVLShift = TargetOverlapConstraintTypeShift + 2,
+ ElementsDependOnVLMask = 1ULL << ElementsDependOnVLShift,
+
+ ElementsDependOnMaskShift = ElementsDependOnVLShift + 1,
+ ElementsDependOnMaskMask = 1ULL << ElementsDependOnMaskShift,
};
// Helper functions to read TSFlags.
@@ -174,10 +177,16 @@ static inline bool hasRoundModeOp(uint64_t TSFlags) {
/// \returns true if this instruction uses vxrm
static inline bool usesVXRM(uint64_t TSFlags) { return TSFlags & UsesVXRMMask; }
-/// \returns true if the result isn't element-wise,
-/// e.g. vredsum.vs/vcompress.vm/viota.m
-static inline bool activeElementsAffectResult(uint64_t TSFlags) {
- return TSFlags & ActiveElementsAffectResultMask;
+/// \returns true if the elements in the body are affected by VL,
+/// e.g. vslide1down.vx/vredsum.vs/viota.m
+static inline bool elementsDependOnVL(uint64_t TSFlags) {
+ return TSFlags & ElementsDependOnVLMask;
+}
+
+/// \returns true if the elements in the body are affected by the mask,
+/// e.g. vredsum.vs/viota.m
+static inline bool elementsDependOnMask(uint64_t TSFlags) {
+ return TSFlags & ElementsDependOnMaskMask;
}
static inline unsigned getVLOpNum(const MCInstrDesc &Desc) {
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 48fe788a81dffc..9cec6278abc2c5 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -3941,12 +3941,11 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
// active elements, like viota.m or vredsum. This transformation is illegal
// for these if we change the active elements (i.e. mask or VL).
const MCInstrDesc &TrueBaseMCID = TII->get(RISCV::getRVVMCOpcode(TrueOpc));
- if (RISCVII::activeElementsAffectResult(TrueBaseMCID.TSFlags)) {
- if (Mask && !usesAllOnesMask(Mask, Glue))
- return false;
- if (TrueVL != VL)
- return false;
- }
+ if (RISCVII::elementsDependOnVL(TrueBaseMCID.TSFlags) && (TrueVL != VL))
+ return false;
+ if (RISCVII::elementsDependOnMask(TrueBaseMCID.TSFlags) &&
+ (Mask && !usesAllOnesMask(Mask, Glue)))
+ return false;
// If we end up changing the VL or mask of True, then we need to make sure it
// doesn't raise any observable fp exceptions, since changing the active
diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
index 95f157064d73e2..a389320adc8763 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrFormats.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
@@ -158,6 +158,15 @@ def OPC_SYSTEM : RISCVOpcode<"SYSTEM", 0b1110011>;
def OPC_OP_VE : RISCVOpcode<"OP_VE", 0b1110111>;
def OPC_CUSTOM_3 : RISCVOpcode<"CUSTOM_3", 0b1111011>;
+class EltDeps<bit vl, bit mask> {
+ bit VL = vl;
+ bit Mask = mask;
+}
+
+def EltDepsNone : EltDeps<vl=0, mask=0>;
+def EltDepsVL : EltDeps<vl=1, mask=0>;
+def EltDepsVLMask : EltDeps<vl=1, mask=1>;
+
class RVInstCommon<dag outs, dag ins, string opcodestr, string argstr,
list<dag> pattern, InstFormat format> : Instruction {
let Namespace = "RISCV";
@@ -224,8 +233,13 @@ class RVInstCommon<dag outs, dag ins, string opcodestr, string argstr,
bits<2> TargetOverlapConstraintType = 0;
let TSFlags{22-21} = TargetOverlapConstraintType;
- bit ActiveElementsAffectResult = 0;
- let TSFlags{23} = ActiveElementsAffectResult;
+ // Most vector instructions are elementwise, but some may depend on the value
+ // of VL (e.g. vslide1down.vx), and others may depend on the VL and mask
+ // (e.g. vredsum.vs, viota.m). Mark these instructions so that peepholes avoid
+ // changing their VL and/or mask.
+ EltDeps ElementsDependOn = EltDepsNone;
+ let TSFlags{23} = ElementsDependOn.VL;
+ let TSFlags{24} = ElementsDependOn.Mask;
}
class RVInst<dag outs, dag ins, string opcodestr, string argstr,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index 878859fc1d0864..738bb5d9bd65bf 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -1503,7 +1503,7 @@ defm VFNCVT_ROD_F_F_W : VNCVTF_FV_VS2<"vfncvt.rod.f.f.w", 0b010010, 0b10101>;
let Predicates = [HasVInstructions] in {
// Vector Single-Width Integer Reduction Instructions
-let RVVConstraint = NoConstraint, ActiveElementsAffectResult = 1 in {
+let RVVConstraint = NoConstraint, ElementsDependOn = EltDepsVLMask in {
defm VREDSUM : VRED_MV_V<"vredsum", 0b000000>;
defm VREDMAXU : VREDMINMAX_MV_V<"vredmaxu", 0b000110>;
defm VREDMAX : VREDMINMAX_MV_V<"vredmax", 0b000111>;
@@ -1512,23 +1512,23 @@ defm VREDMIN : VREDMINMAX_MV_V<"vredmin", 0b000101>;
defm VREDAND : VRED_MV_V<"vredand", 0b000001>;
defm VREDOR : VRED_MV_V<"vredor", 0b000010>;
defm VREDXOR : VRED_MV_V<"vredxor", 0b000011>;
-} // RVVConstraint = NoConstraint, ActiveElementsAffectResult = 1
+} // RVVConstraint = NoConstraint, ElementsDependOn = EltDepsVLMask
// Vector Widening Integer Reduction Instructions
-let Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint, ActiveElementsAffectResult = 1 in {
+let Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint, ElementsDependOn = EltDepsVLMask in {
// Set earlyclobber for following instructions for second and mask operands.
// This has the downside that the earlyclobber constraint is too coarse and
// will impose unnecessary restrictions by not allowing the destination to
// overlap with the first (wide) operand.
defm VWREDSUMU : VWRED_IV_V<"vwredsumu", 0b110000>;
defm VWREDSUM : VWRED_IV_V<"vwredsum", 0b110001>;
-} // Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint, ActiveElementsAffectResult = 1
+} // Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint, ElementsDependOn = EltDepsVLMask
} // Predicates = [HasVInstructions]
let Predicates = [HasVInstructionsAnyF] in {
// Vector Single-Width Floating-Point Reduction Instructions
-let RVVConstraint = NoConstraint, ActiveElementsAffectResult = 1 in {
+let RVVConstraint = NoConstraint, ElementsDependOn = EltDepsVLMask in {
let Uses = [FRM], mayRaiseFPException = true in {
defm VFREDOSUM : VREDO_FV_V<"vfredosum", 0b000011>;
defm VFREDUSUM : VRED_FV_V<"vfredusum", 0b000001>;
@@ -1537,13 +1537,13 @@ let mayRaiseFPException = true in {
defm VFREDMAX : VREDMINMAX_FV_V<"vfredmax", 0b000111>;
defm VFREDMIN : VREDMINMAX_FV_V<"vfredmin", 0b000101>;
}
-} // RVVConstraint = NoConstraint, ActiveElementsAffectResult = 1
+} // RVVConstraint = NoConstraint, ElementsDependOn = EltDepsVLMask
def : InstAlias<"vfredsum.vs $vd, $vs2, $vs1$vm",
(VFREDUSUM_VS VR:$vd, VR:$vs2, VR:$vs1, VMaskOp:$vm), 0>;
// Vector Widening Floating-Point Reduction Instructions
-let Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint, ActiveElementsAffectResult = 1 in {
+let Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint, ElementsDependOn = EltDepsVLMask in {
// Set earlyclobber for following instructions for second and mask operands.
// This has the downside that the earlyclobber constraint is too coarse and
// will impose unnecessary restrictions by not allowing the destination to
@@ -1552,7 +1552,7 @@ let Uses = [FRM], mayRaiseFPException = true in {
defm VFWREDOSUM : VWREDO_FV_V<"vfwredosum", 0b110011>;
defm VFWREDUSUM : VWRED_FV_V<"vfwredusum", 0b110001>;
}
-} // Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint, ActiveElementsAffectResult = 1
+} // Constraints = "@earlyclobber $vd", RVVConstraint = NoConstraint, ElementsDependOn = EltDepsVLMask
def : InstAlias<"vfwredsum.vs $vd, $vs2, $vs1$vm",
(VFWREDUSUM_VS VR:$vd, VR:$vs2, VR:$vs1, VMaskOp:$vm), 0>;
@@ -1586,7 +1586,7 @@ def : InstAlias<"vmornot.mm $vd, $vs2, $vs1",
(VMORN_MM VR:$vd, VR:$vs2, VR:$vs1), 0>;
let hasSideEffects = 0, mayLoad = 0, mayStore = 0,
- RVVConstraint = NoConstraint, ActiveElementsAffectResult = 1 in {
+ RVVConstraint = NoConstraint, ElementsDependOn = EltDepsVLMask in {
// Vector mask population count vcpop
def VCPOP_M : RVInstV<0b010000, 0b10000, OPMVV, (outs GPR:$vd),
@@ -1600,12 +1600,12 @@ def VFIRST_M : RVInstV<0b010000, 0b10001, OPMVV, (outs GPR:$vd),
"vfirst.m", "$vd, $vs2$vm">,
SchedUnaryMC<"WriteVMFFSV", "ReadVMFFSV">;
-} // hasSideEffects = 0, mayLoad = 0, mayStore = 0, RVVConstraint = NoConstraint, ActiveElementsAffectResult = 1
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 0, RVVConstraint = NoConstraint, ElementsDependOn = EltDepsVLMask
def : InstAlias<"vpopc.m $vd, $vs2$vm",
(VCPOP_M GPR:$vd, VR:$vs2, VMaskOp:$vm), 0>;
-let Constraints = "@earlyclobber $vd", RVVConstraint = Iota, ActiveElementsAffectResult = 1 in {
+let Constraints = "@earlyclobber $vd", RVVConstraint = Iota, ElementsDependOn = EltDepsVLMask in {
// vmsbf.m set-before-first mask bit
defm VMSBF_M : VMSFS_MV_V<"vmsbf.m", 0b010100, 0b00001>;
@@ -1616,7 +1616,7 @@ defm VMSOF_M : VMSFS_MV_V<"vmsof.m", 0b010100, 0b00010>;
// Vector Iota Instruction
defm VIOTA_M : VIOTA_MV_V<"viota.m", 0b010100, 0b10000>;
-} // Constraints = "@earlyclobber $vd", RVVConstraint = Iota, ActiveElementsAffectResult = 1
+} // Constraints = "@earlyclobber $vd", RVVConstraint = Iota, ElementsDependOn = EltDepsVLMask
// Vector Element Index Instruction
let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
@@ -1665,7 +1665,7 @@ defm VSLIDEUP_V : VSLD_IV_X_I<"vslideup", 0b001110, /*slidesUp=*/true>;
defm VSLIDE1UP_V : VSLD1_MV_X<"vslide1up", 0b001110>;
} // Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp
defm VSLIDEDOWN_V : VSLD_IV_X_I<"vslidedown", 0b001111, /*slidesUp=*/false>;
-let ActiveElementsAffectResult = 1 in
+let ElementsDependOn = EltDepsVL in
defm VSLIDE1DOWN_V : VSLD1_MV_X<"vslide1down", 0b001111>;
} // Predicates = [HasVInstructions]
@@ -1673,7 +1673,7 @@ let Predicates = [HasVInstructionsAnyF] in {
let Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp in {
defm VFSLIDE1UP_V : VSLD1_FV_F<"vfslide1up", 0b001110>;
} // Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp
-let ActiveElementsAffectResult = 1 in
+let ElementsDependOn = EltDepsVL in
defm VFSLIDE1DOWN_V : VSLD1_FV_F<"vfslide1down", 0b001111>;
} // Predicates = [HasVInstructionsAnyF]
@@ -1688,9 +1688,9 @@ def VRGATHEREI16_VV : VALUVV<0b001110, OPIVV, "vrgatherei16.vv">,
} // Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather
// Vector Compress Instruction
-let Constraints = "@earlyclobber $vd", RVVConstraint = Vcompress, ActiveElementsAffectResult = 1 in {
+let Constraints = "@earlyclobber $vd", RVVConstraint = Vcompress, ElementsDependOn = EltDepsVLMask in {
defm VCOMPRESS_V : VCPR_MV_Mask<"vcompress", 0b010111>;
-} // Constraints = "@earlyclobber $vd", RVVConstraint = Vcompress, ActiveElementsAffectResult = 1
+} // Constraints = "@earlyclobber $vd", RVVConstraint = Vcompress, ElementsDependOn = EltDepsVLMask
let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isMoveReg = 1,
RVVConstraint = NoConstraint in {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
index bd5319af80ffc8..3b726e2e9bdc9b 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
@@ -75,7 +75,7 @@ class RVInstVCCustom2<bits<4> funct6_hi4, bits<3> funct3, dag outs, dag ins,
let Uses = [VTYPE, VL];
let RVVConstraint = NoConstraint;
- let ActiveElementsAffectResult = 1;
+ let ElementsDependOn = EltDepsVLMask;
}
class RVInstVCFCustom2<bits<4> funct6_hi4, bits<3> funct3, dag outs, dag ins,
@@ -99,7 +99,7 @@ class RVInstVCFCustom2<bits<4> funct6_hi4, bits<3> funct3, dag outs, dag ins,
let Uses = [VTYPE, VL];
let RVVConstraint = NoConstraint;
- let ActiveElementsAffectResult = 1;
+ let ElementsDependOn = EltDepsVLMask;
}
class VCIXInfo<string suffix, VCIXType type, DAGOperand TyRd,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
index 555bddc740678d..c0fe624a9b709c 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
@@ -235,13 +235,13 @@ class THStoreUpdate<bits<5> funct5, string opcodestr>
//===----------------------------------------------------------------------===//
multiclass THVdotVMAQA_VX<string opcodestr, bits<6> funct6> {
- let RVVConstraint = WidenV, ActiveElementsAffectResult = 1 in
+ let RVVConstraint = WidenV, ElementsDependOn = EltDepsVLMask in
def _VX : THVdotALUrVX<funct6, OPMVX, opcodestr # ".vx", EarlyClobber=1>;
}
multiclass THVdotVMAQA<string opcodestr, bits<6> funct6>
: THVdotVMAQA_VX<opcodestr, funct6> {
- let RVVConstraint = WidenV, ActiveElementsAffectResult = 1 in
+ let RVVConstraint = WidenV, ElementsDependOn = EltDepsVLMask in
def _VV : THVdotALUrVV<funct6, OPMVX, opcodestr # ".vv", EarlyClobber=1>;
}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index cafd259031746d..e19a11805c9c08 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -140,7 +140,7 @@ let Predicates = [HasStdExtZvkb] in {
defm VROR_V : VROR_IV_V_X_I<"vror", 0b010100>;
} // Predicates = [HasStdExtZvkb]
-let ActiveElementsAffectResult = 1 in {
+let ElementsDependOn = EltDepsVLMask in {
let Predicates = [HasStdExtZvkg], RVVConstraint = NoConstraint in {
def VGHSH_VV : PALUVVNoVmTernary<0b101100, OPMVV, "vghsh.vv">,
@@ -198,7 +198,7 @@ let Predicates = [HasStdExtZvksh], RVVConstraint = VS2Constraint in {
SchedUnaryMC<"WriteVSM3MEV", "ReadVSM3MEV">;
} // Predicates = [HasStdExtZvksh]
-} // ActiveElementsAffectResult = 1
+} // ElementsDependOn = EltDepsVLMask
//===----------------------------------------------------------------------===//
// Pseudo instructions
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 34e5d9224f7150..0f8f9442877e33 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -156,9 +156,9 @@ bool RISCVVectorPeephole::tryToReduceVL(MachineInstr &MI) const {
if (getSEWLMULRatio(MI) != getSEWLMULRatio(*Src))
return false;
- bool ActiveElementsAffectResult = RISCVII::activeElementsAffectResult(
+ bool ElementsDependOnVL = RISCVII::elementsDependOnVL(
TII->get(RISCV::getRVVMCOpcode(Src->getOpcode())).TSFlags);
- if (ActiveElementsAffectResult || Src->mayRaiseFPException())
+ if (ElementsDependOnVL || Src->mayRaiseFPException())
return false;
MachineOperand &SrcVL = Src->getOperand(RISCVII::getVLOpNum(Src->getDesc()));
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
index 2ff775d6d14f43..6700920cebff0a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
@@ -776,15 +776,12 @@ define <vscale x 2 x i32> @vpselect_vslide1up(<vscale x 2 x i32> %passthru, <vsc
ret <vscale x 2 x i32> %b
}
-; FIXME: We can still fold this given that the vmerge and the vslide1down have
-; the same vl.
declare <vscale x 2 x i32> @llvm.riscv.vslide1down.nxv2i32.i32(<vscale x 2 x i32>, <vscale x 2 x i32>, i32, i64)
define <vscale x 2 x i32> @vpselect_vslide1down(<vscale x 2 x i32> %passthru, <vscale x 2 x i32> %v, i32 %x, <vscale x 2 x i1> %m, i32 zeroext %vl) {
; CHECK-LABEL: vpselect_vslide1down:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
-; CHECK-NEXT: vslide1down.vx v9, v9, a0
-; CHECK-NEXT: vmerge.vvm v8, v8, v9, v0
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, mu
+; CHECK-NEXT: vslide1down.vx v8, v9, a0, v0.t
; CHECK-NEXT: ret
%1 = zext i32 %vl to i64
%a = call <vscale x 2 x i32> @llvm.riscv.vslide1down.nxv2i32.i32(<vscale x 2 x i32> undef, <vscale x 2 x i32> %v, i32 %x, i64 %1)
More information about the llvm-commits
mailing list