[llvm] [RISCV] Move performCombineVMergeAndVOps to RISCVVectorPeephole (PR #144076)

[Craig Topper via llvm-commits]

================
@@ -663,6 +674,131 @@ bool RISCVVectorPeephole::foldVMV_V_V(MachineInstr &MI) {
   return true;
 }
 
+/// Try to fold away VMERGE_VVM instructions into their operands:
+///
+/// %true = PseudoVADD_VV ...
+/// %x = PseudoVMERGE_VVM_M1 %false, %false, %true, %mask
+/// ->
+/// %x = PseudoVADD_VV_M1_MASK %false, ..., %mask
+///
+/// We can only fold if vmerge's passthru operand, vmerge's false operand and
+/// %true's passthru operand (if it has one) are the same. This is because we
+/// have to consolidate them into one passthru operand in the result.
+///
+/// If %true is masked, then we can use its mask instead of vmerge's if vmerge's
+/// mask is all ones.
+///
+/// The resulting VL is the minimum of the two VLs.
+///
+/// The resulting policy is the effective policy the vmerge would have had,
+/// i.e. whether or not it's passthru operand was implicit-def.
+bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
+  if (RISCV::getRVVMCOpcode(MI.getOpcode()) != RISCV::VMERGE_VVM)
+    return false;
+
+  Register PassthruReg = MI.getOperand(1).getReg();
+  Register FalseReg = MI.getOperand(2).getReg();
+  Register TrueReg = MI.getOperand(3).getReg();
+  if (!TrueReg.isVirtual() || !MRI->hasOneUse(TrueReg))
+    return false;
+  MachineInstr &True = *MRI->getUniqueVRegDef(TrueReg);
+  if (True.getParent() != MI.getParent())
+    return false;
+  const MachineOperand &MaskOp = MI.getOperand(4);
+  MachineInstr *Mask = MRI->getUniqueVRegDef(MaskOp.getReg());
+  assert(Mask);
+
+  const RISCV::RISCVMaskedPseudoInfo *Info =
+      RISCV::lookupMaskedIntrinsicByUnmasked(True.getOpcode());
+  if (!Info)
+    return false;
+
+  // If the EEW of True is different from vmerge's SEW, then we can't fold.
+  if (!hasSameEEW(MI, True))
+    return false;
+
+  // We require that either passthru and false are the same, or that passthru
+  // is undefined.
+  if (PassthruReg && !isKnownSameDefs(PassthruReg, FalseReg))
+    return false;
+
+  // If True has a passthru operand then it needs to be the same as vmerge's
+  // False, since False will be used for the result's passthru operand.
+  Register TruePassthru = True.getOperand(True.getNumExplicitDefs()).getReg();
+  if (RISCVII::isFirstDefTiedToFirstUse(True.getDesc()) && TruePassthru &&
+      !isKnownSameDefs(TruePassthru, FalseReg))
+    return false;
+
+  // Make sure it doesn't raise any observable fp exceptions, since changing the
+  // active elements will affect how fflags is set.
+  if (True.hasUnmodeledSideEffects() || True.mayRaiseFPException())
+    return false;
+
+  const MachineOperand &VMergeVL =
+      MI.getOperand(RISCVII::getVLOpNum(MI.getDesc()));
+  const MachineOperand &TrueVL =
+      True.getOperand(RISCVII::getVLOpNum(True.getDesc()));
+
+  MachineOperand MinVL = MachineOperand::CreateImm(0);
+  if (RISCV::isVLKnownLE(TrueVL, VMergeVL))
+    MinVL = TrueVL;
+  else if (RISCV::isVLKnownLE(VMergeVL, TrueVL))
+    MinVL = VMergeVL;
+  else
+    return false;
+
+  unsigned RVVTSFlags =
+      TII->get(RISCV::getRVVMCOpcode(True.getOpcode())).TSFlags;
+  if (RISCVII::elementsDependOnVL(RVVTSFlags) && !TrueVL.isIdenticalTo(MinVL))
+    return false;
+  if (RISCVII::elementsDependOnMask(RVVTSFlags) && !isAllOnesMask(Mask))
+    return false;
+
+  // Use a tumu policy, relaxing it to tail agnostic provided that the passthru
+  // operand is undefined.
+  //
+  // However, if the VL became smaller than what the vmerge had originally, then
+  // elements past VL that were previously in the vmerge's body will have moved
+  // to the tail. In that case we always need to use tail undisturbed to
+  // preserve them.
+  uint64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
+  if (!PassthruReg && RISCV::isVLKnownLE(VMergeVL, MinVL))
+    Policy |= RISCVVType::TAIL_AGNOSTIC;
+
+  assert(RISCVII::hasVecPolicyOp(True.getDesc().TSFlags) &&
+         "Foldable unmasked pseudo should have a policy op already");
+
+  // Make sure the mask dominates True, otherwise move down True so it does.
+  // VL will always dominate because if its a register they need to be the same.
+  if (!ensureDominates(MaskOp, True))
+    return false;
+
+  True.setDesc(TII->get(Info->MaskedPseudo));
+
+  // Insert the mask operand.
+  // TODO: Increment MaskOpIdx by number of explicit defs?
+  True.insert(&True.getOperand(Info->MaskOpIdx + True.getNumExplicitDefs()),
+              MachineOperand::CreateReg(MaskOp.getReg(), false));
+
+  // Update the passthru, AVL and policy.
+  True.getOperand(True.getNumExplicitDefs()).setReg(FalseReg);
+  True.removeOperand(RISCVII::getVLOpNum(True.getDesc()));
+  True.insert(&True.getOperand(RISCVII::getVLOpNum(True.getDesc())), MinVL);
----------------
topperc wrote:

Should this be `True.insert(True.operands_begin() + RISCVII::getVLOpNum(True.getDesc()), MinVL)` so we aren't assuming that there's a valid operand after VL for the `getOperand` call.

https://github.com/llvm/llvm-project/pull/144076