[llvm] AMDGPU: VALU data fast-forwarding needs no s_delay (PR #205481)

[Jay Foad via llvm-commits]

================
@@ -346,6 +352,205 @@ class AMDGPUInsertDelayAlu {
     return (Imm & 0x780) ? nullptr : DelayAlu;
   }
 
+  static bool isFastForwardProducer(const MachineInstr &MI,
+                                    const MachineOperand &MO, Register VccReg,
+                                    Register ExecReg) {
+    if (!MO.isReg() || !MO.isDef())
+      return false;
+
+    if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
+      return false;
+
+    const TargetRegisterInfo *TRI =
+        MI.getMF()->getSubtarget().getRegisterInfo();
+    Register Reg = MO.getReg();
+    if (TRI->isSubRegisterEq(Reg, VccReg)) {
+      switch (MI.getOpcode()) {
+      // VOP2 carry-out producers (implicit VCC)
+      case AMDGPU::V_ADD_CO_U32_e32:
+      case AMDGPU::V_SUB_CO_U32_e32:
+      case AMDGPU::V_SUBREV_CO_U32_e32:
+      case AMDGPU::V_ADDC_U32_e32:
+      case AMDGPU::V_SUBB_U32_e32:
+      case AMDGPU::V_SUBBREV_U32_e32:
+      // VOP3 carry-out producers (explicit VCC)
+      case AMDGPU::V_ADD_CO_U32_e64:
+      case AMDGPU::V_SUB_CO_U32_e64:
+      case AMDGPU::V_SUBREV_CO_U32_e64:
+      case AMDGPU::V_ADDC_U32_e64:
+      case AMDGPU::V_SUBB_U32_e64:
+      case AMDGPU::V_SUBBREV_U32_e64:
+        return true;
+      default:
+        // V_CMP* produces condition masks (excluding V_CMPX)
+        if (MI.isCompare() && !AMDGPU::isVCMPX(MI.getOpcode())) {
+          // VOPC: implicit VCC def
+          return true;
+        }
+      }
+    } else if (TRI->isSubRegisterEq(Reg, ExecReg)) {
+      // V_CMPX writes EXEC (implicit)
+      if (AMDGPU::isVCMPX(MI.getOpcode()))
+        return true;
+    } else if (AMDGPU::isSGPR(Reg, TRI)) {
+      switch (MI.getOpcode()) {
+      // VOP3 carry-out producers (explicit SGPR)
+      case AMDGPU::V_ADD_CO_U32_e64:
+      case AMDGPU::V_SUB_CO_U32_e64:
+      case AMDGPU::V_SUBREV_CO_U32_e64:
+      case AMDGPU::V_ADDC_U32_e64:
+      case AMDGPU::V_SUBB_U32_e64:
+      case AMDGPU::V_SUBBREV_U32_e64:
+      case AMDGPU::V_DIV_SCALE_F32_e64:
+      case AMDGPU::V_DIV_SCALE_F64_e64:
+        return true;
+      default:
+        // V_CMP* produces condition masks (excluding V_CMPX)
+        if (MI.isCompare() && !AMDGPU::isVCMPX(MI.getOpcode())) {
+          // VOP3: explicit SGPR def (operand 0)
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  static unsigned int getVOPDComponentOpCode(const MachineInstr &MI,
+                                             unsigned OpNo,
+                                             const MachineOperand &MO) {
+    Register Reg = MO.getReg();
+    auto MIOpCode = MI.getOpcode();
+    // Get the component instruction descriptors for VOPD.
+    auto [OpX, OpY] = AMDGPU::getVOPDComponents(MIOpCode);
+    const MCInstrInfo *MCII = MI.getMF()->getTarget().getMCInstrInfo();
+
+    if (MO.isImplicit()) {
+      const TargetRegisterInfo *TRI =
+          MI.getMF()->getSubtarget().getRegisterInfo();
+      for (unsigned CompOp : {OpX, OpY}) {
+        const MCInstrDesc &CompDesc = MCII->get(CompOp);
+        bool UsesReg = any_of(CompDesc.implicit_uses(), [&](MCPhysReg R) {
+          return TRI->regsOverlap(R, Reg);
+        });
+        if (UsesReg) {
+          MIOpCode = CompOp;
+          break;
+        }
+      }
+    } else {
+      // Explicit source: identify which component/source THIS operand is by
+      // matching its operand index (OpNo). The same register can appear in
+      // multiple slots, so matching by register value is not reliable.
+      const auto &InstInfo = AMDGPU::getVOPDInstInfo(MIOpCode, MCII);
+
+      // Map a parsed source index (0/1/2) to the matching named operand for a
+      // standalone component opcode.
+      static constexpr AMDGPU::OpName SrcNames[] = {
+          AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};
+
+      bool Found = false;
+      for (auto CompIdx : AMDGPU::VOPD::COMPONENTS) {
+        const auto &CInfo = InstInfo[CompIdx];
+        unsigned CompSrcOperandsNum = CInfo.getCompParsedSrcOperandsNum();
+        for (unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum;
+             ++CompSrcIdx) {
+          if (CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx) != OpNo)
+            continue;
+
+          // Re-target analysis to this component's standalone opcode.
+          MIOpCode = (CompIdx == AMDGPU::VOPD::X) ? OpX : OpY;
+
+          // Translate the parsed src index into MIOpCode's operand-index
+          assert(CompSrcIdx < std::size(SrcNames) &&
+                 "unexpected VOPD src index");
+          int NamedIdx =
+              AMDGPU::getNamedOperandIdx(MIOpCode, SrcNames[CompSrcIdx]);
+          if (NamedIdx >= 0)
+            OpNo = static_cast<unsigned>(NamedIdx);
+          Found = true;
+          break;
+        }
+        if (Found)
+          break;
+      }
+    }
+    return MIOpCode;
+  }
+
+  static bool isFastForwardConsumer(const MachineInstr &MI,
+                                    const MachineOperand &MO, Register VccReg,
+                                    Register ExecReg, unsigned OpNo) {
+    if (!MO.isReg() || !MO.isUse())
----------------
jayfoad wrote:

This is checked in the caller. Either remove this or change it to an assert.

https://github.com/llvm/llvm-project/pull/205481