[llvm] [X86][MC][AsmParser] Reject H-byte regs with VEX/EVEX-encoded 8-bit RR (NDD) (PR #160039)

[via llvm-commits]

================
@@ -4044,6 +4044,24 @@ bool X86AsmParser::validateInstruction(MCInst &Inst, const OperandVector &Ops) {
     }
   }
 
+  unsigned Enc = TSFlags & X86II::EncodingMask;
+  if (Enc == X86II::VEX || Enc == X86II::EVEX || Enc == X86II::XOP) {
+    unsigned NumOps = Inst.getNumOperands();
+    for (unsigned i = 0; i != NumOps; ++i) {
+      const MCOperand &MO = Inst.getOperand(i);
+      if (!MO.isReg())
+        continue;
+      MCRegister Reg = MO.getReg();
+      if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH ||
+          Reg == X86::DH) {
+        StringRef RegName = X86IntelInstPrinter::getRegisterName(Reg);
----------------
woruyu wrote:

```
// Disallow AH/BH/CH/DH with VEX/EVEX/XOP encodings.
unsigned Enc = TSFlags & X86II::EncodingMask;
if (Enc == X86II::VEX || Enc == X86II::EVEX || Enc == X86II::XOP) {
  for (unsigned i = 0, e = Inst.getNumOperands(); i != e; ++i) {
    const MCOperand &MO = Inst.getOperand(i);
    if (!MO.isReg()) continue;
    MCRegister Reg = MO.getReg();
    if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH) {
      StringRef RegName = X86IntelInstPrinter::getRegisterName(Reg);
      return Error(Ops[0]->getStartLoc(),
        "can't encode '" + RegName + "' in a VEX/EVEX-prefixed instruction");
    }
  }
}
```

https://github.com/llvm/llvm-project/pull/160039