[llvm] [X86][MC][AsmParser] Reject H-byte regs with VEX/EVEX-encoded 8-bit RR (NDD) (PR #160039)

[Phoebe Wang via llvm-commits]

================
@@ -4031,16 +4034,23 @@ bool X86AsmParser::validateInstruction(MCInst &Inst, const OperandVector &Ops) {
       MCRegister Reg = MO.getReg();
       if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)
         HReg = Reg;
-      if (X86II::isX86_64NonExtLowByteReg(Reg) ||
-          X86II::isX86_64ExtendedReg(Reg))
+      if (Enc == 0 && (X86II::isX86_64NonExtLowByteReg(Reg) ||
+                       X86II::isX86_64ExtendedReg(Reg)))
         UsesRex = true;
     }
 
-    if (UsesRex && HReg) {
+    if (Enc == 0 && UsesRex && HReg) {
       StringRef RegName = X86IntelInstPrinter::getRegisterName(HReg);
       return Error(Ops[0]->getStartLoc(),
-                   "can't encode '" + RegName + "' in an instruction requiring "
-                   "REX prefix");
+                   "can't encode '" + RegName +
+                       "' in an instruction requiring REX prefix");
+    }
+
+    if ((UsesEvex || UsesRex2) && HReg) {
+      StringRef RegName = X86IntelInstPrinter::getRegisterName(HReg);
+      return Error(Ops[0]->getStartLoc(),
+                   "can't encode '" + RegName.str() +
+                       "' in an instruction requiring EVEX/REX2 prefix");
----------------
phoebewang wrote:

This can be combine to:
```
if (HReg) {
  if (UsesRex) ...
```

https://github.com/llvm/llvm-project/pull/160039