[llvm] [X86][MC][AsmParser] Reject H-byte regs with VEX/EVEX-encoded 8-bit RR (NDD) (PR #160039)

[via llvm-commits]

llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-backend-x86

Author: woruyu (woruyu)

<details>
<summary>Changes</summary>

### Summary
This PR resolves https://github.com/llvm/llvm-project/issues/158585.

---
Full diff: https://github.com/llvm/llvm-project/pull/160039.diff


2 Files Affected:

- (modified) llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp (+18) 
- (added) llvm/test/MC/X86/encoder-fail-VEX-EVEX.s (+16) 


``````````diff
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index ce5e92135f706..d1ebe6bdb2dfd 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -4044,6 +4044,24 @@ bool X86AsmParser::validateInstruction(MCInst &Inst, const OperandVector &Ops) {
     }
   }
 
+  unsigned Enc = TSFlags & X86II::EncodingMask;
+  if (Enc == X86II::VEX || Enc == X86II::EVEX || Enc == X86II::XOP) {
+    unsigned NumOps = Inst.getNumOperands();
+    for (unsigned i = 0; i != NumOps; ++i) {
+      const MCOperand &MO = Inst.getOperand(i);
+      if (!MO.isReg())
+        continue;
+      MCRegister Reg = MO.getReg();
+      if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH ||
+          Reg == X86::DH) {
+        StringRef RegName = X86IntelInstPrinter::getRegisterName(Reg);
+        return Error(Ops[0]->getStartLoc(),
+                     "can't encode '" + RegName +
+                         "' in a VEX/EVEX-prefixed instruction");
+      }
+    }
+  }
+
   if ((Opcode == X86::PREFETCHIT0 || Opcode == X86::PREFETCHIT1)) {
     const MCOperand &MO = Inst.getOperand(X86::AddrBaseReg);
     if (!MO.isReg() || MO.getReg() != X86::RIP)
diff --git a/llvm/test/MC/X86/encoder-fail-VEX-EVEX.s b/llvm/test/MC/X86/encoder-fail-VEX-EVEX.s
new file mode 100644
index 0000000000000..90512741c9c6c
--- /dev/null
+++ b/llvm/test/MC/X86/encoder-fail-VEX-EVEX.s
@@ -0,0 +1,16 @@
+// RUN: not llvm-mc -triple x86_64-unknown-unknown -x86-asm-syntax=intel --show-encoding %s 2>&1 | FileCheck %s
+
+// CHECK: error: can't encode 'ah' in a VEX/EVEX-prefixed instruction
+add ah, ah, ah
+
+// CHECK: error: can't encode 'ah' in a VEX/EVEX-prefixed instruction
+and ah, byte ptr [-13426159], ah
+
+// CHECK: error: can't encode 'ah' in a VEX/EVEX-prefixed instruction
+ccmpa {dfv=of,cf} byte ptr [r8 + 4*rax + 291], ah
+
+// CHECK: error: can't encode 'ah' in a VEX/EVEX-prefixed instruction
+ccmpae {dfv=of,cf} byte ptr [r8 + 4*rax + 291], ah
+
+// CHECK: error: can't encode 'ah' in a VEX/EVEX-prefixed instruction
+sar ah, byte ptr [-13426159]
\ No newline at end of file

``````````

</details>


https://github.com/llvm/llvm-project/pull/160039