[llvm] [X86][MC][AsmParser] Reject H-byte regs with VEX/EVEX-encoded 8-bit RR (NDD) (PR #160039)

[via llvm-commits]

https://github.com/woruyu updated https://github.com/llvm/llvm-project/pull/160039

>From 2c3351100415f78d5bc333408ca58dc0d95e33b3 Mon Sep 17 00:00:00 2001
From: woruyu <1214539920 at qq.com>
Date: Sun, 21 Sep 2025 22:05:05 -0900
Subject: [PATCH 1/2] [X86][MC][AsmParser] Reject H-byte regs with
 VEX/EVEX-encoded 8-bit RR (NDD)

---
 llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp | 18 ++++++++++++++++++
 llvm/test/MC/X86/encoder-fail-VEX-EVEX.s       | 16 ++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 llvm/test/MC/X86/encoder-fail-VEX-EVEX.s

diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index ce5e92135f706..d1ebe6bdb2dfd 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -4044,6 +4044,24 @@ bool X86AsmParser::validateInstruction(MCInst &Inst, const OperandVector &Ops) {
     }
   }
 
+  unsigned Enc = TSFlags & X86II::EncodingMask;
+  if (Enc == X86II::VEX || Enc == X86II::EVEX || Enc == X86II::XOP) {
+    unsigned NumOps = Inst.getNumOperands();
+    for (unsigned i = 0; i != NumOps; ++i) {
+      const MCOperand &MO = Inst.getOperand(i);
+      if (!MO.isReg())
+        continue;
+      MCRegister Reg = MO.getReg();
+      if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH ||
+          Reg == X86::DH) {
+        StringRef RegName = X86IntelInstPrinter::getRegisterName(Reg);
+        return Error(Ops[0]->getStartLoc(),
+                     "can't encode '" + RegName +
+                         "' in a VEX/EVEX-prefixed instruction");
+      }
+    }
+  }
+
   if ((Opcode == X86::PREFETCHIT0 || Opcode == X86::PREFETCHIT1)) {
     const MCOperand &MO = Inst.getOperand(X86::AddrBaseReg);
     if (!MO.isReg() || MO.getReg() != X86::RIP)
diff --git a/llvm/test/MC/X86/encoder-fail-VEX-EVEX.s b/llvm/test/MC/X86/encoder-fail-VEX-EVEX.s
new file mode 100644
index 0000000000000..90512741c9c6c
--- /dev/null
+++ b/llvm/test/MC/X86/encoder-fail-VEX-EVEX.s
@@ -0,0 +1,16 @@
+// RUN: not llvm-mc -triple x86_64-unknown-unknown -x86-asm-syntax=intel --show-encoding %s 2>&1 | FileCheck %s
+
+// CHECK: error: can't encode 'ah' in a VEX/EVEX-prefixed instruction
+add ah, ah, ah
+
+// CHECK: error: can't encode 'ah' in a VEX/EVEX-prefixed instruction
+and ah, byte ptr [-13426159], ah
+
+// CHECK: error: can't encode 'ah' in a VEX/EVEX-prefixed instruction
+ccmpa {dfv=of,cf} byte ptr [r8 + 4*rax + 291], ah
+
+// CHECK: error: can't encode 'ah' in a VEX/EVEX-prefixed instruction
+ccmpae {dfv=of,cf} byte ptr [r8 + 4*rax + 291], ah
+
+// CHECK: error: can't encode 'ah' in a VEX/EVEX-prefixed instruction
+sar ah, byte ptr [-13426159]
\ No newline at end of file

>From 7338d2e886da17679350cd25ea0f67b9908c34f0 Mon Sep 17 00:00:00 2001
From: woruyu <1214539920 at qq.com>
Date: Sun, 21 Sep 2025 22:16:15 -0900
Subject: [PATCH 2/2] fix: details

---
 llvm/test/MC/X86/encoder-fail-VEX-EVEX.s | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/test/MC/X86/encoder-fail-VEX-EVEX.s b/llvm/test/MC/X86/encoder-fail-VEX-EVEX.s
index 90512741c9c6c..bd6efddd46901 100644
--- a/llvm/test/MC/X86/encoder-fail-VEX-EVEX.s
+++ b/llvm/test/MC/X86/encoder-fail-VEX-EVEX.s
@@ -13,4 +13,4 @@ ccmpa {dfv=of,cf} byte ptr [r8 + 4*rax + 291], ah
 ccmpae {dfv=of,cf} byte ptr [r8 + 4*rax + 291], ah
 
 // CHECK: error: can't encode 'ah' in a VEX/EVEX-prefixed instruction
-sar ah, byte ptr [-13426159]
\ No newline at end of file
+sar ah, byte ptr [-13426159]