[llvm] [X86][MC]Fix wrong action for encode enqcmd/enqcmds (PR #77571)

[via llvm-commits]

https://github.com/XinWang10 created https://github.com/llvm/llvm-project/pull/77571

enqcmd/enqcmds are special for its mem operand similar to movdir64b(could see https://github.com/llvm/llvm-project/commit/4dd5e9c60efa9), 67 prefex can not only modify its add size,
so it's mem base and index reg should be the same type as source reg, such as
enqcmd(%rdx), rcx, and could not be enqcmd(%edx), rcx.

>From 47fc054fa3f5149445f4cc0e10a08daaf295cb8c Mon Sep 17 00:00:00 2001
From: "Wang, Xin10" <xin10.wang at intel.com>
Date: Wed, 10 Jan 2024 01:49:17 -0800
Subject: [PATCH] [X86][MC]Fix wrong action for encode enqcmd/enqcmds

---
 llvm/lib/Target/X86/X86InstrMisc.td | 12 +++---
 llvm/test/MC/X86/index-operations.s | 58 +++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/X86/X86InstrMisc.td b/llvm/lib/Target/X86/X86InstrMisc.td
index 97c625a64cfc0b..753cf62392a17b 100644
--- a/llvm/lib/Target/X86/X86InstrMisc.td
+++ b/llvm/lib/Target/X86/X86InstrMisc.td
@@ -1523,28 +1523,28 @@ def MOVDIR64B64_EVEX : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem_GR64:$
 // ENQCMD/S - Enqueue 64-byte command as user with 64-byte write atomicity
 //
 let SchedRW = [WriteStore], Defs = [EFLAGS] in {
-  def ENQCMD16 : I<0xF8, MRMSrcMem, (outs), (ins GR16:$dst, i512mem:$src),
+  def ENQCMD16 : I<0xF8, MRMSrcMem, (outs), (ins GR16:$dst, i512mem_GR16:$src),
                  "enqcmd\t{$src, $dst|$dst, $src}",
                  [(set EFLAGS, (X86enqcmd GR16:$dst, addr:$src))]>,
                  T8, XD, AdSize16, Requires<[HasENQCMD, Not64BitMode]>;
-  def ENQCMD32 : I<0xF8, MRMSrcMem, (outs), (ins GR32:$dst, i512mem:$src),
+  def ENQCMD32 : I<0xF8, MRMSrcMem, (outs), (ins GR32:$dst, i512mem_GR32:$src),
                  "enqcmd\t{$src, $dst|$dst, $src}",
                  [(set EFLAGS, (X86enqcmd GR32:$dst, addr:$src))]>,
                  T8, XD, AdSize32, Requires<[HasENQCMD]>;
-  def ENQCMD64 : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem:$src),
+  def ENQCMD64 : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem_GR64:$src),
                  "enqcmd\t{$src, $dst|$dst, $src}",
                  [(set EFLAGS, (X86enqcmd GR64:$dst, addr:$src))]>,
                  T8, XD, AdSize64, Requires<[HasENQCMD, In64BitMode]>;
 
-  def ENQCMDS16 : I<0xF8, MRMSrcMem, (outs), (ins GR16:$dst, i512mem:$src),
+  def ENQCMDS16 : I<0xF8, MRMSrcMem, (outs), (ins GR16:$dst, i512mem_GR16:$src),
                  "enqcmds\t{$src, $dst|$dst, $src}",
                  [(set EFLAGS, (X86enqcmds GR16:$dst, addr:$src))]>,
                  T8, XS, AdSize16, Requires<[HasENQCMD, Not64BitMode]>;
-  def ENQCMDS32 : I<0xF8, MRMSrcMem, (outs), (ins GR32:$dst, i512mem:$src),
+  def ENQCMDS32 : I<0xF8, MRMSrcMem, (outs), (ins GR32:$dst, i512mem_GR32:$src),
                  "enqcmds\t{$src, $dst|$dst, $src}",
                  [(set EFLAGS, (X86enqcmds GR32:$dst, addr:$src))]>,
                  T8, XS, AdSize32, Requires<[HasENQCMD]>;
-  def ENQCMDS64 : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem:$src),
+  def ENQCMDS64 : I<0xF8, MRMSrcMem, (outs), (ins GR64:$dst, i512mem_GR64:$src),
                  "enqcmds\t{$src, $dst|$dst, $src}",
                  [(set EFLAGS, (X86enqcmds GR64:$dst, addr:$src))]>,
                  T8, XS, AdSize64, Requires<[HasENQCMD, In64BitMode]>;
diff --git a/llvm/test/MC/X86/index-operations.s b/llvm/test/MC/X86/index-operations.s
index 899cf4656549f6..59498b0ced1249 100644
--- a/llvm/test/MC/X86/index-operations.s
+++ b/llvm/test/MC/X86/index-operations.s
@@ -188,3 +188,61 @@ movdir64b 291(%esi, %eiz, 4), %ebx
 
 movdir64b 291(%rsi, %riz, 4), %rbx
 // 64: movdir64b 291(%rsi,%riz,4), %rbx # encoding: [0x66,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+
+enqcmd	291(%si), %ecx
+// ERR64: error: invalid 16-bit base register
+// ERR32: invalid operand
+// ERR16: invalid operand
+
+enqcmd	291(%esi), %cx
+// ERR64: error: invalid operand for instruction
+// ERR32: invalid operand
+// ERR16: invalid operand
+
+enqcmd (%rdx), %r15d
+// ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand
+
+enqcmd (%edx), %r15
+// ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand
+
+enqcmd (%eip), %ebx
+// 64: enqcmd (%eip), %ebx # encoding: [0x67,0xf2,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
+
+enqcmd (%rip), %rbx
+// 64: enqcmd (%rip), %rbx # encoding: [0xf2,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
+
+enqcmd 291(%esi, %eiz, 4), %ebx
+// 64: enqcmd 291(%esi,%eiz,4), %ebx # encoding: [0x67,0xf2,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+// 32: enqcmd 291(%esi,%eiz,4), %ebx # encoding: [0xf2,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+
+enqcmd 291(%rsi, %riz, 4), %rbx
+// 64: enqcmd 291(%rsi,%riz,4), %rbx # encoding: [0xf2,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+
+enqcmds	291(%si), %ecx
+// ERR64: error: invalid 16-bit base register
+// ERR32: invalid operand
+// ERR16: invalid operand
+
+enqcmds	291(%esi), %cx
+// ERR64: error: invalid operand for instruction
+// ERR32: invalid operand
+// ERR16: invalid operand
+
+enqcmds (%rdx), %r15d
+// ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand
+
+enqcmds (%edx), %r15
+// ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand
+
+enqcmds (%eip), %ebx
+// 64: enqcmds (%eip), %ebx # encoding: [0x67,0xf3,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
+
+enqcmds (%rip), %rbx
+// 64: enqcmds (%rip), %rbx # encoding: [0xf3,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
+
+enqcmds 291(%esi, %eiz, 4), %ebx
+// 64: enqcmds 291(%esi,%eiz,4), %ebx # encoding: [0x67,0xf3,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+// 32: enqcmds 291(%esi,%eiz,4), %ebx # encoding: [0xf3,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+
+enqcmds 291(%rsi, %riz, 4), %rbx
+// 64: enqcmds 291(%rsi,%riz,4), %rbx # encoding: [0xf3,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]