[PATCH] D39093: Certain VEX instructions ignore the W-bit that shouldn'

[Andrew V. Tischenko via Phabricator via llvm-commits]

avt77 created this revision.

This patch should close Bug 11304 - [x86 disassembler] Certain VEX instructions ignore the W-bit that shouldn.
It is not the complete version of the patch: I want only to ask is it possible to make this fix in the given way? I suppose it's better to do it in *.td file(s) but I don't have any idea how to do it that's why I introduce this my approach. Please review and give me the answer: should I continue in this way or I should switch to *.td file(s). In the last case please give me a hint how to do it.


https://reviews.llvm.org/D39093

Files:
  lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp
  test/MC/Disassembler/X86/x86-64-err.txt


Index: test/MC/Disassembler/X86/x86-64-err.txt
===================================================================
--- test/MC/Disassembler/X86/x86-64-err.txt
+++ test/MC/Disassembler/X86/x86-64-err.txt
@@ -4,3 +4,14 @@
 # 64: warning: invalid instruction encoding
 # 32: into
 0xce
+
+# 64: invalid instruction encoding
+0xc4,0x62,0xf9,0x18,0x20
+# 64: invalid instruction encoding
+0xc4,0x62,0xfd,0x18,0x20
+# 64: invalid instruction encoding
+0xc4,0xc2,0xfd,0x19,0xcc
+# 64: invalid instruction encoding
+0xc4,0xe2,0xfd,0x1a,0x08
+# 64: invalid instruction encoding
+0xc4,0xe3,0xfd,0x39,0xc5,0x01
Index: lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp
===================================================================
--- lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp
+++ lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp
@@ -660,6 +660,41 @@
 
   return 0;
 }
+/*
+#UD If VEX.W = 1 in all modes
+VBLENDVPD, VBLENDVPS, VPBLENDVB, VTESTPD, VTESTPS, VPBLENDD, VPERMD,
+VPERMPS, VPERM2I128, VPSRAVD, VPERMILPD, VPERMILPS, VPERM2F128
+
+VEXTRACTF128, VBROADCASTSS, VBROADCASTSD, VBROADCASTF128,
+VINSERTF128, VMASKMOVPS, VMASKMOVPD, VBROADCASTI128,
+VPBROADCASTB/W/D, VEXTRACTI128, VINSERTI128
+
+VCVTPH2PS, VCVTPS2PH
+
+#UD If VEX.L = 0
+VEXTRACTF128,
+VPERM2F128,
+VBROADCASTSD,
+VBROADCASTF128,
+VINSERTF128, 
+*/
+
+static int checkVEX3bEncoding (struct InternalInstruction* insn) {
+  // The following instructions have to use vex.w == 0
+  switch (insn->opcode) {
+    case 0x39:  // vextracti128   xmm1/m128, ymm2, imm8
+    case 0x38:  // vinserti128    $1, %xmm1, %ymm0, %ymm0
+    case 0x4b:  // vblendvpd      %xmm1, %xmm2, %xmm3, %xmm4
+    case 0x18:  // vbroadcastss   (%rax), %xmm12
+    case 0x19:  // vbroadcastsd   %xmm12, %ymm1
+    case 0x1a:  // vbroadcastf128 (%rax), %ymm1
+      if (wFromEVEX3of4(insn->vectorExtensionPrefix[2]))
+        return -2;  // Invalid vex.w == 1
+    default:
+      break;
+  }
+  return 0;
+}
 
 /*
  * readOpcode - Reads the opcode (excepting the ModR/M byte in the case of
@@ -701,14 +736,17 @@
       return -1;
     case VEX_LOB_0F:
       insn->opcodeType = TWOBYTE;
-      return consumeByte(insn, &insn->opcode);
+      break;
     case VEX_LOB_0F38:
       insn->opcodeType = THREEBYTE_38;
-      return consumeByte(insn, &insn->opcode);
+      break;
     case VEX_LOB_0F3A:
       insn->opcodeType = THREEBYTE_3A;
-      return consumeByte(insn, &insn->opcode);
+      break;
     }
+    if (consumeByte(insn, &insn->opcode))
+      return -1;
+    return checkVEX3bEncoding(insn);
   } else if (insn->vectorExtensionType == TYPE_VEX_2B) {
     insn->opcodeType = TWOBYTE;
     return consumeByte(insn, &insn->opcode);
@@ -764,9 +802,7 @@
    * At this point we have consumed the full opcode.
    * Anything we consume from here on must be unconsumed.
    */
-
   insn->opcode = current;
-
   return 0;
 }
 


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D39093.119582.patch
Type: text/x-patch
Size: 2893 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20171019/1ab150da/attachment.bin>