[llvm] r252743 - Visibly fail if attempting to encode register AH, BH, CH, DH in a REX-prefixed instruction.

[Douglas Katzman via llvm-commits]

Author: dougk
Date: Wed Nov 11 09:51:16 2015
New Revision: 252743

URL: http://llvm.org/viewvc/llvm-project?rev=252743&view=rev
Log:
Visibly fail if attempting to encode register AH,BH,CH,DH in a REX-prefixed instruction.

Differential Revision: http://reviews.llvm.org/D13316
Fixes PR25003

Added:
    llvm/trunk/test/MC/X86/encoder-fail.s
Modified:
    llvm/trunk/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp

Modified: llvm/trunk/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp?rev=252743&r1=252742&r2=252743&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp (original)
+++ llvm/trunk/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp Wed Nov 11 09:51:16 2015
@@ -988,6 +988,8 @@ void X86MCCodeEmitter::EmitVEXOpcodePref
 static unsigned DetermineREXPrefix(const MCInst &MI, uint64_t TSFlags,
                                    const MCInstrDesc &Desc) {
   unsigned REX = 0;
+  bool UsesHighByteReg = false;
+
   if (TSFlags & X86II::REX_W)
     REX |= 1 << 3; // set REX.W
 
@@ -1004,6 +1006,8 @@ static unsigned DetermineREXPrefix(const
     const MCOperand &MO = MI.getOperand(i);
     if (!MO.isReg()) continue;
     unsigned Reg = MO.getReg();
+    if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)
+      UsesHighByteReg = true;
     if (!X86II::isX86_64NonExtLowByteReg(Reg)) continue;
     // FIXME: The caller of DetermineREXPrefix slaps this prefix onto anything
     // that returns non-zero.
@@ -1073,6 +1077,9 @@ static unsigned DetermineREXPrefix(const
     }
     break;
   }
+  if (REX && UsesHighByteReg)
+    report_fatal_error("Cannot encode high byte register in REX-prefixed instruction");
+
   return REX;
 }
 

Added: llvm/trunk/test/MC/X86/encoder-fail.s
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/MC/X86/encoder-fail.s?rev=252743&view=auto
==============================================================================
--- llvm/trunk/test/MC/X86/encoder-fail.s (added)
+++ llvm/trunk/test/MC/X86/encoder-fail.s Wed Nov 11 09:51:16 2015
@@ -0,0 +1,3 @@
+// RUN: not llvm-mc -triple x86_64-unknown-unknown --show-encoding %s 2>&1 | FileCheck %s
+// CHECK: LLVM ERROR: Cannot encode high byte register in REX-prefixed instruction
+ movzx %dh, %rsi