[compiler-rt] r275180 - [compiler-rt] Enhance function padding detection for function interception

[Etienne Bergeron via llvm-commits]

Author: etienneb
Date: Tue Jul 12 10:33:04 2016
New Revision: 275180

URL: http://llvm.org/viewvc/llvm-project?rev=275180&view=rev
Log:
[compiler-rt] Enhance function padding detection for function interception

Summary:
Many CRT (64-bits) functions contains a "hint-nop". The current padding
detection is not able to recognize the 10-bytes padding and the HotPatch
hooking technique cannot be used.

Other patterns may be discover and may be added later.

Reviewers: rnk

Subscribers: llvm-commits, wang0109, chrisha

Differential Revision: http://reviews.llvm.org/D22258

Modified:
    compiler-rt/trunk/lib/interception/interception_win.cc

Modified: compiler-rt/trunk/lib/interception/interception_win.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/interception/interception_win.cc?rev=275180&r1=275179&r2=275180&view=diff
==============================================================================
--- compiler-rt/trunk/lib/interception/interception_win.cc (original)
+++ compiler-rt/trunk/lib/interception/interception_win.cc Tue Jul 12 10:33:04 2016
@@ -202,6 +202,29 @@ static bool IsMemoryPadding(uptr address
   return true;
 }
 
+static const u8 kHintNop10Bytes[] = {
+  0x66, 0x66, 0x0F, 0x1F, 0x84,
+  0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+template<class T>
+static bool FunctionHasPrefix(uptr address, const T &pattern) {
+  u8* function = (u8*)address - sizeof(pattern);
+  for (size_t i = 0; i < sizeof(pattern); ++i)
+    if (function[i] != pattern[i])
+      return false;
+  return true;
+}
+
+static bool FunctionHasPadding(uptr address, uptr size) {
+  if (IsMemoryPadding(address - size, size))
+    return true;
+  if (size <= sizeof(kHintNop10Bytes) &&
+      FunctionHasPrefix(address, kHintNop10Bytes))
+    return true;
+  return false;
+}
+
 static void WritePadding(uptr from, uptr size) {
   _memset((void*)from, 0xCC, (size_t)size);
 }
@@ -617,7 +640,7 @@ bool OverrideFunctionWithHotPatch(
   // Validate that the function is hot patchable.
   size_t instruction_size = GetInstructionSize(old_func);
   if (instruction_size < kShortJumpInstructionLength ||
-      !IsMemoryPadding(header, kHotPatchHeaderLen))
+      !FunctionHasPadding(old_func, kHotPatchHeaderLen))
     return false;
 
   if (orig_old_func) {