[compiler-rt] [win/asan] GetInstructionSize: Detect `66 90` two-byte NOP at 32-bit too. (PR #132267)

[via llvm-commits]

llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-compiler-rt-sanitizer

Author: None (bernhardu)

<details>
<summary>Changes</summary>

Observed in Wine when trying to intercept `ExitThread`, which forwards to `ntdll.RtlExitUserThread`.

`gdb` interprets it as `xchg %ax,%ax`.
`llvm-mc` outputs simply `nop`.

```
==Asan-i386-calls-Dynamic-Test.exe==964==interception_win: unhandled instruction at 0x7be27cf0: 66 90 55 89 e5 56 50 8b
```

```
Wine-gdb> bt
#<!-- -->0  0x789a1766 in __interception::GetInstructionSize (address=<optimized out>, rel_offset=<optimized out>) at C:/llvm-mingw/llvm-mingw/llvm-project/compiler-rt/lib/interception/interception_win.cpp:983
#<!-- -->1  0x789ab480 in __sanitizer::SharedPrintfCode(bool, char const*, char*) () at C:/llvm-mingw/llvm-mingw/llvm-project/compiler-rt/lib/sanitizer_common/sanitizer_printf.cpp:311
#<!-- -->2  0x789a18e7 in __interception::OverrideFunctionWithHotPatch (old_func=2078440688, new_func=2023702608, orig_old_func=warning: (Internal error: pc 0x792f1a2c in read in CU, but not in symtab.)warning: (Error: pc 0x792f1a2c in address map, but not in symtab.)0x792f1a2c) at C:/llvm-mingw/llvm-mingw/llvm-project/compiler-rt/lib/interception/interception_win.cpp:1118
#<!-- -->3  0x789a1f34 in __interception::OverrideFunction (old_func=2078440688, new_func=2023702608, orig_old_func=warning: (Internal error: pc 0x792f1a2c in read in CU, but not in symtab.)warning: (Error: pc 0x792f1a2c in address map, but not in symtab.)0x792f1a2c) at C:/llvm-mingw/llvm-mingw/llvm-project/compiler-rt/lib/interception/interception_win.cpp:1224
#<!-- -->4  0x789a24ce in __interception::OverrideFunction (func_name=0x78a0bc43 <vtable for __asan::AsanThreadContext+1163> "ExitThread", new_func=2023702608, orig_old_func=warning: (Internal error: pc 0x792f1a2c in read in CU, but not in symtab.)warning: (Error: pc 0x792f1a2c in address map, but not in symtab.)0x792f1a2c)    at C:/llvm-mingw/llvm-mingw/llvm-project/compiler-rt/lib/interception/interception_win.cpp:1369
#<!-- -->5  0x789f40ef in __asan::InitializePlatformInterceptors () at C:/llvm-mingw/llvm-mingw/llvm-project/compiler-rt/lib/asan/asan_win.cpp:190
#<!-- -->6  0x789e0c3c in __asan::InitializeAsanInterceptors () at C:/llvm-mingw/llvm-mingw/llvm-project/compiler-rt/lib/asan/asan_interceptors.cpp:802
#<!-- -->7  0x789ee6b5 in __asan::AsanInitInternal () at C:/llvm-mingw/llvm-mingw/llvm-project/compiler-rt/lib/asan/asan_rtl.cpp:442
#<!-- -->8  0x789eefb0 in __asan::AsanInitFromRtl () at C:/llvm-mingw/llvm-mingw/llvm-project/compiler-rt/lib/asan/asan_rtl.cpp:522
#<!-- -->9  __asan::AsanInitializer::AsanInitializer (this=<optimized out>) at C:/llvm-mingw/llvm-mingw/llvm-project/compiler-rt/lib/asan/asan_rtl.cpp:542
#<!-- -->10 __cxx_global_var_init () at C:/llvm-mingw/llvm-mingw/llvm-project/compiler-rt/lib/asan/asan_rtl.cpp:546
...
Wine-gdb> disassemble /r 2078440688,2078440688+20
Dump of assembler code from 0x7be27cf0 to 0x7be27d04:
   0x7be27cf0 <_RtlExitUserThread@<!-- -->4+0>: 66 90                   xchg   %ax,%ax
...
```

CC: @<!-- -->zmodem 

---
Full diff: https://github.com/llvm/llvm-project/pull/132267.diff


2 Files Affected:

- (modified) compiler-rt/lib/interception/interception_win.cpp (+1-1) 
- (modified) compiler-rt/lib/interception/tests/interception_win_test.cpp (+1-1) 


``````````diff
diff --git a/compiler-rt/lib/interception/interception_win.cpp b/compiler-rt/lib/interception/interception_win.cpp
index 002b37468a200..b2974cf1934fb 100644
--- a/compiler-rt/lib/interception/interception_win.cpp
+++ b/compiler-rt/lib/interception/interception_win.cpp
@@ -646,6 +646,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0xC033:  // 33 C0 : xor eax, eax
     case 0xC933:  // 33 C9 : xor ecx, ecx
     case 0xD233:  // 33 D2 : xor edx, edx
+    case 0x9066:  // 66 90 : xchg %ax,%ax (Two-byte NOP)
     case 0xDB84:  // 84 DB : test bl,bl
     case 0xC084:  // 84 C0 : test al,al
     case 0xC984:  // 84 C9 : test cl,cl
@@ -726,7 +727,6 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0x5541:  // push r13
     case 0x5641:  // push r14
     case 0x5741:  // push r15
-    case 0x9066:  // Two-byte NOP
     case 0xc084:  // test al, al
     case 0x018a:  // mov al, byte ptr [rcx]
       return 2;
diff --git a/compiler-rt/lib/interception/tests/interception_win_test.cpp b/compiler-rt/lib/interception/tests/interception_win_test.cpp
index 2a7549d230ae2..893f346d73b8a 100644
--- a/compiler-rt/lib/interception/tests/interception_win_test.cpp
+++ b/compiler-rt/lib/interception/tests/interception_win_test.cpp
@@ -845,6 +845,7 @@ const struct InstructionSizeData {
     { 2, {0x33, 0xC0}, 0, "33 C0 : xor eax, eax"},
     { 2, {0x33, 0xC9}, 0, "33 C9 : xor ecx, ecx"},
     { 2, {0x33, 0xD2}, 0, "33 D2 : xor edx, edx"},
+    { 2, {0x66, 0x90}, 0, "66 90 : xchg %ax,%ax (Two-byte NOP)"},
     { 2, {0x6A, 0x71}, 0, "6A XX : push XX"},
     { 2, {0x84, 0xC0}, 0, "84 C0 : test al,al"},
     { 2, {0x84, 0xC9}, 0, "84 C9 : test cl,cl"},
@@ -887,7 +888,6 @@ const struct InstructionSizeData {
     { 2, {0x41, 0x55}, 0, "41 55 : push r13"},
     { 2, {0x41, 0x56}, 0, "41 56 : push r14"},
     { 2, {0x41, 0x57}, 0, "41 57 : push r15"},
-    { 2, {0x66, 0x90}, 0, "66 90 : Two-byte NOP"},
     { 2, {0x84, 0xc0}, 0, "84 c0 : test al, al"},
     { 2, {0x8a, 0x01}, 0, "8a 01 : mov al, byte ptr [rcx]"},
     { 3, {0x0f, 0xb6, 0x01}, 0, "0f b6 01 : movzx eax, BYTE PTR [rcx]"},

``````````

</details>


https://github.com/llvm/llvm-project/pull/132267