[llvm] [hwasan] Optimize outlined memaccess for fixed shadow on Aarch64 (PR #88544)
Thurston Dang via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 23 13:51:17 PDT 2024
https://github.com/thurstond updated https://github.com/llvm/llvm-project/pull/88544
>From 04148459ea00e05b61c1b62ab0514e7014af1e0b Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston at google.com>
Date: Tue, 23 Apr 2024 00:40:55 +0000
Subject: [PATCH 1/3] [hwasan] Optimize outlined memaccess for fixed shadow on
Aarch64
The HWASan transform currently always uses x20 to pass the shadow base to hwasan_check_memaccess_shortgranules, even
if the shadow base is a constant known at compile time (via -hwasan-mapping-offset). This patch uses the fixed
shadow variant of the hwasan_check_memaccess_shortgranules intrinsic (introduced in
https://github.com/llvm/llvm-project/commit/365bddf634993d5ea357e9715d8aacd7ee40c4b5), allowing the shadow base to
be materialized inside the memaccess callee.
We currently only support this optimization for AArch64. It is a no-op on other platforms, or if -hwasan-mapping-offset is not specified.
Note: when -hwasan-mapping-offset is specified, it is necessary to specify HWASAN_OPTIONS=fixed_shadow_base=... (see ea991a1) to ensure that the runtime will map the shadow appropriately.
---
.../Instrumentation/HWAddressSanitizer.cpp | 31 ++++++++++++++++---
1 file changed, 26 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index a35f24447cc39b..322beb8f0d58f5 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -930,11 +930,32 @@ void HWAddressSanitizer::instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
IRBuilder<> IRB(InsertBefore);
Module *M = IRB.GetInsertBlock()->getParent()->getParent();
- IRB.CreateCall(Intrinsic::getDeclaration(
- M, UseShortGranules
- ? Intrinsic::hwasan_check_memaccess_shortgranules
- : Intrinsic::hwasan_check_memaccess),
- {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
+ bool useFixedShadowIntrinsic = false;
+ // The memaccess fixed shadow intrinsic is only supported on AArch64,
+ // which allows a 16-bit immediate to be left-shifted by 32.
+ // Since kShadowBaseAlignment == 32, and Linux by default will not
+ // mmap above 48-bits, practically any valid shadow offset is
+ // representable.
+ // In particular, an offset of 4TB (1024 << 32) is representable, and
+ // ought to be good enough for anybody.
+ if (TargetTriple.isAArch64() && ClMappingOffset.getNumOccurrences() > 0) {
+ uint16_t offset_shifted = Mapping.Offset >> 32;
+ useFixedShadowIntrinsic = (uint64_t)offset_shifted << 32 == Mapping.Offset;
+ }
+
+ if (useFixedShadowIntrinsic)
+ IRB.CreateCall(
+ Intrinsic::getDeclaration(
+ M, UseShortGranules
+ ? Intrinsic::hwasan_check_memaccess_shortgranules_fixedshadow
+ : Intrinsic::hwasan_check_memaccess_fixedshadow),
+ {Ptr, ConstantInt::get(Int32Ty, AccessInfo), ConstantInt::get(Int64Ty, Mapping.Offset)});
+ else
+ IRB.CreateCall(Intrinsic::getDeclaration(
+ M, UseShortGranules
+ ? Intrinsic::hwasan_check_memaccess_shortgranules
+ : Intrinsic::hwasan_check_memaccess),
+ {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
}
void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
>From cdbecc6603e835cd9ff2b10fcdd1ba5bd467a1b6 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston at google.com>
Date: Tue, 23 Apr 2024 20:35:07 +0000
Subject: [PATCH 2/3] Update fixed-shadow.ll test; use 'Mapping.Offset !=
kDynamicShadowSentinel' per Vitaly's feedback
---
.../Instrumentation/HWAddressSanitizer.cpp | 2 +-
.../HWAddressSanitizer/fixed-shadow.ll | 20 +++++++++----------
2 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index 322beb8f0d58f5..61fb9d2d4196cd 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -938,7 +938,7 @@ void HWAddressSanitizer::instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
// representable.
// In particular, an offset of 4TB (1024 << 32) is representable, and
// ought to be good enough for anybody.
- if (TargetTriple.isAArch64() && ClMappingOffset.getNumOccurrences() > 0) {
+ if (TargetTriple.isAArch64() && Mapping.Offset != kDynamicShadowSentinel) {
uint16_t offset_shifted = Mapping.Offset >> 32;
useFixedShadowIntrinsic = (uint64_t)offset_shifted << 32 == Mapping.Offset;
}
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/fixed-shadow.ll b/llvm/test/Instrumentation/HWAddressSanitizer/fixed-shadow.ll
index 05a927f0385ba0..980189c5607f31 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/fixed-shadow.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/fixed-shadow.ll
@@ -12,7 +12,7 @@ define i8 @test_load8(ptr %a) sanitize_hwaddress {
; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr inttoptr (i64 4398046511104 to ptr))
-; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 0, i64 4398046511104)
; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[A]], align 4
; CHECK-NEXT: ret i8 [[B]]
;
@@ -26,7 +26,7 @@ define i16 @test_load16(ptr %a) sanitize_hwaddress {
; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr inttoptr (i64 4398046511104 to ptr))
-; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 1, i64 4398046511104)
; CHECK-NEXT: [[B:%.*]] = load i16, ptr [[A]], align 4
; CHECK-NEXT: ret i16 [[B]]
;
@@ -40,7 +40,7 @@ define i32 @test_load32(ptr %a) sanitize_hwaddress {
; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr inttoptr (i64 4398046511104 to ptr))
-; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 2, i64 4398046511104)
; CHECK-NEXT: [[B:%.*]] = load i32, ptr [[A]], align 4
; CHECK-NEXT: ret i32 [[B]]
;
@@ -54,7 +54,7 @@ define i64 @test_load64(ptr %a) sanitize_hwaddress {
; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr inttoptr (i64 4398046511104 to ptr))
-; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 3, i64 4398046511104)
; CHECK-NEXT: [[B:%.*]] = load i64, ptr [[A]], align 8
; CHECK-NEXT: ret i64 [[B]]
;
@@ -68,7 +68,7 @@ define i128 @test_load128(ptr %a) sanitize_hwaddress {
; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr inttoptr (i64 4398046511104 to ptr))
-; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 4, i64 4398046511104)
; CHECK-NEXT: [[B:%.*]] = load i128, ptr [[A]], align 16
; CHECK-NEXT: ret i128 [[B]]
;
@@ -97,7 +97,7 @@ define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress {
; CHECK-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr inttoptr (i64 4398046511104 to ptr))
-; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 16, i64 4398046511104)
; CHECK-NEXT: store i8 [[B]], ptr [[A]], align 4
; CHECK-NEXT: ret void
;
@@ -111,7 +111,7 @@ define void @test_store16(ptr %a, i16 %b) sanitize_hwaddress {
; CHECK-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr inttoptr (i64 4398046511104 to ptr))
-; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 17, i64 4398046511104)
; CHECK-NEXT: store i16 [[B]], ptr [[A]], align 4
; CHECK-NEXT: ret void
;
@@ -125,7 +125,7 @@ define void @test_store32(ptr %a, i32 %b) sanitize_hwaddress {
; CHECK-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr inttoptr (i64 4398046511104 to ptr))
-; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 18, i64 4398046511104)
; CHECK-NEXT: store i32 [[B]], ptr [[A]], align 4
; CHECK-NEXT: ret void
;
@@ -139,7 +139,7 @@ define void @test_store64(ptr %a, i64 %b) sanitize_hwaddress {
; CHECK-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr inttoptr (i64 4398046511104 to ptr))
-; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 19, i64 4398046511104)
; CHECK-NEXT: store i64 [[B]], ptr [[A]], align 8
; CHECK-NEXT: ret void
;
@@ -153,7 +153,7 @@ define void @test_store128(ptr %a, i128 %b) sanitize_hwaddress {
; CHECK-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr inttoptr (i64 4398046511104 to ptr))
-; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; CHECK-NEXT: call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 20, i64 4398046511104)
; CHECK-NEXT: store i128 [[B]], ptr [[A]], align 16
; CHECK-NEXT: ret void
;
>From c6ef62611c703011aa9aec21cf34620d65c90012 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston at google.com>
Date: Tue, 23 Apr 2024 20:50:51 +0000
Subject: [PATCH 3/3] Fix formatting
---
llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index 61fb9d2d4196cd..88b85234034038 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -949,7 +949,8 @@ void HWAddressSanitizer::instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
M, UseShortGranules
? Intrinsic::hwasan_check_memaccess_shortgranules_fixedshadow
: Intrinsic::hwasan_check_memaccess_fixedshadow),
- {Ptr, ConstantInt::get(Int32Ty, AccessInfo), ConstantInt::get(Int64Ty, Mapping.Offset)});
+ {Ptr, ConstantInt::get(Int32Ty, AccessInfo),
+ ConstantInt::get(Int64Ty, Mapping.Offset)});
else
IRB.CreateCall(Intrinsic::getDeclaration(
M, UseShortGranules
More information about the llvm-commits
mailing list