[compiler-rt] [llvm] [ASan] Correctly handle vectorized pointer sub/cmp for `invalid-pointer-pair` (PR #213546)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 15 01:46:36 PDT 2026
https://github.com/flovent updated https://github.com/llvm/llvm-project/pull/213546
>From a124f3c3bc0dcc95f9d11d3c222d26203dfe01c6 Mon Sep 17 00:00:00 2001
From: flovent <flbven at protonmail.com>
Date: Sun, 2 Aug 2026 21:15:29 +0800
Subject: [PATCH 1/5] [ASan] Correctly handle vectorized pointer sub/cmp for
`invalid-pointer-pair`
Before this PR, asan will treat vector operands just like pointer and pass it to `__sanitizer_ptr_sub/__sanitizer_ptr_cmp(i64, i64)`, which leads to assertion failure because it doesn't matches the needed parameter type.
This PR extracts vector's elements and creates runtime call for each pair of them.
Closes #212453
---
.../invalid-pointer-pairs-vector-extract.cpp | 16 +++++++++
.../Instrumentation/AddressSanitizer.cpp | 35 +++++++++++++++----
.../asan-detect-invalid-pointer-pair.ll | 30 ++++++++++++++++
3 files changed, 74 insertions(+), 7 deletions(-)
create mode 100644 compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
diff --git a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
new file mode 100644
index 0000000000000..c6c69dd5c3470
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
@@ -0,0 +1,16 @@
+// RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
+
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1:halt_on_error=0 %run %t
+
+int main() {
+ const char* begins[2] = {"abc", "def"};
+ const char* ends[2] = {"abcde", "defgh"};
+ long lengths[2] {0};
+ // CHECK: ERROR: AddressSanitizer: invalid-pointer-pair
+ // CHECK: #{{[0-9]+ .*}} in main {{.*}}invalid-pointer-pairs-vector-extract.cpp:[[@LINE+1]]
+ lengths[0] = ends[0] - begins[0];
+ // CHECK: ERROR: AddressSanitizer: invalid-pointer-pair
+ // CHECK: #{{[0-9]+ .*}} in main {{.*}}invalid-pointer-pairs-vector-extract.cpp:[[@LINE+1]]
+ lengths[1] = ends[1] - begins[1];
+ return 0;
+}
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index d358318195265..dec1920c035e6 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -852,7 +852,7 @@ struct AddressSanitizer {
void instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
InterestingMemoryOperand &O, bool UseCalls,
const DataLayout &DL, RuntimeCallInserter &RTCI);
- void instrumentPointerComparisonOrSubtraction(Instruction *I,
+ bool instrumentPointerComparisonOrSubtraction(Instruction *I,
RuntimeCallInserter &RTCI);
void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
Value *Addr, MaybeAlign Alignment,
@@ -1698,16 +1698,38 @@ bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) {
return true;
}
-void AddressSanitizer::instrumentPointerComparisonOrSubtraction(
+bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
Instruction *I, RuntimeCallInserter &RTCI) {
IRBuilder<> IRB(I);
FunctionCallee F = isa<ICmpInst>(I) ? AsanPtrCmpFunction : AsanPtrSubFunction;
Value *Param[2] = {I->getOperand(0), I->getOperand(1)};
- for (Value *&i : Param) {
- if (i->getType()->isPointerTy())
- i = IRB.CreatePointerCast(i, IntptrTy);
+
+ if (const auto *Ty = Param[0]->getType(); Ty->isVectorTy()) {
+ const auto *VTy = dyn_cast<FixedVectorType>(Ty);
+ // Skip scalable vectors
+ if (!VTy)
+ return false;
+
+ assert(Param[1]->getType()->isVectorTy() &&
+ VTy->getElementCount() ==
+ cast<VectorType>(Param[1]->getType())->getElementCount() &&
+ "invalid vector pointer pair instrumentation operands");
+ for (unsigned Index = 0, NumElements = VTy->getNumElements();
+ Index != NumElements; ++Index) {
+ Value *ScalarParam[2] = {
+ IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)),
+ IRB.CreateExtractElement(Param[1], IRB.getInt32(Index))};
+ for (Value *&P : ScalarParam)
+ P = IRB.CreatePointerCast(P, IntptrTy);
+ RTCI.createRuntimeCall(IRB, F, ScalarParam);
+ }
+ return true;
}
+
+ for (Value *&P : Param)
+ P = IRB.CreatePointerCast(P, IntptrTy);
RTCI.createRuntimeCall(IRB, F, Param);
+ return true;
}
static void doInstrumentAddress(AddressSanitizer *Pass, Instruction *I,
@@ -3225,8 +3247,7 @@ bool AddressSanitizer::instrumentFunction(Function &F,
}
for (auto *Inst : PointerComparisonsOrSubtracts) {
- instrumentPointerComparisonOrSubtraction(Inst, RTCI);
- FunctionModified = true;
+ FunctionModified |= instrumentPointerComparisonOrSubtraction(Inst, RTCI);
}
if (ChangedStack || !NoReturnCalls.empty())
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
index daad076d0692b..7632a79436662 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
@@ -41,3 +41,33 @@ define i32 @mysub_ptrtoaddr(ptr %p, ptr %q) sanitize_address {
%w = trunc i64 %z to i32
ret i32 %w
}
+
+define <2 x i64> @mysub_vector(<2 x ptr> %p, <2 x ptr> %q) sanitize_address {
+; ALL-LABEL: @mysub_vector
+; NOSUB-NOT: call void @__sanitizer_ptr_sub
+ %x = ptrtoint <2 x ptr> %p to <2 x i64>
+ %y = ptrtoint <2 x ptr> %q to <2 x i64>
+; SUB: [[P0:%[0-9A-Za-z]+]] = extractelement <2 x i64> %x, i32 0
+; SUB: [[Q0:%[0-9A-Za-z]+]] = extractelement <2 x i64> %y, i32 0
+; SUB: call void @__sanitizer_ptr_sub(i64 [[P0]], i64 [[Q0]])
+; SUB: [[P1:%[0-9A-Za-z]+]] = extractelement <2 x i64> %x, i32 1
+; SUB: [[Q1:%[0-9A-Za-z]+]] = extractelement <2 x i64> %y, i32 1
+; SUB: call void @__sanitizer_ptr_sub(i64 [[P1]], i64 [[Q1]])
+ %z = sub <2 x i64> %x, %y
+ ret <2 x i64> %z
+}
+
+define <2 x i1> @mycmp_vector(<2 x ptr> %p, <2 x ptr> %q) sanitize_address {
+; ALL-LABEL: @mycmp_vector
+; NOCMP-NOT: call void @__sanitizer_ptr_cmp
+ %x = ptrtoint <2 x ptr> %p to <2 x i64>
+ %y = ptrtoint <2 x ptr> %q to <2 x i64>
+; CMP: [[P0:%[0-9A-Za-z]+]] = extractelement <2 x i64> %x, i32 0
+; CMP: [[Q0:%[0-9A-Za-z]+]] = extractelement <2 x i64> %y, i32 0
+; CMP: call void @__sanitizer_ptr_cmp(i64 [[P0]], i64 [[Q0]])
+; CMP: [[P1:%[0-9A-Za-z]+]] = extractelement <2 x i64> %x, i32 1
+; CMP: [[Q1:%[0-9A-Za-z]+]] = extractelement <2 x i64> %y, i32 1
+; CMP: call void @__sanitizer_ptr_cmp(i64 [[P1]], i64 [[Q1]])
+ %z = icmp ult <2 x i64> %x, %y
+ ret <2 x i1> %z
+}
>From 5b8c5520f5002b8b41bcdcb8f3cc567f9bfbdabc Mon Sep 17 00:00:00 2001
From: flovent <flbven at protonmail.com>
Date: Sun, 2 Aug 2026 21:36:36 +0800
Subject: [PATCH 2/5] O2
---
.../asan/TestCases/invalid-pointer-pairs-vector-extract.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
index c6c69dd5c3470..328f64000658c 100644
--- a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
+++ b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
@@ -1,4 +1,4 @@
-// RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
+// RUN: %clangxx_asan -O2 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1:halt_on_error=0 %run %t
>From f3ad625e2d25e96eb20c0d89214b679ab423ed25 Mon Sep 17 00:00:00 2001
From: flovent <flbven at protonmail.com>
Date: Sun, 2 Aug 2026 21:53:35 +0800
Subject: [PATCH 3/5] [NFC] clang-format test file
---
.../asan/TestCases/invalid-pointer-pairs-vector-extract.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
index 328f64000658c..f18b1e5a6c807 100644
--- a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
+++ b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
@@ -3,9 +3,9 @@
// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1:halt_on_error=0 %run %t
int main() {
- const char* begins[2] = {"abc", "def"};
- const char* ends[2] = {"abcde", "defgh"};
- long lengths[2] {0};
+ const char *begins[2] = {"abc", "def"};
+ const char *ends[2] = {"abcde", "defgh"};
+ long lengths[2]{0};
// CHECK: ERROR: AddressSanitizer: invalid-pointer-pair
// CHECK: #{{[0-9]+ .*}} in main {{.*}}invalid-pointer-pairs-vector-extract.cpp:[[@LINE+1]]
lengths[0] = ends[0] - begins[0];
>From ffa59a3bc2258034246095e8f8125dacd703f5b1 Mon Sep 17 00:00:00 2001
From: flovent <flbven at protonmail.com>
Date: Sat, 15 Aug 2026 16:43:05 +0800
Subject: [PATCH 4/5] Correct test case
---
.../invalid-pointer-pairs-vector-extract.cpp | 28 ++++++++++++++-----
1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
index f18b1e5a6c807..dca8ef549aa44 100644
--- a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
+++ b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
@@ -1,16 +1,30 @@
// RUN: %clangxx_asan -O2 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1:halt_on_error=0 %run %t 2>&1 | FileCheck %s
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2:halt_on_error=0 %run %t 2>&1 | FileCheck %s
-// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1:halt_on_error=0 %run %t
+#include <stdio.h>
+#include <stdlib.h>
-int main() {
- const char *begins[2] = {"abc", "def"};
- const char *ends[2] = {"abcde", "defgh"};
- long lengths[2]{0};
+__attribute__((noinline)) void span_lengths(const char **begins,
+ const char **ends,
+ long *lengths) {
// CHECK: ERROR: AddressSanitizer: invalid-pointer-pair
- // CHECK: #{{[0-9]+ .*}} in main {{.*}}invalid-pointer-pairs-vector-extract.cpp:[[@LINE+1]]
+ // CHECK: #{{[0-9]+ .*}} in span_lengths
lengths[0] = ends[0] - begins[0];
// CHECK: ERROR: AddressSanitizer: invalid-pointer-pair
- // CHECK: #{{[0-9]+ .*}} in main {{.*}}invalid-pointer-pairs-vector-extract.cpp:[[@LINE+1]]
+ // CHECK: #{{[0-9]+ .*}} in span_lengths
lengths[1] = ends[1] - begins[1];
+}
+
+int main() {
+ char *a = (char *)malloc(16);
+ char *b = (char *)malloc(16);
+ const char *begins[2] = {a, b};
+ const char *ends[2] = {b + 2, a + 10};
+ long lengths[2] = {0};
+ span_lengths(begins, ends, lengths);
+ fprintf(stderr, "lengths: %ld %ld\n", lengths[0], lengths[1]);
+ free(a);
+ free(b);
return 0;
}
>From 0b1dc0d5d5600fb5ce39fa273100ec500eddb152 Mon Sep 17 00:00:00 2001
From: flovent <flbven at protonmail.com>
Date: Sat, 15 Aug 2026 16:46:20 +0800
Subject: [PATCH 5/5] [NFC] Code format
---
.../asan/TestCases/invalid-pointer-pairs-vector-extract.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
index dca8ef549aa44..6381175e92816 100644
--- a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
+++ b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
@@ -6,8 +6,7 @@
#include <stdlib.h>
__attribute__((noinline)) void span_lengths(const char **begins,
- const char **ends,
- long *lengths) {
+ const char **ends, long *lengths) {
// CHECK: ERROR: AddressSanitizer: invalid-pointer-pair
// CHECK: #{{[0-9]+ .*}} in span_lengths
lengths[0] = ends[0] - begins[0];
More information about the llvm-commits
mailing list