[compiler-rt] [llvm] [ASan] Correctly handle vectorized pointer sub/cmp for `invalid-pointer-pair` (PR #213546)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 2 06:37:13 PDT 2026
https://github.com/flovent created https://github.com/llvm/llvm-project/pull/213546
Before this PR, asan will treat vector operands just like pointer and pass it to `__sanitizer_ptr_sub/__sanitizer_ptr_cmp(i64, i64)`,
which leads to assertion failure because it doesn't matches the needed parameter type.
This PR extracts vector's elements and creates runtime call for each pair of them.
Closes #212453 .
>From add1064412d722c630f9aa79a6a7a753fc1a467c Mon Sep 17 00:00:00 2001
From: flovent <flbven at protonmail.com>
Date: Sun, 2 Aug 2026 21:15:29 +0800
Subject: [PATCH 1/2] [ASan] Correctly handle vectorized pointer sub/cmp for
`invalid-pointer-pair`
Before this PR, asan will treat vector operands just like pointer and pass it to `__sanitizer_ptr_sub/__sanitizer_ptr_cmp(i64, i64)`, which leads to assertion failure because it doesn't matches the needed parameter type.
This PR extracts vector's elements and creates runtime call for each pair of them.
Closes #212453
---
.../invalid-pointer-pairs-vector-extract.cpp | 16 +++++++++
.../Instrumentation/AddressSanitizer.cpp | 35 +++++++++++++++----
.../asan-detect-invalid-pointer-pair.ll | 30 ++++++++++++++++
3 files changed, 74 insertions(+), 7 deletions(-)
create mode 100644 compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
diff --git a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
new file mode 100644
index 0000000000000..c6c69dd5c3470
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
@@ -0,0 +1,16 @@
+// RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
+
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1:halt_on_error=0 %run %t
+
+int main() {
+ const char* begins[2] = {"abc", "def"};
+ const char* ends[2] = {"abcde", "defgh"};
+ long lengths[2] {0};
+ // CHECK: ERROR: AddressSanitizer: invalid-pointer-pair
+ // CHECK: #{{[0-9]+ .*}} in main {{.*}}invalid-pointer-pairs-vector-extract.cpp:[[@LINE+1]]
+ lengths[0] = ends[0] - begins[0];
+ // CHECK: ERROR: AddressSanitizer: invalid-pointer-pair
+ // CHECK: #{{[0-9]+ .*}} in main {{.*}}invalid-pointer-pairs-vector-extract.cpp:[[@LINE+1]]
+ lengths[1] = ends[1] - begins[1];
+ return 0;
+}
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 2fae7d8c15ff3..0dea7da7c7a20 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -852,7 +852,7 @@ struct AddressSanitizer {
void instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
InterestingMemoryOperand &O, bool UseCalls,
const DataLayout &DL, RuntimeCallInserter &RTCI);
- void instrumentPointerComparisonOrSubtraction(Instruction *I,
+ bool instrumentPointerComparisonOrSubtraction(Instruction *I,
RuntimeCallInserter &RTCI);
void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
Value *Addr, MaybeAlign Alignment,
@@ -1698,16 +1698,38 @@ bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) {
return true;
}
-void AddressSanitizer::instrumentPointerComparisonOrSubtraction(
+bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
Instruction *I, RuntimeCallInserter &RTCI) {
IRBuilder<> IRB(I);
FunctionCallee F = isa<ICmpInst>(I) ? AsanPtrCmpFunction : AsanPtrSubFunction;
Value *Param[2] = {I->getOperand(0), I->getOperand(1)};
- for (Value *&i : Param) {
- if (i->getType()->isPointerTy())
- i = IRB.CreatePointerCast(i, IntptrTy);
+
+ if (const auto *Ty = Param[0]->getType(); Ty->isVectorTy()) {
+ const auto *VTy = dyn_cast<FixedVectorType>(Ty);
+ // Skip scalable vectors
+ if (!VTy)
+ return false;
+
+ assert(Param[1]->getType()->isVectorTy() &&
+ VTy->getElementCount() ==
+ cast<VectorType>(Param[1]->getType())->getElementCount() &&
+ "invalid vector pointer pair instrumentation operands");
+ for (unsigned Index = 0, NumElements = VTy->getNumElements();
+ Index != NumElements; ++Index) {
+ Value *ScalarParam[2] = {
+ IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)),
+ IRB.CreateExtractElement(Param[1], IRB.getInt32(Index))};
+ for (Value *&P : ScalarParam)
+ P = IRB.CreatePointerCast(P, IntptrTy);
+ RTCI.createRuntimeCall(IRB, F, ScalarParam);
+ }
+ return true;
}
+
+ for (Value *&P : Param)
+ P = IRB.CreatePointerCast(P, IntptrTy);
RTCI.createRuntimeCall(IRB, F, Param);
+ return true;
}
static void doInstrumentAddress(AddressSanitizer *Pass, Instruction *I,
@@ -3225,8 +3247,7 @@ bool AddressSanitizer::instrumentFunction(Function &F,
}
for (auto *Inst : PointerComparisonsOrSubtracts) {
- instrumentPointerComparisonOrSubtraction(Inst, RTCI);
- FunctionModified = true;
+ FunctionModified |= instrumentPointerComparisonOrSubtraction(Inst, RTCI);
}
if (ChangedStack || !NoReturnCalls.empty())
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
index daad076d0692b..7632a79436662 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
@@ -41,3 +41,33 @@ define i32 @mysub_ptrtoaddr(ptr %p, ptr %q) sanitize_address {
%w = trunc i64 %z to i32
ret i32 %w
}
+
+define <2 x i64> @mysub_vector(<2 x ptr> %p, <2 x ptr> %q) sanitize_address {
+; ALL-LABEL: @mysub_vector
+; NOSUB-NOT: call void @__sanitizer_ptr_sub
+ %x = ptrtoint <2 x ptr> %p to <2 x i64>
+ %y = ptrtoint <2 x ptr> %q to <2 x i64>
+; SUB: [[P0:%[0-9A-Za-z]+]] = extractelement <2 x i64> %x, i32 0
+; SUB: [[Q0:%[0-9A-Za-z]+]] = extractelement <2 x i64> %y, i32 0
+; SUB: call void @__sanitizer_ptr_sub(i64 [[P0]], i64 [[Q0]])
+; SUB: [[P1:%[0-9A-Za-z]+]] = extractelement <2 x i64> %x, i32 1
+; SUB: [[Q1:%[0-9A-Za-z]+]] = extractelement <2 x i64> %y, i32 1
+; SUB: call void @__sanitizer_ptr_sub(i64 [[P1]], i64 [[Q1]])
+ %z = sub <2 x i64> %x, %y
+ ret <2 x i64> %z
+}
+
+define <2 x i1> @mycmp_vector(<2 x ptr> %p, <2 x ptr> %q) sanitize_address {
+; ALL-LABEL: @mycmp_vector
+; NOCMP-NOT: call void @__sanitizer_ptr_cmp
+ %x = ptrtoint <2 x ptr> %p to <2 x i64>
+ %y = ptrtoint <2 x ptr> %q to <2 x i64>
+; CMP: [[P0:%[0-9A-Za-z]+]] = extractelement <2 x i64> %x, i32 0
+; CMP: [[Q0:%[0-9A-Za-z]+]] = extractelement <2 x i64> %y, i32 0
+; CMP: call void @__sanitizer_ptr_cmp(i64 [[P0]], i64 [[Q0]])
+; CMP: [[P1:%[0-9A-Za-z]+]] = extractelement <2 x i64> %x, i32 1
+; CMP: [[Q1:%[0-9A-Za-z]+]] = extractelement <2 x i64> %y, i32 1
+; CMP: call void @__sanitizer_ptr_cmp(i64 [[P1]], i64 [[Q1]])
+ %z = icmp ult <2 x i64> %x, %y
+ ret <2 x i1> %z
+}
>From 8bb9d819de928740c9c50852a092cb4fd8cdcea2 Mon Sep 17 00:00:00 2001
From: flovent <flbven at protonmail.com>
Date: Sun, 2 Aug 2026 21:36:36 +0800
Subject: [PATCH 2/2] O2
---
.../asan/TestCases/invalid-pointer-pairs-vector-extract.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
index c6c69dd5c3470..328f64000658c 100644
--- a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
+++ b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
@@ -1,4 +1,4 @@
-// RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
+// RUN: %clangxx_asan -O2 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1:halt_on_error=0 %run %t
More information about the llvm-commits
mailing list