[compiler-rt] [llvm] [ASan] Correctly handle vectorized pointer sub/cmp for `invalid-pointer-pair` (PR #213546)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 16 17:09:03 PDT 2026
https://github.com/flovent updated https://github.com/llvm/llvm-project/pull/213546
>From 52414cc34e0e66d5607c7b4fb31477d5345132d6 Mon Sep 17 00:00:00 2001
From: flovent <flbven at protonmail.com>
Date: Sun, 2 Aug 2026 21:15:29 +0800
Subject: [PATCH 1/5] [ASan] Correctly handle vectorized pointer sub/cmp for
`invalid-pointer-pair`
Before this PR, asan will treat vector operands just like pointer and pass it to `__sanitizer_ptr_sub/__sanitizer_ptr_cmp(i64, i64)`, which leads to assertion failure because it doesn't matches the needed parameter type.
This PR extracts vector's elements and creates runtime call for each pair of them.
Closes #212453
---
.../invalid-pointer-pairs-vector-extract.cpp | 3 --
.../Instrumentation/AddressSanitizer.cpp | 35 +++++++++++++++----
.../asan-detect-invalid-pointer-pair.ll | 30 ++++++++++++++++
3 files changed, 58 insertions(+), 10 deletions(-)
diff --git a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
index 2eea39ba91e17..1861f7f88396c 100644
--- a/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
+++ b/compiler-rt/test/asan/TestCases/invalid-pointer-pairs-vector-extract.cpp
@@ -3,9 +3,6 @@
// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1:halt_on_error=0 %run %t 2>&1 | FileCheck %s
// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2:halt_on_error=0 %run %t 2>&1 | FileCheck %s
-// UNSUPPORTED: windows
-// XFAIL: *
-
#include <cstdint>
#include <stdio.h>
#include <stdlib.h>
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index d358318195265..dec1920c035e6 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -852,7 +852,7 @@ struct AddressSanitizer {
void instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
InterestingMemoryOperand &O, bool UseCalls,
const DataLayout &DL, RuntimeCallInserter &RTCI);
- void instrumentPointerComparisonOrSubtraction(Instruction *I,
+ bool instrumentPointerComparisonOrSubtraction(Instruction *I,
RuntimeCallInserter &RTCI);
void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
Value *Addr, MaybeAlign Alignment,
@@ -1698,16 +1698,38 @@ bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) {
return true;
}
-void AddressSanitizer::instrumentPointerComparisonOrSubtraction(
+bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
Instruction *I, RuntimeCallInserter &RTCI) {
IRBuilder<> IRB(I);
FunctionCallee F = isa<ICmpInst>(I) ? AsanPtrCmpFunction : AsanPtrSubFunction;
Value *Param[2] = {I->getOperand(0), I->getOperand(1)};
- for (Value *&i : Param) {
- if (i->getType()->isPointerTy())
- i = IRB.CreatePointerCast(i, IntptrTy);
+
+ if (const auto *Ty = Param[0]->getType(); Ty->isVectorTy()) {
+ const auto *VTy = dyn_cast<FixedVectorType>(Ty);
+ // Skip scalable vectors
+ if (!VTy)
+ return false;
+
+ assert(Param[1]->getType()->isVectorTy() &&
+ VTy->getElementCount() ==
+ cast<VectorType>(Param[1]->getType())->getElementCount() &&
+ "invalid vector pointer pair instrumentation operands");
+ for (unsigned Index = 0, NumElements = VTy->getNumElements();
+ Index != NumElements; ++Index) {
+ Value *ScalarParam[2] = {
+ IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)),
+ IRB.CreateExtractElement(Param[1], IRB.getInt32(Index))};
+ for (Value *&P : ScalarParam)
+ P = IRB.CreatePointerCast(P, IntptrTy);
+ RTCI.createRuntimeCall(IRB, F, ScalarParam);
+ }
+ return true;
}
+
+ for (Value *&P : Param)
+ P = IRB.CreatePointerCast(P, IntptrTy);
RTCI.createRuntimeCall(IRB, F, Param);
+ return true;
}
static void doInstrumentAddress(AddressSanitizer *Pass, Instruction *I,
@@ -3225,8 +3247,7 @@ bool AddressSanitizer::instrumentFunction(Function &F,
}
for (auto *Inst : PointerComparisonsOrSubtracts) {
- instrumentPointerComparisonOrSubtraction(Inst, RTCI);
- FunctionModified = true;
+ FunctionModified |= instrumentPointerComparisonOrSubtraction(Inst, RTCI);
}
if (ChangedStack || !NoReturnCalls.empty())
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
index daad076d0692b..7632a79436662 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
@@ -41,3 +41,33 @@ define i32 @mysub_ptrtoaddr(ptr %p, ptr %q) sanitize_address {
%w = trunc i64 %z to i32
ret i32 %w
}
+
+define <2 x i64> @mysub_vector(<2 x ptr> %p, <2 x ptr> %q) sanitize_address {
+; ALL-LABEL: @mysub_vector
+; NOSUB-NOT: call void @__sanitizer_ptr_sub
+ %x = ptrtoint <2 x ptr> %p to <2 x i64>
+ %y = ptrtoint <2 x ptr> %q to <2 x i64>
+; SUB: [[P0:%[0-9A-Za-z]+]] = extractelement <2 x i64> %x, i32 0
+; SUB: [[Q0:%[0-9A-Za-z]+]] = extractelement <2 x i64> %y, i32 0
+; SUB: call void @__sanitizer_ptr_sub(i64 [[P0]], i64 [[Q0]])
+; SUB: [[P1:%[0-9A-Za-z]+]] = extractelement <2 x i64> %x, i32 1
+; SUB: [[Q1:%[0-9A-Za-z]+]] = extractelement <2 x i64> %y, i32 1
+; SUB: call void @__sanitizer_ptr_sub(i64 [[P1]], i64 [[Q1]])
+ %z = sub <2 x i64> %x, %y
+ ret <2 x i64> %z
+}
+
+define <2 x i1> @mycmp_vector(<2 x ptr> %p, <2 x ptr> %q) sanitize_address {
+; ALL-LABEL: @mycmp_vector
+; NOCMP-NOT: call void @__sanitizer_ptr_cmp
+ %x = ptrtoint <2 x ptr> %p to <2 x i64>
+ %y = ptrtoint <2 x ptr> %q to <2 x i64>
+; CMP: [[P0:%[0-9A-Za-z]+]] = extractelement <2 x i64> %x, i32 0
+; CMP: [[Q0:%[0-9A-Za-z]+]] = extractelement <2 x i64> %y, i32 0
+; CMP: call void @__sanitizer_ptr_cmp(i64 [[P0]], i64 [[Q0]])
+; CMP: [[P1:%[0-9A-Za-z]+]] = extractelement <2 x i64> %x, i32 1
+; CMP: [[Q1:%[0-9A-Za-z]+]] = extractelement <2 x i64> %y, i32 1
+; CMP: call void @__sanitizer_ptr_cmp(i64 [[P1]], i64 [[Q1]])
+ %z = icmp ult <2 x i64> %x, %y
+ ret <2 x i1> %z
+}
>From 0c470560e53b834666dcbfcbc1abcf99172cdc14 Mon Sep 17 00:00:00 2001
From: flovent <flbven at protonmail.com>
Date: Sun, 16 Aug 2026 08:42:52 +0800
Subject: [PATCH 2/5] [NFC] Mark scalable vectors as TODO
---
llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index dec1920c035e6..6dbb490fbfb62 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -1706,7 +1706,7 @@ bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
if (const auto *Ty = Param[0]->getType(); Ty->isVectorTy()) {
const auto *VTy = dyn_cast<FixedVectorType>(Ty);
- // Skip scalable vectors
+ // TODO: Add support for scalable vectors if possible.
if (!VTy)
return false;
>From 43ffe3cc6a511bbc347a658e76c5338419bbbb18 Mon Sep 17 00:00:00 2001
From: flovent <flbven at protonmail.com>
Date: Sun, 16 Aug 2026 08:49:27 +0800
Subject: [PATCH 3/5] Better assertion
---
llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 6dbb490fbfb62..60ba26d02131a 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -1710,9 +1710,7 @@ bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
if (!VTy)
return false;
- assert(Param[1]->getType()->isVectorTy() &&
- VTy->getElementCount() ==
- cast<VectorType>(Param[1]->getType())->getElementCount() &&
+ assert(Param[0]->getType() == Param[1]->getType() &&
"invalid vector pointer pair instrumentation operands");
for (unsigned Index = 0, NumElements = VTy->getNumElements();
Index != NumElements; ++Index) {
>From 8a48e6dd8b8ad56bf9b17cb3bc0fba74a1095028 Mon Sep 17 00:00:00 2001
From: flovent <flbven at protonmail.com>
Date: Sun, 16 Aug 2026 09:00:13 +0800
Subject: [PATCH 4/5] [NFC] Fold for loop
---
llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 60ba26d02131a..e2a74d989f154 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -1715,10 +1715,8 @@ bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
for (unsigned Index = 0, NumElements = VTy->getNumElements();
Index != NumElements; ++Index) {
Value *ScalarParam[2] = {
- IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)),
- IRB.CreateExtractElement(Param[1], IRB.getInt32(Index))};
- for (Value *&P : ScalarParam)
- P = IRB.CreatePointerCast(P, IntptrTy);
+ IRB.CreatePointerCast(IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)), IntptrTy),
+ IRB.CreatePointerCast(IRB.CreateExtractElement(Param[1], IRB.getInt32(Index)), IntptrTy)};
RTCI.createRuntimeCall(IRB, F, ScalarParam);
}
return true;
>From 73ca0a30e0b942172dac8c6288a1d09c19223d4f Mon Sep 17 00:00:00 2001
From: flovent <flbven at protonmail.com>
Date: Sun, 16 Aug 2026 09:44:09 +0800
Subject: [PATCH 5/5] [NFC] Code format
---
llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index e2a74d989f154..06d60003631d0 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -1715,8 +1715,12 @@ bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
for (unsigned Index = 0, NumElements = VTy->getNumElements();
Index != NumElements; ++Index) {
Value *ScalarParam[2] = {
- IRB.CreatePointerCast(IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)), IntptrTy),
- IRB.CreatePointerCast(IRB.CreateExtractElement(Param[1], IRB.getInt32(Index)), IntptrTy)};
+ IRB.CreatePointerCast(
+ IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)),
+ IntptrTy),
+ IRB.CreatePointerCast(
+ IRB.CreateExtractElement(Param[1], IRB.getInt32(Index)),
+ IntptrTy)};
RTCI.createRuntimeCall(IRB, F, ScalarParam);
}
return true;
More information about the llvm-commits
mailing list