[llvm] [ASan] Convert pointer-pair operands based on type (PR #218494)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 25 23:18:51 PDT 2026
https://github.com/im-lunex updated https://github.com/llvm/llvm-project/pull/218494
>From e0cbdd99feddf9f70e7d15177eb9b0e08df219e8 Mon Sep 17 00:00:00 2001
From: im-lunex <thisissamir04 at gmail.com>
Date: Tue, 25 Aug 2026 00:56:51 +0600
Subject: [PATCH 1/2] [ASan] Convert pointer-pair operands based on type
---
.../Instrumentation/AddressSanitizer.cpp | 18 +++++--
.../asan-detect-invalid-pointer-pair.ll | 48 +++++++++++++++++++
2 files changed, 61 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 06d60003631d0..0d7ed944dc5a9 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -1698,6 +1698,14 @@ bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) {
return true;
}
+static Value *convertToIntptr(IRBuilder<> &IRB, Value *V, Type *IntptrTy) {
+ if (V->getType()->isPtrOrPtrVectorTy())
+ return IRB.CreatePointerCast(V, IntptrTy);
+ assert(V->getType()->isIntOrIntVectorTy() &&
+ "unexpected pointer-pair operand type");
+ return IRB.CreateZExtOrTrunc(V, IntptrTy);
+}
+
bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
Instruction *I, RuntimeCallInserter &RTCI) {
IRBuilder<> IRB(I);
@@ -1715,11 +1723,11 @@ bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
for (unsigned Index = 0, NumElements = VTy->getNumElements();
Index != NumElements; ++Index) {
Value *ScalarParam[2] = {
- IRB.CreatePointerCast(
- IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)),
+ convertToIntptr(
+ IRB, IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)),
IntptrTy),
- IRB.CreatePointerCast(
- IRB.CreateExtractElement(Param[1], IRB.getInt32(Index)),
+ convertToIntptr(
+ IRB, IRB.CreateExtractElement(Param[1], IRB.getInt32(Index)),
IntptrTy)};
RTCI.createRuntimeCall(IRB, F, ScalarParam);
}
@@ -1727,7 +1735,7 @@ bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
}
for (Value *&P : Param)
- P = IRB.CreatePointerCast(P, IntptrTy);
+ P = convertToIntptr(IRB, P, IntptrTy);
RTCI.createRuntimeCall(IRB, F, Param);
return true;
}
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
index 7632a79436662..6c340f64e4ced 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
@@ -71,3 +71,51 @@ define <2 x i1> @mycmp_vector(<2 x ptr> %p, <2 x ptr> %q) sanitize_address {
%z = icmp ult <2 x i64> %x, %y
ret <2 x i1> %z
}
+
+define i32 @mysub_ptrtoint_trunc(ptr %p, ptr %q) sanitize_address {
+; ALL-LABEL: @mysub_ptrtoint_trunc
+; NOSUB-NOT: call void @__sanitizer_ptr_sub
+; SUB: [[P:%[0-9A-Za-z]+]] = ptrtoint ptr %p to i32
+; SUB: [[Q:%[0-9A-Za-z]+]] = ptrtoint ptr %q to i32
+ %x = ptrtoint ptr %p to i32
+ %y = ptrtoint ptr %q to i32
+ %z = sub i32 %x, %y
+; SUB: [[XP:%[0-9A-Za-z]+]] = zext i32 [[P]] to i64
+; SUB: [[XQ:%[0-9A-Za-z]+]] = zext i32 [[Q]] to i64
+; SUB: call void @__sanitizer_ptr_sub(i64 [[XP]], i64 [[XQ]])
+ ret i32 %z
+}
+
+define <2 x i32> @mysub_vector_ptrtoint_trunc(<2 x ptr> %p, <2 x ptr> %q) sanitize_address {
+; ALL-LABEL: @mysub_vector_ptrtoint_trunc
+; NOSUB-NOT: call void @__sanitizer_ptr_sub
+ %x = ptrtoint <2 x ptr> %p to <2 x i32>
+ %y = ptrtoint <2 x ptr> %q to <2 x i32>
+; SUB: [[X0:%[0-9A-Za-z]+]] = extractelement <2 x i32> %x, i32 0
+; SUB: [[ZX0:%[0-9A-Za-z]+]] = zext i32 [[X0]] to i64
+; SUB: [[Y0:%[0-9A-Za-z]+]] = extractelement <2 x i32> %y, i32 0
+; SUB: [[ZY0:%[0-9A-Za-z]+]] = zext i32 [[Y0]] to i64
+; SUB: call void @__sanitizer_ptr_sub(i64 [[ZX0]], i64 [[ZY0]])
+; SUB: [[X1:%[0-9A-Za-z]+]] = extractelement <2 x i32> %x, i32 1
+; SUB: [[ZX1:%[0-9A-Za-z]+]] = zext i32 [[X1]] to i64
+; SUB: [[Y1:%[0-9A-Za-z]+]] = extractelement <2 x i32> %y, i32 1
+; SUB: [[ZY1:%[0-9A-Za-z]+]] = zext i32 [[Y1]] to i64
+; SUB: call void @__sanitizer_ptr_sub(i64 [[ZX1]], i64 [[ZY1]])
+ %z = sub <2 x i32> %x, %y
+ ret <2 x i32> %z
+}
+
+define i128 @mysub_ptrtoint_widen(ptr %p, ptr %q) sanitize_address {
+; ALL-LABEL: @mysub_ptrtoint_widen
+; NOSUB-NOT: call void @__sanitizer_ptr_sub
+; SUB: [[P:%[0-9A-Za-z]+]] = ptrtoint ptr %p to i128
+; SUB: [[Q:%[0-9A-Za-z]+]] = ptrtoint ptr %q to i128
+ %x = ptrtoint ptr %p to i128
+ %y = ptrtoint ptr %q to i128
+; SUB-NOT: zext i128
+; SUB: [[XP:%[0-9A-Za-z]+]] = trunc i128 [[P]] to i64
+; SUB: [[XQ:%[0-9A-Za-z]+]] = trunc i128 [[Q]] to i64
+; SUB: call void @__sanitizer_ptr_sub(i64 [[XP]], i64 [[XQ]])
+ %z = sub i128 %x, %y
+ ret i128 %z
+}
>From caeb82009d8bf85a0e946af9c17ae4bdf6a5097f Mon Sep 17 00:00:00 2001
From: im-lunex <thisissamir04 at gmail.com>
Date: Wed, 26 Aug 2026 12:15:41 +0600
Subject: [PATCH 2/2] skip non-pointer-width pointer-pair operands
---
.../Instrumentation/AddressSanitizer.cpp | 28 +++++++++++--------
.../asan-detect-invalid-pointer-pair.ll | 27 ++----------------
2 files changed, 19 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 0d7ed944dc5a9..3bcd6fcf90396 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -1698,19 +1698,23 @@ bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) {
return true;
}
-static Value *convertToIntptr(IRBuilder<> &IRB, Value *V, Type *IntptrTy) {
- if (V->getType()->isPtrOrPtrVectorTy())
- return IRB.CreatePointerCast(V, IntptrTy);
- assert(V->getType()->isIntOrIntVectorTy() &&
- "unexpected pointer-pair operand type");
- return IRB.CreateZExtOrTrunc(V, IntptrTy);
+static bool isPointerPairOperand(Value *V, Type *IntptrTy) {
+ Type *Ty = V->getType();
+ if (Ty->isPtrOrPtrVectorTy())
+ return true;
+ return Ty->isIntOrIntVectorTy() &&
+ Ty->getScalarSizeInBits() == IntptrTy->getScalarSizeInBits();
}
bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
Instruction *I, RuntimeCallInserter &RTCI) {
+ Value *Param[2] = {I->getOperand(0), I->getOperand(1)};
+ if (!isPointerPairOperand(Param[0], IntptrTy) ||
+ !isPointerPairOperand(Param[1], IntptrTy))
+ return false;
+
IRBuilder<> IRB(I);
FunctionCallee F = isa<ICmpInst>(I) ? AsanPtrCmpFunction : AsanPtrSubFunction;
- Value *Param[2] = {I->getOperand(0), I->getOperand(1)};
if (const auto *Ty = Param[0]->getType(); Ty->isVectorTy()) {
const auto *VTy = dyn_cast<FixedVectorType>(Ty);
@@ -1723,11 +1727,11 @@ bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
for (unsigned Index = 0, NumElements = VTy->getNumElements();
Index != NumElements; ++Index) {
Value *ScalarParam[2] = {
- convertToIntptr(
- IRB, IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)),
+ IRB.CreatePointerCast(
+ IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)),
IntptrTy),
- convertToIntptr(
- IRB, IRB.CreateExtractElement(Param[1], IRB.getInt32(Index)),
+ IRB.CreatePointerCast(
+ IRB.CreateExtractElement(Param[1], IRB.getInt32(Index)),
IntptrTy)};
RTCI.createRuntimeCall(IRB, F, ScalarParam);
}
@@ -1735,7 +1739,7 @@ bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
}
for (Value *&P : Param)
- P = convertToIntptr(IRB, P, IntptrTy);
+ P = IRB.CreatePointerCast(P, IntptrTy);
RTCI.createRuntimeCall(IRB, F, Param);
return true;
}
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
index 6c340f64e4ced..b8cc2b30343fd 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
@@ -74,48 +74,27 @@ define <2 x i1> @mycmp_vector(<2 x ptr> %p, <2 x ptr> %q) sanitize_address {
define i32 @mysub_ptrtoint_trunc(ptr %p, ptr %q) sanitize_address {
; ALL-LABEL: @mysub_ptrtoint_trunc
-; NOSUB-NOT: call void @__sanitizer_ptr_sub
-; SUB: [[P:%[0-9A-Za-z]+]] = ptrtoint ptr %p to i32
-; SUB: [[Q:%[0-9A-Za-z]+]] = ptrtoint ptr %q to i32
+; ALL-NOT: call void @__sanitizer_ptr_sub
%x = ptrtoint ptr %p to i32
%y = ptrtoint ptr %q to i32
%z = sub i32 %x, %y
-; SUB: [[XP:%[0-9A-Za-z]+]] = zext i32 [[P]] to i64
-; SUB: [[XQ:%[0-9A-Za-z]+]] = zext i32 [[Q]] to i64
-; SUB: call void @__sanitizer_ptr_sub(i64 [[XP]], i64 [[XQ]])
ret i32 %z
}
define <2 x i32> @mysub_vector_ptrtoint_trunc(<2 x ptr> %p, <2 x ptr> %q) sanitize_address {
; ALL-LABEL: @mysub_vector_ptrtoint_trunc
-; NOSUB-NOT: call void @__sanitizer_ptr_sub
+; ALL-NOT: call void @__sanitizer_ptr_sub
%x = ptrtoint <2 x ptr> %p to <2 x i32>
%y = ptrtoint <2 x ptr> %q to <2 x i32>
-; SUB: [[X0:%[0-9A-Za-z]+]] = extractelement <2 x i32> %x, i32 0
-; SUB: [[ZX0:%[0-9A-Za-z]+]] = zext i32 [[X0]] to i64
-; SUB: [[Y0:%[0-9A-Za-z]+]] = extractelement <2 x i32> %y, i32 0
-; SUB: [[ZY0:%[0-9A-Za-z]+]] = zext i32 [[Y0]] to i64
-; SUB: call void @__sanitizer_ptr_sub(i64 [[ZX0]], i64 [[ZY0]])
-; SUB: [[X1:%[0-9A-Za-z]+]] = extractelement <2 x i32> %x, i32 1
-; SUB: [[ZX1:%[0-9A-Za-z]+]] = zext i32 [[X1]] to i64
-; SUB: [[Y1:%[0-9A-Za-z]+]] = extractelement <2 x i32> %y, i32 1
-; SUB: [[ZY1:%[0-9A-Za-z]+]] = zext i32 [[Y1]] to i64
-; SUB: call void @__sanitizer_ptr_sub(i64 [[ZX1]], i64 [[ZY1]])
%z = sub <2 x i32> %x, %y
ret <2 x i32> %z
}
define i128 @mysub_ptrtoint_widen(ptr %p, ptr %q) sanitize_address {
; ALL-LABEL: @mysub_ptrtoint_widen
-; NOSUB-NOT: call void @__sanitizer_ptr_sub
-; SUB: [[P:%[0-9A-Za-z]+]] = ptrtoint ptr %p to i128
-; SUB: [[Q:%[0-9A-Za-z]+]] = ptrtoint ptr %q to i128
+; ALL-NOT: call void @__sanitizer_ptr_sub
%x = ptrtoint ptr %p to i128
%y = ptrtoint ptr %q to i128
-; SUB-NOT: zext i128
-; SUB: [[XP:%[0-9A-Za-z]+]] = trunc i128 [[P]] to i64
-; SUB: [[XQ:%[0-9A-Za-z]+]] = trunc i128 [[Q]] to i64
-; SUB: call void @__sanitizer_ptr_sub(i64 [[XP]], i64 [[XQ]])
%z = sub i128 %x, %y
ret i128 %z
}
More information about the llvm-commits
mailing list